From 5053f1ea83194ecb0a5693ad5dada5b25bf0f7e6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 Apr 2022 11:03:36 -0700 Subject: [PATCH 0001/1172] WIP --- datasette/views/table.py | 1750 ++++++++++++++++++++++++-------------- 1 file changed, 1089 insertions(+), 661 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index dc85165e..988afd25 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,16 +1,26 @@ +from __future__ import annotations +from typing import TYPE_CHECKING +import asyncio +from asyncinject import Registry +from typing import List, Tuple import urllib import itertools +import time import json import markupsafe +from datasette import Request, Response, __version__ from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette.utils import ( + add_cors_headers, await_me_maybe, CustomRow, MultiParams, + InvalidSql, append_querystring, + call_with_supported_arguments, compound_keys_after_sql, format_bytes, tilde_decode, @@ -24,15 +34,19 @@ from datasette.utils import ( path_with_format, path_with_removed_args, path_with_replaced_args, + sqlite3, to_css_class, urlsafe_components, value_as_boolean, ) from datasette.utils.asgi import BadRequest, NotFound from datasette.filters import Filters -from .base import DataView, DatasetteError, ureg +from .base import DataView, DatasetteError, ureg, BaseView from .database import QueryView +if TYPE_CHECKING: + from datasette.app import Datasette, Database + LINK_WITH_LABEL = ( '{label} {id}' ) @@ -273,666 +287,6 @@ class RowTableShared(DataView): return columns, cell_rows -class TableView(RowTableShared): - name = "table" - - async def post(self, request): - database_route = tilde_decode(request.url_vars["database"]) - try: - db = self.ds.get_database(route=database_route) - except KeyError: - raise NotFound("Database not found: {}".format(database_route)) - database = db.name - table = tilde_decode(request.url_vars["table"]) - # Handle POST to a canned query - canned_query = await self.ds.get_canned_query(database, table, request.actor) - assert canned_query, "You may only POST to a canned query" - return await QueryView(self.ds).data( - request, - canned_query["sql"], - metadata=canned_query, - editable=False, - canned_query=table, - named_parameters=canned_query.get("params"), - write=bool(canned_query.get("write")), - ) - - async def columns_to_select(self, table_columns, pks, request): - columns = list(table_columns) - if "_col" in request.args: - columns = list(pks) - _cols = request.args.getlist("_col") - bad_columns = [column for column in _cols if column not in table_columns] - if bad_columns: - raise DatasetteError( - "_col={} - invalid columns".format(", ".join(bad_columns)), - status=400, - ) - # De-duplicate maintaining order: - columns.extend(dict.fromkeys(_cols)) - if "_nocol" in request.args: - # Return all columns EXCEPT these - bad_columns = [ - column - for column in request.args.getlist("_nocol") - if (column not in table_columns) or (column in pks) - ] - if bad_columns: - raise DatasetteError( - "_nocol={} - invalid columns".format(", ".join(bad_columns)), - status=400, - ) - tmp_columns = [ - column - for column in columns - if column not in request.args.getlist("_nocol") - ] - columns = tmp_columns - return columns - - async def data( - self, - request, - default_labels=False, - _next=None, - _size=None, - ): - database_route = tilde_decode(request.url_vars["database"]) - table = tilde_decode(request.url_vars["table"]) - try: - db = self.ds.get_database(route=database_route) - except KeyError: - raise NotFound("Database not found: {}".format(database_route)) - database = db.name - - # If this is a canned query, not a table, then dispatch to QueryView instead - canned_query = await self.ds.get_canned_query(database, table, request.actor) - if canned_query: - return await QueryView(self.ds).data( - request, - canned_query["sql"], - metadata=canned_query, - editable=False, - canned_query=table, - named_parameters=canned_query.get("params"), - write=bool(canned_query.get("write")), - ) - - is_view = bool(await db.get_view_definition(table)) - table_exists = bool(await db.table_exists(table)) - - # If table or view not found, return 404 - if not is_view and not table_exists: - raise NotFound(f"Table not found: {table}") - - # Ensure user has permission to view this table - await self.ds.ensure_permissions( - request.actor, - [ - ("view-table", (database, table)), - ("view-database", database), - "view-instance", - ], - ) - - private = not await self.ds.permission_allowed( - None, "view-table", (database, table), default=True - ) - - # Handle ?_filter_column and redirect, if present - redirect_params = filters_should_redirect(request.args) - if redirect_params: - return self.redirect( - request, - path_with_added_args(request, redirect_params), - forward_querystring=False, - ) - - # If ?_sort_by_desc=on (from checkbox) redirect to _sort_desc=(_sort) - if "_sort_by_desc" in request.args: - return self.redirect( - request, - path_with_added_args( - request, - { - "_sort_desc": request.args.get("_sort"), - "_sort_by_desc": None, - "_sort": None, - }, - ), - forward_querystring=False, - ) - - # Introspect columns and primary keys for table - pks = await db.primary_keys(table) - table_columns = await db.table_columns(table) - - # Take ?_col= and ?_nocol= into account - specified_columns = await self.columns_to_select(table_columns, pks, request) - select_specified_columns = ", ".join( - escape_sqlite(t) for t in specified_columns - ) - select_all_columns = ", ".join(escape_sqlite(t) for t in table_columns) - - # rowid tables (no specified primary key) need a different SELECT - use_rowid = not pks and not is_view - if use_rowid: - select_specified_columns = f"rowid, {select_specified_columns}" - select_all_columns = f"rowid, {select_all_columns}" - order_by = "rowid" - order_by_pks = "rowid" - else: - order_by_pks = ", ".join([escape_sqlite(pk) for pk in pks]) - order_by = order_by_pks - - if is_view: - order_by = "" - - nocount = request.args.get("_nocount") - nofacet = request.args.get("_nofacet") - nosuggest = request.args.get("_nosuggest") - - if request.args.get("_shape") in ("array", "object"): - nocount = True - nofacet = True - - table_metadata = self.ds.table_metadata(database, table) - units = table_metadata.get("units", {}) - - # Arguments that start with _ and don't contain a __ are - # special - things like ?_search= - and should not be - # treated as filters. - filter_args = [] - for key in request.args: - if not (key.startswith("_") and "__" not in key): - for v in request.args.getlist(key): - filter_args.append((key, v)) - - # Build where clauses from query string arguments - filters = Filters(sorted(filter_args), units, ureg) - where_clauses, params = filters.build_where_clauses(table) - - # Execute filters_from_request plugin hooks - including the default - # ones that live in datasette/filters.py - extra_context_from_filters = {} - extra_human_descriptions = [] - - for hook in pm.hook.filters_from_request( - request=request, - table=table, - database=database, - datasette=self.ds, - ): - filter_arguments = await await_me_maybe(hook) - if filter_arguments: - where_clauses.extend(filter_arguments.where_clauses) - params.update(filter_arguments.params) - extra_human_descriptions.extend(filter_arguments.human_descriptions) - extra_context_from_filters.update(filter_arguments.extra_context) - - # Deal with custom sort orders - sortable_columns = await self.sortable_columns_for_table( - database, table, use_rowid - ) - sort = request.args.get("_sort") - sort_desc = request.args.get("_sort_desc") - - if not sort and not sort_desc: - sort = table_metadata.get("sort") - sort_desc = table_metadata.get("sort_desc") - - if sort and sort_desc: - raise DatasetteError("Cannot use _sort and _sort_desc at the same time") - - if sort: - if sort not in sortable_columns: - raise DatasetteError(f"Cannot sort table by {sort}") - - order_by = escape_sqlite(sort) - - if sort_desc: - if sort_desc not in sortable_columns: - raise DatasetteError(f"Cannot sort table by {sort_desc}") - - order_by = f"{escape_sqlite(sort_desc)} desc" - - from_sql = "from {table_name} {where}".format( - table_name=escape_sqlite(table), - where=("where {} ".format(" and ".join(where_clauses))) - if where_clauses - else "", - ) - # Copy of params so we can mutate them later: - from_sql_params = dict(**params) - - count_sql = f"select count(*) {from_sql}" - - # Handle pagination driven by ?_next= - _next = _next or request.args.get("_next") - offset = "" - if _next: - sort_value = None - if is_view: - # _next is an offset - offset = f" offset {int(_next)}" - else: - components = urlsafe_components(_next) - # If a sort order is applied and there are multiple components, - # the first of these is the sort value - if (sort or sort_desc) and (len(components) > 1): - sort_value = components[0] - # Special case for if non-urlencoded first token was $null - if _next.split(",")[0] == "$null": - sort_value = None - components = components[1:] - - # Figure out the SQL for next-based-on-primary-key first - next_by_pk_clauses = [] - if use_rowid: - next_by_pk_clauses.append(f"rowid > :p{len(params)}") - params[f"p{len(params)}"] = components[0] - else: - # Apply the tie-breaker based on primary keys - if len(components) == len(pks): - param_len = len(params) - next_by_pk_clauses.append( - compound_keys_after_sql(pks, param_len) - ) - for i, pk_value in enumerate(components): - params[f"p{param_len + i}"] = pk_value - - # Now add the sort SQL, which may incorporate next_by_pk_clauses - if sort or sort_desc: - if sort_value is None: - if sort_desc: - # Just items where column is null ordered by pk - where_clauses.append( - "({column} is null and {next_clauses})".format( - column=escape_sqlite(sort_desc), - next_clauses=" and ".join(next_by_pk_clauses), - ) - ) - else: - where_clauses.append( - "({column} is not null or ({column} is null and {next_clauses}))".format( - column=escape_sqlite(sort), - next_clauses=" and ".join(next_by_pk_clauses), - ) - ) - else: - where_clauses.append( - "({column} {op} :p{p}{extra_desc_only} or ({column} = :p{p} and {next_clauses}))".format( - column=escape_sqlite(sort or sort_desc), - op=">" if sort else "<", - p=len(params), - extra_desc_only="" - if sort - else " or {column2} is null".format( - column2=escape_sqlite(sort or sort_desc) - ), - next_clauses=" and ".join(next_by_pk_clauses), - ) - ) - params[f"p{len(params)}"] = sort_value - order_by = f"{order_by}, {order_by_pks}" - else: - where_clauses.extend(next_by_pk_clauses) - - where_clause = "" - if where_clauses: - where_clause = f"where {' and '.join(where_clauses)} " - - if order_by: - order_by = f"order by {order_by}" - - extra_args = {} - # Handle ?_size=500 - page_size = _size or request.args.get("_size") or table_metadata.get("size") - if page_size: - if page_size == "max": - page_size = self.ds.max_returned_rows - try: - page_size = int(page_size) - if page_size < 0: - raise ValueError - - except ValueError: - raise BadRequest("_size must be a positive integer") - - if page_size > self.ds.max_returned_rows: - raise BadRequest(f"_size must be <= {self.ds.max_returned_rows}") - - extra_args["page_size"] = page_size - else: - page_size = self.ds.page_size - - # Facets are calculated against SQL without order by or limit - sql_no_order_no_limit = ( - "select {select_all_columns} from {table_name} {where}".format( - select_all_columns=select_all_columns, - table_name=escape_sqlite(table), - where=where_clause, - ) - ) - - # This is the SQL that populates the main table on the page - sql = "select {select_specified_columns} from {table_name} {where}{order_by} limit {page_size}{offset}".format( - select_specified_columns=select_specified_columns, - table_name=escape_sqlite(table), - where=where_clause, - order_by=order_by, - page_size=page_size + 1, - offset=offset, - ) - - if request.args.get("_timelimit"): - extra_args["custom_time_limit"] = int(request.args.get("_timelimit")) - - # Execute the main query! - results = await db.execute(sql, params, truncate=True, **extra_args) - - # Calculate the total count for this query - filtered_table_rows_count = None - if ( - not db.is_mutable - and self.ds.inspect_data - and count_sql == f"select count(*) from {table} " - ): - # We can use a previously cached table row count - try: - filtered_table_rows_count = self.ds.inspect_data[database]["tables"][ - table - ]["count"] - except KeyError: - pass - - # Otherwise run a select count(*) ... - if count_sql and filtered_table_rows_count is None and not nocount: - try: - count_rows = list(await db.execute(count_sql, from_sql_params)) - filtered_table_rows_count = count_rows[0][0] - except QueryInterrupted: - pass - - # Faceting - if not self.ds.setting("allow_facet") and any( - arg.startswith("_facet") for arg in request.args - ): - raise BadRequest("_facet= is not allowed") - - # pylint: disable=no-member - facet_classes = list( - itertools.chain.from_iterable(pm.hook.register_facet_classes()) - ) - facet_results = {} - facets_timed_out = [] - facet_instances = [] - for klass in facet_classes: - facet_instances.append( - klass( - self.ds, - request, - database, - sql=sql_no_order_no_limit, - params=params, - table=table, - metadata=table_metadata, - row_count=filtered_table_rows_count, - ) - ) - - if not nofacet: - for facet in facet_instances: - ( - instance_facet_results, - instance_facets_timed_out, - ) = await facet.facet_results() - for facet_info in instance_facet_results: - base_key = facet_info["name"] - key = base_key - i = 1 - while key in facet_results: - i += 1 - key = f"{base_key}_{i}" - facet_results[key] = facet_info - facets_timed_out.extend(instance_facets_timed_out) - - # Calculate suggested facets - suggested_facets = [] - if ( - self.ds.setting("suggest_facets") - and self.ds.setting("allow_facet") - and not _next - and not nofacet - and not nosuggest - ): - for facet in facet_instances: - suggested_facets.extend(await facet.suggest()) - - # Figure out columns and rows for the query - columns = [r[0] for r in results.description] - rows = list(results.rows) - - # Expand labeled columns if requested - expanded_columns = [] - expandable_columns = await self.expandable_columns(database, table) - columns_to_expand = None - try: - all_labels = value_as_boolean(request.args.get("_labels", "")) - except ValueError: - all_labels = default_labels - # Check for explicit _label= - if "_label" in request.args: - columns_to_expand = request.args.getlist("_label") - if columns_to_expand is None and all_labels: - # expand all columns with foreign keys - columns_to_expand = [fk["column"] for fk, _ in expandable_columns] - - if columns_to_expand: - expanded_labels = {} - for fk, _ in expandable_columns: - column = fk["column"] - if column not in columns_to_expand: - continue - if column not in columns: - continue - expanded_columns.append(column) - # Gather the values - column_index = columns.index(column) - values = [row[column_index] for row in rows] - # Expand them - expanded_labels.update( - await self.ds.expand_foreign_keys(database, table, column, values) - ) - if expanded_labels: - # Rewrite the rows - new_rows = [] - for row in rows: - new_row = CustomRow(columns) - for column in row.keys(): - value = row[column] - if (column, value) in expanded_labels and value is not None: - new_row[column] = { - "value": value, - "label": expanded_labels[(column, value)], - } - else: - new_row[column] = value - new_rows.append(new_row) - rows = new_rows - - # Pagination next link - next_value = None - next_url = None - if 0 < page_size < len(rows): - if is_view: - next_value = int(_next or 0) + page_size - else: - next_value = path_from_row_pks(rows[-2], pks, use_rowid) - # If there's a sort or sort_desc, add that value as a prefix - if (sort or sort_desc) and not is_view: - prefix = rows[-2][sort or sort_desc] - if isinstance(prefix, dict) and "value" in prefix: - prefix = prefix["value"] - if prefix is None: - prefix = "$null" - else: - prefix = tilde_encode(str(prefix)) - next_value = f"{prefix},{next_value}" - added_args = {"_next": next_value} - if sort: - added_args["_sort"] = sort - else: - added_args["_sort_desc"] = sort_desc - else: - added_args = {"_next": next_value} - next_url = self.ds.absolute_url( - request, self.ds.urls.path(path_with_replaced_args(request, added_args)) - ) - rows = rows[:page_size] - - # human_description_en combines filters AND search, if provided - human_description_en = filters.human_description_en( - extra=extra_human_descriptions - ) - - if sort or sort_desc: - sorted_by = "sorted by {}{}".format( - (sort or sort_desc), " descending" if sort_desc else "" - ) - human_description_en = " ".join( - [b for b in [human_description_en, sorted_by] if b] - ) - - async def extra_template(): - nonlocal sort - - display_columns, display_rows = await self.display_columns_and_rows( - database, - table, - results.description, - rows, - link_column=not is_view, - truncate_cells=self.ds.setting("truncate_cells_html"), - ) - metadata = ( - (self.ds.metadata("databases") or {}) - .get(database, {}) - .get("tables", {}) - .get(table, {}) - ) - self.ds.update_with_inherited_metadata(metadata) - - form_hidden_args = [] - for key in request.args: - if ( - key.startswith("_") - and key not in ("_sort", "_search", "_next") - and "__" not in key - ): - for value in request.args.getlist(key): - form_hidden_args.append((key, value)) - - # if no sort specified AND table has a single primary key, - # set sort to that so arrow is displayed - if not sort and not sort_desc: - if 1 == len(pks): - sort = pks[0] - elif use_rowid: - sort = "rowid" - - async def table_actions(): - links = [] - for hook in pm.hook.table_actions( - datasette=self.ds, - table=table, - database=database, - actor=request.actor, - request=request, - ): - extra_links = await await_me_maybe(hook) - if extra_links: - links.extend(extra_links) - return links - - # filter_columns combine the columns we know are available - # in the table with any additional columns (such as rowid) - # which are available in the query - filter_columns = list(columns) + [ - table_column - for table_column in table_columns - if table_column not in columns - ] - d = { - "table_actions": table_actions, - "use_rowid": use_rowid, - "filters": filters, - "display_columns": display_columns, - "filter_columns": filter_columns, - "display_rows": display_rows, - "facets_timed_out": facets_timed_out, - "sorted_facet_results": sorted( - facet_results.values(), - key=lambda f: (len(f["results"]), f["name"]), - reverse=True, - ), - "form_hidden_args": form_hidden_args, - "is_sortable": any(c["sortable"] for c in display_columns), - "fix_path": self.ds.urls.path, - "path_with_replaced_args": path_with_replaced_args, - "path_with_removed_args": path_with_removed_args, - "append_querystring": append_querystring, - "request": request, - "sort": sort, - "sort_desc": sort_desc, - "disable_sort": is_view, - "custom_table_templates": [ - f"_table-{to_css_class(database)}-{to_css_class(table)}.html", - f"_table-table-{to_css_class(database)}-{to_css_class(table)}.html", - "_table.html", - ], - "metadata": metadata, - "view_definition": await db.get_view_definition(table), - "table_definition": await db.get_table_definition(table), - "datasette_allow_facet": "true" - if self.ds.setting("allow_facet") - else "false", - } - d.update(extra_context_from_filters) - return d - - return ( - { - "database": database, - "table": table, - "is_view": is_view, - "human_description_en": human_description_en, - "rows": rows[:page_size], - "truncated": results.truncated, - "filtered_table_rows_count": filtered_table_rows_count, - "expanded_columns": expanded_columns, - "expandable_columns": expandable_columns, - "columns": columns, - "primary_keys": pks, - "units": units, - "query": {"sql": sql, "params": params}, - "facet_results": facet_results, - "suggested_facets": suggested_facets, - "next": next_value and str(next_value) or None, - "next_url": next_url, - "private": private, - "allow_execute_sql": await self.ds.permission_allowed( - request.actor, "execute-sql", database, default=True - ), - }, - extra_template, - ( - f"table-{to_css_class(database)}-{to_css_class(table)}.html", - "table.html", - ), - ) - - async def _sql_params_pks(db, table, pk_values): pks = await db.primary_keys(table) use_rowid = not pks @@ -1077,3 +431,1077 @@ class RowView(RowTableShared): ) foreign_key_tables.append({**fk, **{"count": count, "link": link}}) return foreign_key_tables + + +async def filter_args(request: Request) -> List[Tuple[str, str]]: + # Arguments that start with _ and don't contain a __ are + # special - things like ?_search= - and should not be + # treated as filters. + filter_args = [] + for key in request.args: + if not (key.startswith("_") and "__" not in key): + for v in request.args.getlist(key): + filter_args.append((key, v)) + return filter_args + + +async def database(request: Request, datasette: Datasette) -> Database: + database_route = tilde_decode(request.url_vars["database"]) + try: + return datasette.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + + +async def table_name(request: Request) -> str: + return tilde_decode(request.url_vars["table"]) + + +class TableView(BaseView): + name = "" + + async def options(self, request, *args, **kwargs): + r = Response.text("ok") + if self.ds.cors: + add_cors_headers(r.headers) + return r + + def redirect(self, request, path, forward_querystring=True, remove_args=None): + if request.query_string and "?" not in path and forward_querystring: + path = f"{path}?{request.query_string}" + if remove_args: + path = path_with_removed_args(request, remove_args, path=path) + r = Response.redirect(path) + r.headers["Link"] = f"<{path}>; rel=preload" + if self.ds.cors: + add_cors_headers(r.headers) + return r + + async def display_columns_and_rows( + self, database, table, description, rows, link_column=False, truncate_cells=0 + ): + """Returns columns, rows for specified table - including fancy foreign key treatment""" + db = self.ds.databases[database] + table_metadata = self.ds.table_metadata(database, table) + column_descriptions = table_metadata.get("columns") or {} + column_details = {col.name: col for col in await db.table_column_details(table)} + sortable_columns = await self.sortable_columns_for_table(database, table, True) + pks = await db.primary_keys(table) + pks_for_display = pks + if not pks_for_display: + pks_for_display = ["rowid"] + + columns = [] + for r in description: + if r[0] == "rowid" and "rowid" not in column_details: + type_ = "integer" + notnull = 0 + else: + type_ = column_details[r[0]].type + notnull = column_details[r[0]].notnull + columns.append( + { + "name": r[0], + "sortable": r[0] in sortable_columns, + "is_pk": r[0] in pks_for_display, + "type": type_, + "notnull": notnull, + "description": column_descriptions.get(r[0]), + } + ) + + column_to_foreign_key_table = { + fk["column"]: fk["other_table"] + for fk in await db.foreign_keys_for_table(table) + } + + cell_rows = [] + base_url = self.ds.setting("base_url") + for row in rows: + cells = [] + # Unless we are a view, the first column is a link - either to the rowid + # or to the simple or compound primary key + if link_column: + is_special_link_column = len(pks) != 1 + pk_path = path_from_row_pks(row, pks, not pks, False) + cells.append( + { + "column": pks[0] if len(pks) == 1 else "Link", + "value_type": "pk", + "is_special_link_column": is_special_link_column, + "raw": pk_path, + "value": markupsafe.Markup( + '{flat_pks}'.format( + base_url=base_url, + table_path=self.ds.urls.table(database, table), + flat_pks=str(markupsafe.escape(pk_path)), + flat_pks_quoted=path_from_row_pks(row, pks, not pks), + ) + ), + } + ) + + for value, column_dict in zip(row, columns): + column = column_dict["name"] + if link_column and len(pks) == 1 and column == pks[0]: + # If there's a simple primary key, don't repeat the value as it's + # already shown in the link column. + continue + + # First let the plugins have a go + # pylint: disable=no-member + plugin_display_value = None + for candidate in pm.hook.render_cell( + value=value, + column=column, + table=table, + database=database, + datasette=self.ds, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value: + display_value = plugin_display_value + elif isinstance(value, bytes): + formatted = format_bytes(len(value)) + display_value = markupsafe.Markup( + '<Binary: {:,} byte{}>'.format( + self.ds.urls.row_blob( + database, + table, + path_from_row_pks(row, pks, not pks), + column, + ), + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "", + len(value), + "" if len(value) == 1 else "s", + ) + ) + elif isinstance(value, dict): + # It's an expanded foreign key - display link to other row + label = value["label"] + value = value["value"] + # The table we link to depends on the column + other_table = column_to_foreign_key_table[column] + link_template = ( + LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE + ) + display_value = markupsafe.Markup( + link_template.format( + database=database, + base_url=base_url, + table=tilde_encode(other_table), + link_id=tilde_encode(str(value)), + id=str(markupsafe.escape(value)), + label=str(markupsafe.escape(label)) or "-", + ) + ) + elif value in ("", None): + display_value = markupsafe.Markup(" ") + elif is_url(str(value).strip()): + display_value = markupsafe.Markup( + '{url}'.format( + url=markupsafe.escape(value.strip()) + ) + ) + elif column in table_metadata.get("units", {}) and value != "": + # Interpret units using pint + value = value * ureg(table_metadata["units"][column]) + # Pint uses floating point which sometimes introduces errors in the compact + # representation, which we have to round off to avoid ugliness. In the vast + # majority of cases this rounding will be inconsequential. I hope. + value = round(value.to_compact(), 6) + display_value = markupsafe.Markup( + f"{value:~P}".replace(" ", " ") + ) + else: + display_value = str(value) + if truncate_cells and len(display_value) > truncate_cells: + display_value = display_value[:truncate_cells] + "\u2026" + + cells.append( + { + "column": column, + "value": display_value, + "raw": value, + "value_type": "none" + if value is None + else str(type(value).__name__), + } + ) + cell_rows.append(Row(cells)) + + if link_column: + # Add the link column header. + # If it's a simple primary key, we have to remove and re-add that column name at + # the beginning of the header row. + first_column = None + if len(pks) == 1: + columns = [col for col in columns if col["name"] != pks[0]] + first_column = { + "name": pks[0], + "sortable": len(pks) == 1, + "is_pk": True, + "type": column_details[pks[0]].type, + "notnull": column_details[pks[0]].notnull, + } + else: + first_column = { + "name": "Link", + "sortable": False, + "is_pk": False, + "type": "", + "notnull": 0, + } + columns = [first_column] + columns + return columns, cell_rows + + async def expandable_columns(self, database, table): + # Returns list of (fk_dict, label_column-or-None) pairs for that table + expandables = [] + db = self.ds.databases[database] + for fk in await db.foreign_keys_for_table(table): + label_column = await db.label_column_for_table(fk["other_table"]) + expandables.append((fk, label_column)) + return expandables + + async def sortable_columns_for_table(self, database, table, use_rowid): + db = self.ds.databases[database] + table_metadata = self.ds.table_metadata(database, table) + if "sortable_columns" in table_metadata: + sortable_columns = set(table_metadata["sortable_columns"]) + else: + sortable_columns = set(await db.table_columns(table)) + if use_rowid: + sortable_columns.add("rowid") + return sortable_columns + + async def columns_to_select(self, table_columns, pks, request): + columns = list(table_columns) + if "_col" in request.args: + columns = list(pks) + _cols = request.args.getlist("_col") + bad_columns = [column for column in _cols if column not in table_columns] + if bad_columns: + raise DatasetteError( + "_col={} - invalid columns".format(", ".join(bad_columns)), + status=400, + ) + # De-duplicate maintaining order: + columns.extend(dict.fromkeys(_cols)) + if "_nocol" in request.args: + # Return all columns EXCEPT these + bad_columns = [ + column + for column in request.args.getlist("_nocol") + if (column not in table_columns) or (column in pks) + ] + if bad_columns: + raise DatasetteError( + "_nocol={} - invalid columns".format(", ".join(bad_columns)), + status=400, + ) + tmp_columns = [ + column + for column in columns + if column not in request.args.getlist("_nocol") + ] + columns = tmp_columns + return columns + + async def data(self, request): + + registry = Registry(filter_args, database, table_name) + + breakpoint() + resolved = await registry.resolve_multi( + ["filter_args", "database", "table_name"], + results=dict(datasette=self.ds, request=request), + ) + print(resolved) + + database_route = tilde_decode(request.url_vars["database"]) + table = tilde_decode(request.url_vars["table"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database_name = db.name + + # If this is a canned query, not a table, then dispatch to QueryView instead + canned_query = await self.ds.get_canned_query( + database_name, table, request.actor + ) + if canned_query: + return await QueryView(self.ds).data( + request, + canned_query["sql"], + metadata=canned_query, + editable=False, + canned_query=table, + named_parameters=canned_query.get("params"), + write=bool(canned_query.get("write")), + ) + + is_view = bool(await db.get_view_definition(table)) + table_exists = bool(await db.table_exists(table)) + + # If table or view not found, return 404 + if not is_view and not table_exists: + raise NotFound(f"Table not found: {table}") + + # Ensure user has permission to view this table + await self.ds.ensure_permissions( + request.actor, + [ + ("view-table", (database_name, table)), + ("view-database", database_name), + "view-instance", + ], + ) + + private = not await self.ds.permission_allowed( + None, "view-table", (database_name, table), default=True + ) + + # Handle ?_filter_column and redirect, if present + redirect_params = filters_should_redirect(request.args) + if redirect_params: + return self.redirect( + request, + path_with_added_args(request, redirect_params), + forward_querystring=False, + ) + + # If ?_sort_by_desc=on (from checkbox) redirect to _sort_desc=(_sort) + if "_sort_by_desc" in request.args: + return self.redirect( + request, + path_with_added_args( + request, + { + "_sort_desc": request.args.get("_sort"), + "_sort_by_desc": None, + "_sort": None, + }, + ), + forward_querystring=False, + ) + + # Introspect columns and primary keys for table + pks = await db.primary_keys(table) + table_columns = await db.table_columns(table) + + # Take ?_col= and ?_nocol= into account + specified_columns = await self.columns_to_select(table_columns, pks, request) + select_specified_columns = ", ".join( + escape_sqlite(t) for t in specified_columns + ) + select_all_columns = ", ".join(escape_sqlite(t) for t in table_columns) + + # rowid tables (no specified primary key) need a different SELECT + use_rowid = not pks and not is_view + if use_rowid: + select_specified_columns = f"rowid, {select_specified_columns}" + select_all_columns = f"rowid, {select_all_columns}" + order_by = "rowid" + order_by_pks = "rowid" + else: + order_by_pks = ", ".join([escape_sqlite(pk) for pk in pks]) + order_by = order_by_pks + + if is_view: + order_by = "" + + nocount = request.args.get("_nocount") + nofacet = request.args.get("_nofacet") + nosuggest = request.args.get("_nosuggest") + + if request.args.get("_shape") in ("array", "object"): + nocount = True + nofacet = True + + table_metadata = self.ds.table_metadata(database_name, table) + units = table_metadata.get("units", {}) + + filter_args_ = filter_args(request) + + # Build where clauses from query string arguments + filters = Filters(sorted(filter_args_), units, ureg) + where_clauses, params = filters.build_where_clauses(table) + + # Execute filters_from_request plugin hooks - including the default + # ones that live in datasette/filters.py + extra_context_from_filters = {} + extra_human_descriptions = [] + + for hook in pm.hook.filters_from_request( + request=request, + table=table, + database=database_name, + datasette=self.ds, + ): + filter_arguments = await await_me_maybe(hook) + if filter_arguments: + where_clauses.extend(filter_arguments.where_clauses) + params.update(filter_arguments.params) + extra_human_descriptions.extend(filter_arguments.human_descriptions) + extra_context_from_filters.update(filter_arguments.extra_context) + + # Deal with custom sort orders + sortable_columns = await self.sortable_columns_for_table( + database_name, table, use_rowid + ) + sort = request.args.get("_sort") + sort_desc = request.args.get("_sort_desc") + + if not sort and not sort_desc: + sort = table_metadata.get("sort") + sort_desc = table_metadata.get("sort_desc") + + if sort and sort_desc: + raise DatasetteError("Cannot use _sort and _sort_desc at the same time") + + if sort: + if sort not in sortable_columns: + raise DatasetteError(f"Cannot sort table by {sort}") + + order_by = escape_sqlite(sort) + + if sort_desc: + if sort_desc not in sortable_columns: + raise DatasetteError(f"Cannot sort table by {sort_desc}") + + order_by = f"{escape_sqlite(sort_desc)} desc" + + from_sql = "from {table_name} {where}".format( + table_name=escape_sqlite(table), + where=("where {} ".format(" and ".join(where_clauses))) + if where_clauses + else "", + ) + # Copy of params so we can mutate them later: + from_sql_params = dict(**params) + + count_sql = f"select count(*) {from_sql}" + + # Handle pagination driven by ?_next= + _next = request.args.get("_next") + offset = "" + if _next: + sort_value = None + if is_view: + # _next is an offset + offset = f" offset {int(_next)}" + else: + components = urlsafe_components(_next) + # If a sort order is applied and there are multiple components, + # the first of these is the sort value + if (sort or sort_desc) and (len(components) > 1): + sort_value = components[0] + # Special case for if non-urlencoded first token was $null + if _next.split(",")[0] == "$null": + sort_value = None + components = components[1:] + + # Figure out the SQL for next-based-on-primary-key first + next_by_pk_clauses = [] + if use_rowid: + next_by_pk_clauses.append(f"rowid > :p{len(params)}") + params[f"p{len(params)}"] = components[0] + else: + # Apply the tie-breaker based on primary keys + if len(components) == len(pks): + param_len = len(params) + next_by_pk_clauses.append( + compound_keys_after_sql(pks, param_len) + ) + for i, pk_value in enumerate(components): + params[f"p{param_len + i}"] = pk_value + + # Now add the sort SQL, which may incorporate next_by_pk_clauses + if sort or sort_desc: + if sort_value is None: + if sort_desc: + # Just items where column is null ordered by pk + where_clauses.append( + "({column} is null and {next_clauses})".format( + column=escape_sqlite(sort_desc), + next_clauses=" and ".join(next_by_pk_clauses), + ) + ) + else: + where_clauses.append( + "({column} is not null or ({column} is null and {next_clauses}))".format( + column=escape_sqlite(sort), + next_clauses=" and ".join(next_by_pk_clauses), + ) + ) + else: + where_clauses.append( + "({column} {op} :p{p}{extra_desc_only} or ({column} = :p{p} and {next_clauses}))".format( + column=escape_sqlite(sort or sort_desc), + op=">" if sort else "<", + p=len(params), + extra_desc_only="" + if sort + else " or {column2} is null".format( + column2=escape_sqlite(sort or sort_desc) + ), + next_clauses=" and ".join(next_by_pk_clauses), + ) + ) + params[f"p{len(params)}"] = sort_value + order_by = f"{order_by}, {order_by_pks}" + else: + where_clauses.extend(next_by_pk_clauses) + + where_clause = "" + if where_clauses: + where_clause = f"where {' and '.join(where_clauses)} " + + if order_by: + order_by = f"order by {order_by}" + + extra_args = {} + # Handle ?_size=500 + page_size = request.args.get("_size") or table_metadata.get("size") + if page_size: + if page_size == "max": + page_size = self.ds.max_returned_rows + try: + page_size = int(page_size) + if page_size < 0: + raise ValueError + + except ValueError: + raise BadRequest("_size must be a positive integer") + + if page_size > self.ds.max_returned_rows: + raise BadRequest(f"_size must be <= {self.ds.max_returned_rows}") + + extra_args["page_size"] = page_size + else: + page_size = self.ds.page_size + + # Facets are calculated against SQL without order by or limit + sql_no_order_no_limit = ( + "select {select_all_columns} from {table_name} {where}".format( + select_all_columns=select_all_columns, + table_name=escape_sqlite(table), + where=where_clause, + ) + ) + + # This is the SQL that populates the main table on the page + sql = "select {select_specified_columns} from {table_name} {where}{order_by} limit {page_size}{offset}".format( + select_specified_columns=select_specified_columns, + table_name=escape_sqlite(table), + where=where_clause, + order_by=order_by, + page_size=page_size + 1, + offset=offset, + ) + + if request.args.get("_timelimit"): + extra_args["custom_time_limit"] = int(request.args.get("_timelimit")) + + # Execute the main query! + results = await db.execute(sql, params, truncate=True, **extra_args) + + # Calculate the total count for this query + filtered_table_rows_count = None + if ( + not db.is_mutable + and self.ds.inspect_data + and count_sql == f"select count(*) from {table} " + ): + # We can use a previously cached table row count + try: + filtered_table_rows_count = self.ds.inspect_data[database_name][ + "tables" + ][table]["count"] + except KeyError: + pass + + # Otherwise run a select count(*) ... + if count_sql and filtered_table_rows_count is None and not nocount: + try: + count_rows = list(await db.execute(count_sql, from_sql_params)) + filtered_table_rows_count = count_rows[0][0] + except QueryInterrupted: + pass + + # Faceting + if not self.ds.setting("allow_facet") and any( + arg.startswith("_facet") for arg in request.args + ): + raise BadRequest("_facet= is not allowed") + + # pylint: disable=no-member + facet_classes = list( + itertools.chain.from_iterable(pm.hook.register_facet_classes()) + ) + facet_results = {} + facets_timed_out = [] + facet_instances = [] + for klass in facet_classes: + facet_instances.append( + klass( + self.ds, + request, + database_name, + sql=sql_no_order_no_limit, + params=params, + table=table, + metadata=table_metadata, + row_count=filtered_table_rows_count, + ) + ) + + if not nofacet: + for facet in facet_instances: + ( + instance_facet_results, + instance_facets_timed_out, + ) = await facet.facet_results() + for facet_info in instance_facet_results: + base_key = facet_info["name"] + key = base_key + i = 1 + while key in facet_results: + i += 1 + key = f"{base_key}_{i}" + facet_results[key] = facet_info + facets_timed_out.extend(instance_facets_timed_out) + + # Calculate suggested facets + suggested_facets = [] + if ( + self.ds.setting("suggest_facets") + and self.ds.setting("allow_facet") + and not _next + and not nofacet + and not nosuggest + ): + for facet in facet_instances: + suggested_facets.extend(await facet.suggest()) + + # Figure out columns and rows for the query + columns = [r[0] for r in results.description] + rows = list(results.rows) + + # Expand labeled columns if requested + expanded_columns = [] + expandable_columns = await self.expandable_columns(database_name, table) + columns_to_expand = None + try: + all_labels = value_as_boolean(request.args.get("_labels", "")) + except ValueError: + # TODO: Should be True for HTML views + all_labels = False + # Check for explicit _label= + if "_label" in request.args: + columns_to_expand = request.args.getlist("_label") + if columns_to_expand is None and all_labels: + # expand all columns with foreign keys + columns_to_expand = [fk["column"] for fk, _ in expandable_columns] + + if columns_to_expand: + expanded_labels = {} + for fk, _ in expandable_columns: + column = fk["column"] + if column not in columns_to_expand: + continue + if column not in columns: + continue + expanded_columns.append(column) + # Gather the values + column_index = columns.index(column) + values = [row[column_index] for row in rows] + # Expand them + expanded_labels.update( + await self.ds.expand_foreign_keys( + database_name, table, column, values + ) + ) + if expanded_labels: + # Rewrite the rows + new_rows = [] + for row in rows: + new_row = CustomRow(columns) + for column in row.keys(): + value = row[column] + if (column, value) in expanded_labels and value is not None: + new_row[column] = { + "value": value, + "label": expanded_labels[(column, value)], + } + else: + new_row[column] = value + new_rows.append(new_row) + rows = new_rows + + # Pagination next link + next_value = None + next_url = None + if 0 < page_size < len(rows): + if is_view: + next_value = int(_next or 0) + page_size + else: + next_value = path_from_row_pks(rows[-2], pks, use_rowid) + # If there's a sort or sort_desc, add that value as a prefix + if (sort or sort_desc) and not is_view: + prefix = rows[-2][sort or sort_desc] + if isinstance(prefix, dict) and "value" in prefix: + prefix = prefix["value"] + if prefix is None: + prefix = "$null" + else: + prefix = tilde_encode(str(prefix)) + next_value = f"{prefix},{next_value}" + added_args = {"_next": next_value} + if sort: + added_args["_sort"] = sort + else: + added_args["_sort_desc"] = sort_desc + else: + added_args = {"_next": next_value} + next_url = self.ds.absolute_url( + request, self.ds.urls.path(path_with_replaced_args(request, added_args)) + ) + rows = rows[:page_size] + + # human_description_en combines filters AND search, if provided + human_description_en = filters.human_description_en( + extra=extra_human_descriptions + ) + + if sort or sort_desc: + sorted_by = "sorted by {}{}".format( + (sort or sort_desc), " descending" if sort_desc else "" + ) + human_description_en = " ".join( + [b for b in [human_description_en, sorted_by] if b] + ) + + async def extra_template(): + nonlocal sort + + display_columns, display_rows = await self.display_columns_and_rows( + database_name, + table, + results.description, + rows, + link_column=not is_view, + truncate_cells=self.ds.setting("truncate_cells_html"), + ) + metadata = ( + (self.ds.metadata("databases") or {}) + .get(database_name, {}) + .get("tables", {}) + .get(table, {}) + ) + self.ds.update_with_inherited_metadata(metadata) + + form_hidden_args = [] + for key in request.args: + if ( + key.startswith("_") + and key not in ("_sort", "_search", "_next") + and "__" not in key + ): + for value in request.args.getlist(key): + form_hidden_args.append((key, value)) + + # if no sort specified AND table has a single primary key, + # set sort to that so arrow is displayed + if not sort and not sort_desc: + if 1 == len(pks): + sort = pks[0] + elif use_rowid: + sort = "rowid" + + async def table_actions(): + links = [] + for hook in pm.hook.table_actions( + datasette=self.ds, + table=table, + database=database_name, + actor=request.actor, + request=request, + ): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + + # filter_columns combine the columns we know are available + # in the table with any additional columns (such as rowid) + # which are available in the query + filter_columns = list(columns) + [ + table_column + for table_column in table_columns + if table_column not in columns + ] + d = { + "table_actions": table_actions, + "use_rowid": use_rowid, + "filters": filters, + "display_columns": display_columns, + "filter_columns": filter_columns, + "display_rows": display_rows, + "facets_timed_out": facets_timed_out, + "sorted_facet_results": sorted( + facet_results.values(), + key=lambda f: (len(f["results"]), f["name"]), + reverse=True, + ), + "form_hidden_args": form_hidden_args, + "is_sortable": any(c["sortable"] for c in display_columns), + "fix_path": self.ds.urls.path, + "path_with_replaced_args": path_with_replaced_args, + "path_with_removed_args": path_with_removed_args, + "append_querystring": append_querystring, + "request": request, + "sort": sort, + "sort_desc": sort_desc, + "disable_sort": is_view, + "custom_table_templates": [ + f"_table-{to_css_class(database_name)}-{to_css_class(table)}.html", + f"_table-table-{to_css_class(database_name)}-{to_css_class(table)}.html", + "_table.html", + ], + "metadata": metadata, + "view_definition": await db.get_view_definition(table), + "table_definition": await db.get_table_definition(table), + "datasette_allow_facet": "true" + if self.ds.setting("allow_facet") + else "false", + } + d.update(extra_context_from_filters) + return d + + return ( + { + "database": database_name, + "table": table, + "is_view": is_view, + "human_description_en": human_description_en, + "rows": rows[:page_size], + "truncated": results.truncated, + "filtered_table_rows_count": filtered_table_rows_count, + "expanded_columns": expanded_columns, + "expandable_columns": expandable_columns, + "columns": columns, + "primary_keys": pks, + "units": units, + "query": {"sql": sql, "params": params}, + "facet_results": facet_results, + "suggested_facets": suggested_facets, + "next": next_value and str(next_value) or None, + "next_url": next_url, + "private": private, + "allow_execute_sql": await self.ds.permission_allowed( + request.actor, "execute-sql", database, default=True + ), + }, + extra_template, + ( + f"table-{to_css_class(database_name)}-{to_css_class(table)}.html", + "table.html", + ), + ) + + async def get(self, request): + database_route = tilde_decode(request.url_vars["database"]) + + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database = db.name + + _format = request.url_vars["format"] + + if _format == "csv": + return await self.as_csv(request, database_route) + + extra_template_data = {} + start = time.perf_counter() + status_code = None + templates = [] + try: + response_or_template_contexts = await self.data(request) + if isinstance(response_or_template_contexts, Response): + return response_or_template_contexts + # If it has four items, it includes an HTTP status code + if len(response_or_template_contexts) == 4: + ( + data, + extra_template_data, + templates, + status_code, + ) = response_or_template_contexts + else: + data, extra_template_data, templates = response_or_template_contexts + except QueryInterrupted: + raise DatasetteError( + """ + SQL query took too long. The time limit is controlled by the + sql_time_limit_ms + configuration option. + """, + title="SQL Interrupted", + status=400, + message_is_html=True, + ) + except (sqlite3.OperationalError, InvalidSql) as e: + raise DatasetteError(str(e), title="Invalid SQL", status=400) + + except sqlite3.OperationalError as e: + raise DatasetteError(str(e)) + + except DatasetteError: + raise + + end = time.perf_counter() + data["query_ms"] = (end - start) * 1000 + for key in ("source", "source_url", "license", "license_url"): + value = self.ds.metadata(key) + if value: + data[key] = value + + # Special case for .jsono extension - redirect to _shape=objects + if _format == "jsono": + return self.redirect( + request, + path_with_added_args( + request, + {"_shape": "objects"}, + path=request.path.rsplit(".jsono", 1)[0] + ".json", + ), + forward_querystring=False, + ) + + if _format in self.ds.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + self.ds.renderers[_format][0], + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name=self.name, + # These will be deprecated in Datasette 1.0: + args=request.args, + data=data, + ) + if asyncio.iscoroutine(result): + result = await result + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code", status_code or 200), + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + if status_code is not None: + # Over-ride the status code + r.status = status_code + else: + assert False, f"{result} should be dict or Response" + else: + extras = {} + if callable(extra_template_data): + extras = extra_template_data() + if asyncio.iscoroutine(extras): + extras = await extras + else: + extras = extra_template_data + url_labels_extra = {} + if data.get("expandable_columns"): + url_labels_extra = {"_labels": "on"} + + renderers = {} + for key, (_, can_render) in self.ds.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=self.ds, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name=self.name, + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = self.ds.urls.path( + path_with_format( + request=request, format=key, extra_qs={**url_labels_extra} + ) + ) + + url_csv_args = {"_size": "max", **url_labels_extra} + url_csv = self.ds.urls.path( + path_with_format(request=request, format="csv", extra_qs=url_csv_args) + ) + url_csv_path = url_csv.split("?")[0] + context = { + **data, + **extras, + **{ + "renderers": renderers, + "url_csv": url_csv, + "url_csv_path": url_csv_path, + "url_csv_hidden_args": [ + (key, value) + for key, value in urllib.parse.parse_qsl(request.query_string) + if key not in ("_labels", "_facet", "_size") + ] + + [("_size", "max")], + "datasette_version": __version__, + "settings": self.ds.settings_dict(), + }, + } + if "metadata" not in context: + context["metadata"] = self.ds.metadata + r = await self.render(templates, request=request, context=context) + if status_code is not None: + r.status = status_code + + ttl = request.args.get("_ttl", None) + if ttl is None or not ttl.isdigit(): + ttl = self.ds.setting("default_cache_ttl") + + return self.set_response_headers(r, ttl) + + def set_response_headers(self, response, ttl): + # Set far-future cache expiry + if self.ds.cache_headers and response.status == 200: + ttl = int(ttl) + if ttl == 0: + ttl_header = "no-cache" + else: + ttl_header = f"max-age={ttl}" + response.headers["Cache-Control"] = ttl_header + response.headers["Referrer-Policy"] = "no-referrer" + if self.ds.cors: + add_cors_headers(response.headers) + return response From 579f59dcec43a91dd7d404e00b87a00afd8515f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 Apr 2022 11:33:35 -0700 Subject: [PATCH 0002/1172] Refactor to remove RowTableShared class, closes #1719 Refs #1715 --- datasette/app.py | 3 +- datasette/views/row.py | 142 +++++++++++ datasette/views/table.py | 497 +++++++++++++++------------------------ 3 files changed, 328 insertions(+), 314 deletions(-) create mode 100644 datasette/views/row.py diff --git a/datasette/app.py b/datasette/app.py index c9eede26..d269372c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -40,7 +40,8 @@ from .views.special import ( PermissionsDebugView, MessagesDebugView, ) -from .views.table import RowView, TableView +from .views.table import TableView +from .views.row import RowView from .renderer import json_renderer from .url_builder import Urls from .database import Database, QueryInterrupted diff --git a/datasette/views/row.py b/datasette/views/row.py new file mode 100644 index 00000000..b1c7362d --- /dev/null +++ b/datasette/views/row.py @@ -0,0 +1,142 @@ +from datasette.utils.asgi import NotFound +from datasette.database import QueryInterrupted +from .base import DataView +from datasette.utils import ( + tilde_decode, + urlsafe_components, + to_css_class, + escape_sqlite, +) +from .table import _sql_params_pks, display_columns_and_rows + + +class RowView(DataView): + name = "row" + + async def data(self, request, default_labels=False): + database_route = tilde_decode(request.url_vars["database"]) + table = tilde_decode(request.url_vars["table"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database = db.name + await self.ds.ensure_permissions( + request.actor, + [ + ("view-table", (database, table)), + ("view-database", database), + "view-instance", + ], + ) + pk_values = urlsafe_components(request.url_vars["pks"]) + try: + db = self.ds.get_database(route=database_route) + except KeyError: + raise NotFound("Database not found: {}".format(database_route)) + database = db.name + sql, params, pks = await _sql_params_pks(db, table, pk_values) + results = await db.execute(sql, params, truncate=True) + columns = [r[0] for r in results.description] + rows = list(results.rows) + if not rows: + raise NotFound(f"Record not found: {pk_values}") + + async def template_data(): + display_columns, display_rows = await display_columns_and_rows( + self.ds, + database, + table, + results.description, + rows, + link_column=False, + truncate_cells=0, + ) + for column in display_columns: + column["sortable"] = False + return { + "foreign_key_tables": await self.foreign_key_tables( + database, table, pk_values + ), + "display_columns": display_columns, + "display_rows": display_rows, + "custom_table_templates": [ + f"_table-{to_css_class(database)}-{to_css_class(table)}.html", + f"_table-row-{to_css_class(database)}-{to_css_class(table)}.html", + "_table.html", + ], + "metadata": (self.ds.metadata("databases") or {}) + .get(database, {}) + .get("tables", {}) + .get(table, {}), + } + + data = { + "database": database, + "table": table, + "rows": rows, + "columns": columns, + "primary_keys": pks, + "primary_key_values": pk_values, + "units": self.ds.table_metadata(database, table).get("units", {}), + } + + if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): + data["foreign_key_tables"] = await self.foreign_key_tables( + database, table, pk_values + ) + + return ( + data, + template_data, + ( + f"row-{to_css_class(database)}-{to_css_class(table)}.html", + "row.html", + ), + ) + + async def foreign_key_tables(self, database, table, pk_values): + if len(pk_values) != 1: + return [] + db = self.ds.databases[database] + all_foreign_keys = await db.get_all_foreign_keys() + foreign_keys = all_foreign_keys[table]["incoming"] + if len(foreign_keys) == 0: + return [] + + sql = "select " + ", ".join( + [ + "(select count(*) from {table} where {column}=:id)".format( + table=escape_sqlite(fk["other_table"]), + column=escape_sqlite(fk["other_column"]), + ) + for fk in foreign_keys + ] + ) + try: + rows = list(await db.execute(sql, {"id": pk_values[0]})) + except QueryInterrupted: + # Almost certainly hit the timeout + return [] + + foreign_table_counts = dict( + zip( + [(fk["other_table"], fk["other_column"]) for fk in foreign_keys], + list(rows[0]), + ) + ) + foreign_key_tables = [] + for fk in foreign_keys: + count = ( + foreign_table_counts.get((fk["other_table"], fk["other_column"])) or 0 + ) + key = fk["other_column"] + if key.startswith("_"): + key += "__exact" + link = "{}?{}={}".format( + self.ds.urls.table(database, fk["other_table"]), + key, + ",".join(pk_values), + ) + foreign_key_tables.append({**fk, **{"count": count, "link": link}}) + return foreign_key_tables diff --git a/datasette/views/table.py b/datasette/views/table.py index dc85165e..37fb2ebb 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,4 +1,3 @@ -import urllib import itertools import json @@ -9,7 +8,6 @@ from datasette.database import QueryInterrupted from datasette.utils import ( await_me_maybe, CustomRow, - MultiParams, append_querystring, compound_keys_after_sql, format_bytes, @@ -21,7 +19,6 @@ from datasette.utils import ( is_url, path_from_row_pks, path_with_added_args, - path_with_format, path_with_removed_args, path_with_replaced_args, to_css_class, @@ -68,7 +65,9 @@ class Row: return json.dumps(d, default=repr, indent=2) -class RowTableShared(DataView): +class TableView(DataView): + name = "table" + async def sortable_columns_for_table(self, database, table, use_rowid): db = self.ds.databases[database] table_metadata = self.ds.table_metadata(database, table) @@ -89,193 +88,6 @@ class RowTableShared(DataView): expandables.append((fk, label_column)) return expandables - async def display_columns_and_rows( - self, database, table, description, rows, link_column=False, truncate_cells=0 - ): - """Returns columns, rows for specified table - including fancy foreign key treatment""" - db = self.ds.databases[database] - table_metadata = self.ds.table_metadata(database, table) - column_descriptions = table_metadata.get("columns") or {} - column_details = {col.name: col for col in await db.table_column_details(table)} - sortable_columns = await self.sortable_columns_for_table(database, table, True) - pks = await db.primary_keys(table) - pks_for_display = pks - if not pks_for_display: - pks_for_display = ["rowid"] - - columns = [] - for r in description: - if r[0] == "rowid" and "rowid" not in column_details: - type_ = "integer" - notnull = 0 - else: - type_ = column_details[r[0]].type - notnull = column_details[r[0]].notnull - columns.append( - { - "name": r[0], - "sortable": r[0] in sortable_columns, - "is_pk": r[0] in pks_for_display, - "type": type_, - "notnull": notnull, - "description": column_descriptions.get(r[0]), - } - ) - - column_to_foreign_key_table = { - fk["column"]: fk["other_table"] - for fk in await db.foreign_keys_for_table(table) - } - - cell_rows = [] - base_url = self.ds.setting("base_url") - for row in rows: - cells = [] - # Unless we are a view, the first column is a link - either to the rowid - # or to the simple or compound primary key - if link_column: - is_special_link_column = len(pks) != 1 - pk_path = path_from_row_pks(row, pks, not pks, False) - cells.append( - { - "column": pks[0] if len(pks) == 1 else "Link", - "value_type": "pk", - "is_special_link_column": is_special_link_column, - "raw": pk_path, - "value": markupsafe.Markup( - '{flat_pks}'.format( - base_url=base_url, - table_path=self.ds.urls.table(database, table), - flat_pks=str(markupsafe.escape(pk_path)), - flat_pks_quoted=path_from_row_pks(row, pks, not pks), - ) - ), - } - ) - - for value, column_dict in zip(row, columns): - column = column_dict["name"] - if link_column and len(pks) == 1 and column == pks[0]: - # If there's a simple primary key, don't repeat the value as it's - # already shown in the link column. - continue - - # First let the plugins have a go - # pylint: disable=no-member - plugin_display_value = None - for candidate in pm.hook.render_cell( - value=value, - column=column, - table=table, - database=database, - datasette=self.ds, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - display_value = plugin_display_value - elif isinstance(value, bytes): - formatted = format_bytes(len(value)) - display_value = markupsafe.Markup( - '<Binary: {:,} byte{}>'.format( - self.ds.urls.row_blob( - database, - table, - path_from_row_pks(row, pks, not pks), - column, - ), - ' title="{}"'.format(formatted) - if "bytes" not in formatted - else "", - len(value), - "" if len(value) == 1 else "s", - ) - ) - elif isinstance(value, dict): - # It's an expanded foreign key - display link to other row - label = value["label"] - value = value["value"] - # The table we link to depends on the column - other_table = column_to_foreign_key_table[column] - link_template = ( - LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE - ) - display_value = markupsafe.Markup( - link_template.format( - database=database, - base_url=base_url, - table=tilde_encode(other_table), - link_id=tilde_encode(str(value)), - id=str(markupsafe.escape(value)), - label=str(markupsafe.escape(label)) or "-", - ) - ) - elif value in ("", None): - display_value = markupsafe.Markup(" ") - elif is_url(str(value).strip()): - display_value = markupsafe.Markup( - '{url}'.format( - url=markupsafe.escape(value.strip()) - ) - ) - elif column in table_metadata.get("units", {}) and value != "": - # Interpret units using pint - value = value * ureg(table_metadata["units"][column]) - # Pint uses floating point which sometimes introduces errors in the compact - # representation, which we have to round off to avoid ugliness. In the vast - # majority of cases this rounding will be inconsequential. I hope. - value = round(value.to_compact(), 6) - display_value = markupsafe.Markup( - f"{value:~P}".replace(" ", " ") - ) - else: - display_value = str(value) - if truncate_cells and len(display_value) > truncate_cells: - display_value = display_value[:truncate_cells] + "\u2026" - - cells.append( - { - "column": column, - "value": display_value, - "raw": value, - "value_type": "none" - if value is None - else str(type(value).__name__), - } - ) - cell_rows.append(Row(cells)) - - if link_column: - # Add the link column header. - # If it's a simple primary key, we have to remove and re-add that column name at - # the beginning of the header row. - first_column = None - if len(pks) == 1: - columns = [col for col in columns if col["name"] != pks[0]] - first_column = { - "name": pks[0], - "sortable": len(pks) == 1, - "is_pk": True, - "type": column_details[pks[0]].type, - "notnull": column_details[pks[0]].notnull, - } - else: - first_column = { - "name": "Link", - "sortable": False, - "is_pk": False, - "type": "", - "notnull": 0, - } - columns = [first_column] + columns - return columns, cell_rows - - -class TableView(RowTableShared): - name = "table" - async def post(self, request): database_route = tilde_decode(request.url_vars["database"]) try: @@ -807,13 +619,17 @@ class TableView(RowTableShared): async def extra_template(): nonlocal sort - display_columns, display_rows = await self.display_columns_and_rows( + display_columns, display_rows = await display_columns_and_rows( + self.ds, database, table, results.description, rows, link_column=not is_view, truncate_cells=self.ds.setting("truncate_cells_html"), + sortable_columns=await self.sortable_columns_for_table( + database, table, use_rowid=True + ), ) metadata = ( (self.ds.metadata("databases") or {}) @@ -948,132 +764,187 @@ async def _sql_params_pks(db, table, pk_values): return sql, params, pks -class RowView(RowTableShared): - name = "row" +async def display_columns_and_rows( + datasette, + database, + table, + description, + rows, + link_column=False, + truncate_cells=0, + sortable_columns=None, +): + """Returns columns, rows for specified table - including fancy foreign key treatment""" + sortable_columns = sortable_columns or set() + db = datasette.databases[database] + table_metadata = datasette.table_metadata(database, table) + column_descriptions = table_metadata.get("columns") or {} + column_details = {col.name: col for col in await db.table_column_details(table)} + pks = await db.primary_keys(table) + pks_for_display = pks + if not pks_for_display: + pks_for_display = ["rowid"] - async def data(self, request, default_labels=False): - database_route = tilde_decode(request.url_vars["database"]) - table = tilde_decode(request.url_vars["table"]) - try: - db = self.ds.get_database(route=database_route) - except KeyError: - raise NotFound("Database not found: {}".format(database_route)) - database = db.name - await self.ds.ensure_permissions( - request.actor, - [ - ("view-table", (database, table)), - ("view-database", database), - "view-instance", - ], - ) - pk_values = urlsafe_components(request.url_vars["pks"]) - try: - db = self.ds.get_database(route=database_route) - except KeyError: - raise NotFound("Database not found: {}".format(database_route)) - database = db.name - sql, params, pks = await _sql_params_pks(db, table, pk_values) - results = await db.execute(sql, params, truncate=True) - columns = [r[0] for r in results.description] - rows = list(results.rows) - if not rows: - raise NotFound(f"Record not found: {pk_values}") - - async def template_data(): - display_columns, display_rows = await self.display_columns_and_rows( - database, - table, - results.description, - rows, - link_column=False, - truncate_cells=0, - ) - for column in display_columns: - column["sortable"] = False - return { - "foreign_key_tables": await self.foreign_key_tables( - database, table, pk_values - ), - "display_columns": display_columns, - "display_rows": display_rows, - "custom_table_templates": [ - f"_table-{to_css_class(database)}-{to_css_class(table)}.html", - f"_table-row-{to_css_class(database)}-{to_css_class(table)}.html", - "_table.html", - ], - "metadata": (self.ds.metadata("databases") or {}) - .get(database, {}) - .get("tables", {}) - .get(table, {}), + columns = [] + for r in description: + if r[0] == "rowid" and "rowid" not in column_details: + type_ = "integer" + notnull = 0 + else: + type_ = column_details[r[0]].type + notnull = column_details[r[0]].notnull + columns.append( + { + "name": r[0], + "sortable": r[0] in sortable_columns, + "is_pk": r[0] in pks_for_display, + "type": type_, + "notnull": notnull, + "description": column_descriptions.get(r[0]), } - - data = { - "database": database, - "table": table, - "rows": rows, - "columns": columns, - "primary_keys": pks, - "primary_key_values": pk_values, - "units": self.ds.table_metadata(database, table).get("units", {}), - } - - if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): - data["foreign_key_tables"] = await self.foreign_key_tables( - database, table, pk_values - ) - - return ( - data, - template_data, - ( - f"row-{to_css_class(database)}-{to_css_class(table)}.html", - "row.html", - ), ) - async def foreign_key_tables(self, database, table, pk_values): - if len(pk_values) != 1: - return [] - db = self.ds.databases[database] - all_foreign_keys = await db.get_all_foreign_keys() - foreign_keys = all_foreign_keys[table]["incoming"] - if len(foreign_keys) == 0: - return [] + column_to_foreign_key_table = { + fk["column"]: fk["other_table"] for fk in await db.foreign_keys_for_table(table) + } - sql = "select " + ", ".join( - [ - "(select count(*) from {table} where {column}=:id)".format( - table=escape_sqlite(fk["other_table"]), - column=escape_sqlite(fk["other_column"]), + cell_rows = [] + base_url = datasette.setting("base_url") + for row in rows: + cells = [] + # Unless we are a view, the first column is a link - either to the rowid + # or to the simple or compound primary key + if link_column: + is_special_link_column = len(pks) != 1 + pk_path = path_from_row_pks(row, pks, not pks, False) + cells.append( + { + "column": pks[0] if len(pks) == 1 else "Link", + "value_type": "pk", + "is_special_link_column": is_special_link_column, + "raw": pk_path, + "value": markupsafe.Markup( + '{flat_pks}'.format( + base_url=base_url, + table_path=datasette.urls.table(database, table), + flat_pks=str(markupsafe.escape(pk_path)), + flat_pks_quoted=path_from_row_pks(row, pks, not pks), + ) + ), + } + ) + + for value, column_dict in zip(row, columns): + column = column_dict["name"] + if link_column and len(pks) == 1 and column == pks[0]: + # If there's a simple primary key, don't repeat the value as it's + # already shown in the link column. + continue + + # First let the plugins have a go + # pylint: disable=no-member + plugin_display_value = None + for candidate in pm.hook.render_cell( + value=value, + column=column, + table=table, + database=database, + datasette=datasette, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value: + display_value = plugin_display_value + elif isinstance(value, bytes): + formatted = format_bytes(len(value)) + display_value = markupsafe.Markup( + '<Binary: {:,} byte{}>'.format( + datasette.urls.row_blob( + database, + table, + path_from_row_pks(row, pks, not pks), + column, + ), + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "", + len(value), + "" if len(value) == 1 else "s", + ) ) - for fk in foreign_keys - ] - ) - try: - rows = list(await db.execute(sql, {"id": pk_values[0]})) - except QueryInterrupted: - # Almost certainly hit the timeout - return [] + elif isinstance(value, dict): + # It's an expanded foreign key - display link to other row + label = value["label"] + value = value["value"] + # The table we link to depends on the column + other_table = column_to_foreign_key_table[column] + link_template = LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE + display_value = markupsafe.Markup( + link_template.format( + database=database, + base_url=base_url, + table=tilde_encode(other_table), + link_id=tilde_encode(str(value)), + id=str(markupsafe.escape(value)), + label=str(markupsafe.escape(label)) or "-", + ) + ) + elif value in ("", None): + display_value = markupsafe.Markup(" ") + elif is_url(str(value).strip()): + display_value = markupsafe.Markup( + '{url}'.format( + url=markupsafe.escape(value.strip()) + ) + ) + elif column in table_metadata.get("units", {}) and value != "": + # Interpret units using pint + value = value * ureg(table_metadata["units"][column]) + # Pint uses floating point which sometimes introduces errors in the compact + # representation, which we have to round off to avoid ugliness. In the vast + # majority of cases this rounding will be inconsequential. I hope. + value = round(value.to_compact(), 6) + display_value = markupsafe.Markup(f"{value:~P}".replace(" ", " ")) + else: + display_value = str(value) + if truncate_cells and len(display_value) > truncate_cells: + display_value = display_value[:truncate_cells] + "\u2026" - foreign_table_counts = dict( - zip( - [(fk["other_table"], fk["other_column"]) for fk in foreign_keys], - list(rows[0]), + cells.append( + { + "column": column, + "value": display_value, + "raw": value, + "value_type": "none" + if value is None + else str(type(value).__name__), + } ) - ) - foreign_key_tables = [] - for fk in foreign_keys: - count = ( - foreign_table_counts.get((fk["other_table"], fk["other_column"])) or 0 - ) - key = fk["other_column"] - if key.startswith("_"): - key += "__exact" - link = "{}?{}={}".format( - self.ds.urls.table(database, fk["other_table"]), - key, - ",".join(pk_values), - ) - foreign_key_tables.append({**fk, **{"count": count, "link": link}}) - return foreign_key_tables + cell_rows.append(Row(cells)) + + if link_column: + # Add the link column header. + # If it's a simple primary key, we have to remove and re-add that column name at + # the beginning of the header row. + first_column = None + if len(pks) == 1: + columns = [col for col in columns if col["name"] != pks[0]] + first_column = { + "name": pks[0], + "sortable": len(pks) == 1, + "is_pk": True, + "type": column_details[pks[0]].type, + "notnull": column_details[pks[0]].notnull, + } + else: + first_column = { + "name": "Link", + "sortable": False, + "is_pk": False, + "type": "", + "notnull": 0, + } + columns = [first_column] + columns + return columns, cell_rows From c101f0efeec4f6e49298a542c5e2b59236cfa0ff Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 Apr 2022 15:34:29 -0700 Subject: [PATCH 0003/1172] datasette-total-page-time example of asgi_wrapper --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4560ec9a..3c9ae2e2 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -801,7 +801,7 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att return wrap_with_databases_header -Examples: `datasette-cors `__, `datasette-pyinstrument `__ +Examples: `datasette-cors `__, `datasette-pyinstrument `__, `datasette-total-page-time `__ .. _plugin_hook_startup: From 8a0c38f0b89543e652a968a90d480859cb102510 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 Apr 2022 13:56:27 -0700 Subject: [PATCH 0004/1172] Rename database->database_name and table-> table_name, refs #1715 --- datasette/views/table.py | 143 +++++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 67 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 37fb2ebb..d66adb82 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -68,22 +68,22 @@ class Row: class TableView(DataView): name = "table" - async def sortable_columns_for_table(self, database, table, use_rowid): - db = self.ds.databases[database] - table_metadata = self.ds.table_metadata(database, table) + async def sortable_columns_for_table(self, database_name, table_name, use_rowid): + db = self.ds.databases[database_name] + table_metadata = self.ds.table_metadata(database_name, table_name) if "sortable_columns" in table_metadata: sortable_columns = set(table_metadata["sortable_columns"]) else: - sortable_columns = set(await db.table_columns(table)) + sortable_columns = set(await db.table_columns(table_name)) if use_rowid: sortable_columns.add("rowid") return sortable_columns - async def expandable_columns(self, database, table): + async def expandable_columns(self, database_name, table_name): # Returns list of (fk_dict, label_column-or-None) pairs for that table expandables = [] - db = self.ds.databases[database] - for fk in await db.foreign_keys_for_table(table): + db = self.ds.databases[database_name] + for fk in await db.foreign_keys_for_table(table_name): label_column = await db.label_column_for_table(fk["other_table"]) expandables.append((fk, label_column)) return expandables @@ -94,17 +94,19 @@ class TableView(DataView): db = self.ds.get_database(route=database_route) except KeyError: raise NotFound("Database not found: {}".format(database_route)) - database = db.name - table = tilde_decode(request.url_vars["table"]) + database_name = db.name + table_name = tilde_decode(request.url_vars["table"]) # Handle POST to a canned query - canned_query = await self.ds.get_canned_query(database, table, request.actor) + canned_query = await self.ds.get_canned_query( + database_name, table_name, request.actor + ) assert canned_query, "You may only POST to a canned query" return await QueryView(self.ds).data( request, canned_query["sql"], metadata=canned_query, editable=False, - canned_query=table, + canned_query=table_name, named_parameters=canned_query.get("params"), write=bool(canned_query.get("write")), ) @@ -150,45 +152,47 @@ class TableView(DataView): _size=None, ): database_route = tilde_decode(request.url_vars["database"]) - table = tilde_decode(request.url_vars["table"]) + table_name = tilde_decode(request.url_vars["table"]) try: db = self.ds.get_database(route=database_route) except KeyError: raise NotFound("Database not found: {}".format(database_route)) - database = db.name + database_name = db.name # If this is a canned query, not a table, then dispatch to QueryView instead - canned_query = await self.ds.get_canned_query(database, table, request.actor) + canned_query = await self.ds.get_canned_query( + database_name, table_name, request.actor + ) if canned_query: return await QueryView(self.ds).data( request, canned_query["sql"], metadata=canned_query, editable=False, - canned_query=table, + canned_query=table_name, named_parameters=canned_query.get("params"), write=bool(canned_query.get("write")), ) - is_view = bool(await db.get_view_definition(table)) - table_exists = bool(await db.table_exists(table)) + is_view = bool(await db.get_view_definition(table_name)) + table_exists = bool(await db.table_exists(table_name)) # If table or view not found, return 404 if not is_view and not table_exists: - raise NotFound(f"Table not found: {table}") + raise NotFound(f"Table not found: {table_name}") # Ensure user has permission to view this table await self.ds.ensure_permissions( request.actor, [ - ("view-table", (database, table)), - ("view-database", database), + ("view-table", (database_name, table_name)), + ("view-database", database_name), "view-instance", ], ) private = not await self.ds.permission_allowed( - None, "view-table", (database, table), default=True + None, "view-table", (database_name, table_name), default=True ) # Handle ?_filter_column and redirect, if present @@ -216,8 +220,8 @@ class TableView(DataView): ) # Introspect columns and primary keys for table - pks = await db.primary_keys(table) - table_columns = await db.table_columns(table) + pks = await db.primary_keys(table_name) + table_columns = await db.table_columns(table_name) # Take ?_col= and ?_nocol= into account specified_columns = await self.columns_to_select(table_columns, pks, request) @@ -248,7 +252,7 @@ class TableView(DataView): nocount = True nofacet = True - table_metadata = self.ds.table_metadata(database, table) + table_metadata = self.ds.table_metadata(database_name, table_name) units = table_metadata.get("units", {}) # Arguments that start with _ and don't contain a __ are @@ -262,7 +266,7 @@ class TableView(DataView): # Build where clauses from query string arguments filters = Filters(sorted(filter_args), units, ureg) - where_clauses, params = filters.build_where_clauses(table) + where_clauses, params = filters.build_where_clauses(table_name) # Execute filters_from_request plugin hooks - including the default # ones that live in datasette/filters.py @@ -271,8 +275,8 @@ class TableView(DataView): for hook in pm.hook.filters_from_request( request=request, - table=table, - database=database, + table=table_name, + database=database_name, datasette=self.ds, ): filter_arguments = await await_me_maybe(hook) @@ -284,7 +288,7 @@ class TableView(DataView): # Deal with custom sort orders sortable_columns = await self.sortable_columns_for_table( - database, table, use_rowid + database_name, table_name, use_rowid ) sort = request.args.get("_sort") sort_desc = request.args.get("_sort_desc") @@ -309,7 +313,7 @@ class TableView(DataView): order_by = f"{escape_sqlite(sort_desc)} desc" from_sql = "from {table_name} {where}".format( - table_name=escape_sqlite(table), + table_name=escape_sqlite(table_name), where=("where {} ".format(" and ".join(where_clauses))) if where_clauses else "", @@ -422,7 +426,7 @@ class TableView(DataView): sql_no_order_no_limit = ( "select {select_all_columns} from {table_name} {where}".format( select_all_columns=select_all_columns, - table_name=escape_sqlite(table), + table_name=escape_sqlite(table_name), where=where_clause, ) ) @@ -430,7 +434,7 @@ class TableView(DataView): # This is the SQL that populates the main table on the page sql = "select {select_specified_columns} from {table_name} {where}{order_by} limit {page_size}{offset}".format( select_specified_columns=select_specified_columns, - table_name=escape_sqlite(table), + table_name=escape_sqlite(table_name), where=where_clause, order_by=order_by, page_size=page_size + 1, @@ -448,13 +452,13 @@ class TableView(DataView): if ( not db.is_mutable and self.ds.inspect_data - and count_sql == f"select count(*) from {table} " + and count_sql == f"select count(*) from {table_name} " ): # We can use a previously cached table row count try: - filtered_table_rows_count = self.ds.inspect_data[database]["tables"][ - table - ]["count"] + filtered_table_rows_count = self.ds.inspect_data[database_name][ + "tables" + ][table_name]["count"] except KeyError: pass @@ -484,10 +488,10 @@ class TableView(DataView): klass( self.ds, request, - database, + database_name, sql=sql_no_order_no_limit, params=params, - table=table, + table=table_name, metadata=table_metadata, row_count=filtered_table_rows_count, ) @@ -527,7 +531,7 @@ class TableView(DataView): # Expand labeled columns if requested expanded_columns = [] - expandable_columns = await self.expandable_columns(database, table) + expandable_columns = await self.expandable_columns(database_name, table_name) columns_to_expand = None try: all_labels = value_as_boolean(request.args.get("_labels", "")) @@ -554,7 +558,9 @@ class TableView(DataView): values = [row[column_index] for row in rows] # Expand them expanded_labels.update( - await self.ds.expand_foreign_keys(database, table, column, values) + await self.ds.expand_foreign_keys( + database_name, table_name, column, values + ) ) if expanded_labels: # Rewrite the rows @@ -621,21 +627,21 @@ class TableView(DataView): display_columns, display_rows = await display_columns_and_rows( self.ds, - database, - table, + database_name, + table_name, results.description, rows, link_column=not is_view, truncate_cells=self.ds.setting("truncate_cells_html"), sortable_columns=await self.sortable_columns_for_table( - database, table, use_rowid=True + database_name, table_name, use_rowid=True ), ) metadata = ( (self.ds.metadata("databases") or {}) - .get(database, {}) + .get(database_name, {}) .get("tables", {}) - .get(table, {}) + .get(table_name, {}) ) self.ds.update_with_inherited_metadata(metadata) @@ -661,8 +667,8 @@ class TableView(DataView): links = [] for hook in pm.hook.table_actions( datasette=self.ds, - table=table, - database=database, + table=table_name, + database=database_name, actor=request.actor, request=request, ): @@ -703,13 +709,13 @@ class TableView(DataView): "sort_desc": sort_desc, "disable_sort": is_view, "custom_table_templates": [ - f"_table-{to_css_class(database)}-{to_css_class(table)}.html", - f"_table-table-{to_css_class(database)}-{to_css_class(table)}.html", + f"_table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", + f"_table-table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", "_table.html", ], "metadata": metadata, - "view_definition": await db.get_view_definition(table), - "table_definition": await db.get_table_definition(table), + "view_definition": await db.get_view_definition(table_name), + "table_definition": await db.get_table_definition(table_name), "datasette_allow_facet": "true" if self.ds.setting("allow_facet") else "false", @@ -719,8 +725,8 @@ class TableView(DataView): return ( { - "database": database, - "table": table, + "database": database_name, + "table": table_name, "is_view": is_view, "human_description_en": human_description_en, "rows": rows[:page_size], @@ -738,12 +744,12 @@ class TableView(DataView): "next_url": next_url, "private": private, "allow_execute_sql": await self.ds.permission_allowed( - request.actor, "execute-sql", database, default=True + request.actor, "execute-sql", database_name, default=True ), }, extra_template, ( - f"table-{to_css_class(database)}-{to_css_class(table)}.html", + f"table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", "table.html", ), ) @@ -766,8 +772,8 @@ async def _sql_params_pks(db, table, pk_values): async def display_columns_and_rows( datasette, - database, - table, + database_name, + table_name, description, rows, link_column=False, @@ -776,11 +782,13 @@ async def display_columns_and_rows( ): """Returns columns, rows for specified table - including fancy foreign key treatment""" sortable_columns = sortable_columns or set() - db = datasette.databases[database] - table_metadata = datasette.table_metadata(database, table) + db = datasette.databases[database_name] + table_metadata = datasette.table_metadata(database_name, table_name) column_descriptions = table_metadata.get("columns") or {} - column_details = {col.name: col for col in await db.table_column_details(table)} - pks = await db.primary_keys(table) + column_details = { + col.name: col for col in await db.table_column_details(table_name) + } + pks = await db.primary_keys(table_name) pks_for_display = pks if not pks_for_display: pks_for_display = ["rowid"] @@ -805,7 +813,8 @@ async def display_columns_and_rows( ) column_to_foreign_key_table = { - fk["column"]: fk["other_table"] for fk in await db.foreign_keys_for_table(table) + fk["column"]: fk["other_table"] + for fk in await db.foreign_keys_for_table(table_name) } cell_rows = [] @@ -826,7 +835,7 @@ async def display_columns_and_rows( "value": markupsafe.Markup( '{flat_pks}'.format( base_url=base_url, - table_path=datasette.urls.table(database, table), + table_path=datasette.urls.table(database_name, table_name), flat_pks=str(markupsafe.escape(pk_path)), flat_pks_quoted=path_from_row_pks(row, pks, not pks), ) @@ -847,8 +856,8 @@ async def display_columns_and_rows( for candidate in pm.hook.render_cell( value=value, column=column, - table=table, - database=database, + table=table_name, + database=database_name, datasette=datasette, ): candidate = await await_me_maybe(candidate) @@ -862,8 +871,8 @@ async def display_columns_and_rows( display_value = markupsafe.Markup( '<Binary: {:,} byte{}>'.format( datasette.urls.row_blob( - database, - table, + database_name, + table_name, path_from_row_pks(row, pks, not pks), column, ), @@ -883,7 +892,7 @@ async def display_columns_and_rows( link_template = LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE display_value = markupsafe.Markup( link_template.format( - database=database, + database=database_name, base_url=base_url, table=tilde_encode(other_table), link_id=tilde_encode(str(value)), From 942411ef946e9a34a2094944d3423cddad27efd3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 Apr 2022 15:48:56 -0700 Subject: [PATCH 0005/1172] Execute some TableView queries in parallel Use ?_noparallel=1 to opt out (undocumented, useful for benchmark comparisons) Refs #1723, #1715 --- datasette/views/table.py | 91 +++++++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 25 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index d66adb82..23289b29 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1,3 +1,4 @@ +import asyncio import itertools import json @@ -5,6 +6,7 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted +from datasette import tracer from datasette.utils import ( await_me_maybe, CustomRow, @@ -150,6 +152,16 @@ class TableView(DataView): default_labels=False, _next=None, _size=None, + ): + with tracer.trace_child_tasks(): + return await self._data_traced(request, default_labels, _next, _size) + + async def _data_traced( + self, + request, + default_labels=False, + _next=None, + _size=None, ): database_route = tilde_decode(request.url_vars["database"]) table_name = tilde_decode(request.url_vars["table"]) @@ -159,6 +171,20 @@ class TableView(DataView): raise NotFound("Database not found: {}".format(database_route)) database_name = db.name + # For performance profiling purposes, ?_noparallel=1 turns off asyncio.gather + async def _gather_parallel(*args): + return await asyncio.gather(*args) + + async def _gather_sequential(*args): + results = [] + for fn in args: + results.append(await fn) + return results + + gather = ( + _gather_sequential if request.args.get("_noparallel") else _gather_parallel + ) + # If this is a canned query, not a table, then dispatch to QueryView instead canned_query = await self.ds.get_canned_query( database_name, table_name, request.actor @@ -174,8 +200,12 @@ class TableView(DataView): write=bool(canned_query.get("write")), ) - is_view = bool(await db.get_view_definition(table_name)) - table_exists = bool(await db.table_exists(table_name)) + is_view, table_exists = map( + bool, + await gather( + db.get_view_definition(table_name), db.table_exists(table_name) + ), + ) # If table or view not found, return 404 if not is_view and not table_exists: @@ -497,33 +527,44 @@ class TableView(DataView): ) ) - if not nofacet: - for facet in facet_instances: - ( + async def execute_facets(): + if not nofacet: + # Run them in parallel + facet_awaitables = [facet.facet_results() for facet in facet_instances] + facet_awaitable_results = await gather(*facet_awaitables) + for ( instance_facet_results, instance_facets_timed_out, - ) = await facet.facet_results() - for facet_info in instance_facet_results: - base_key = facet_info["name"] - key = base_key - i = 1 - while key in facet_results: - i += 1 - key = f"{base_key}_{i}" - facet_results[key] = facet_info - facets_timed_out.extend(instance_facets_timed_out) + ) in facet_awaitable_results: + for facet_info in instance_facet_results: + base_key = facet_info["name"] + key = base_key + i = 1 + while key in facet_results: + i += 1 + key = f"{base_key}_{i}" + facet_results[key] = facet_info + facets_timed_out.extend(instance_facets_timed_out) - # Calculate suggested facets suggested_facets = [] - if ( - self.ds.setting("suggest_facets") - and self.ds.setting("allow_facet") - and not _next - and not nofacet - and not nosuggest - ): - for facet in facet_instances: - suggested_facets.extend(await facet.suggest()) + + async def execute_suggested_facets(): + # Calculate suggested facets + if ( + self.ds.setting("suggest_facets") + and self.ds.setting("allow_facet") + and not _next + and not nofacet + and not nosuggest + ): + # Run them in parallel + facet_suggest_awaitables = [ + facet.suggest() for facet in facet_instances + ] + for suggest_result in await gather(*facet_suggest_awaitables): + suggested_facets.extend(suggest_result) + + await gather(execute_facets(), execute_suggested_facets()) # Figure out columns and rows for the query columns = [r[0] for r in results.description] From 94a3171b01fde5c52697aeeff052e3ad4bab5391 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 Apr 2022 13:29:11 -0700 Subject: [PATCH 0006/1172] .plugin_config() can return None --- docs/internals.rst | 4 ++++ docs/writing_plugins.rst | 2 ++ 2 files changed, 6 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index aad608dc..18822d47 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -288,6 +288,10 @@ All databases are listed, irrespective of user permissions. This means that the This method lets you read plugin configuration values that were set in ``metadata.json``. See :ref:`writing_plugins_configuration` for full details of how this method should be used. +The return value will be the value from the configuration file - usually a dictionary. + +If the plugin is not configured the return value will be ``None``. + .. _datasette_render_template: await .render_template(template, context=None, request=None) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 89f7f5eb..9aee70f6 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -182,6 +182,8 @@ When you are writing plugins, you can access plugin configuration like this usin This will return the ``{"latitude_column": "lat", "longitude_column": "lng"}`` in the above example. +If there is no configuration for that plugin, the method will return ``None``. + If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option like so:: { From 4afc1afc721ac0d14f58b0f8339c1bf431d5313c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 12:13:11 -0700 Subject: [PATCH 0007/1172] Depend on click-default-group-wheel>=1.2.2 Refs #1733 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7f0562fd..fcb43aa1 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( install_requires=[ "asgiref>=3.2.10,<3.6.0", "click>=7.1.1,<8.2.0", - "click-default-group~=1.2.2", + "click-default-group-wheel>=1.2.2", "Jinja2>=2.10.3,<3.1.0", "hupper~=1.9", "httpx>=0.20", From 7e03394734307a5761e4c98d902b6a8cab188562 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 12:20:14 -0700 Subject: [PATCH 0008/1172] Optional uvicorn import for Pyodide, refs #1733 --- datasette/app.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index d269372c..a5330458 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -26,7 +26,6 @@ from itsdangerous import URLSafeSerializer from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound -import uvicorn from .views.base import DatasetteError, ureg from .views.database import DatabaseDownload, DatabaseView @@ -806,6 +805,15 @@ class Datasette: datasette_version = {"version": __version__} if self.version_note: datasette_version["note"] = self.version_note + + try: + # Optional import to avoid breaking Pyodide + # https://github.com/simonw/datasette/issues/1733#issuecomment-1115268245 + import uvicorn + + uvicorn_version = uvicorn.__version__ + except ImportError: + uvicorn_version = None info = { "python": { "version": ".".join(map(str, sys.version_info[:3])), @@ -813,7 +821,7 @@ class Datasette: }, "datasette": datasette_version, "asgi": "3.0", - "uvicorn": uvicorn.__version__, + "uvicorn": uvicorn_version, "sqlite": { "version": sqlite_version, "fts_versions": fts_versions, From 687907aa2b1bde4de6ae7155b0e2a949ca015ca9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 12:39:06 -0700 Subject: [PATCH 0009/1172] Remove python-baseconv dependency, refs #1733, closes #1734 --- datasette/actor_auth_cookie.py | 2 +- datasette/utils/baseconv.py | 59 ++++++++++++++++++++++++++++++++++ docs/authentication.rst | 4 +-- setup.py | 1 - tests/test_auth.py | 2 +- 5 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 datasette/utils/baseconv.py diff --git a/datasette/actor_auth_cookie.py b/datasette/actor_auth_cookie.py index 15ecd331..368213af 100644 --- a/datasette/actor_auth_cookie.py +++ b/datasette/actor_auth_cookie.py @@ -1,6 +1,6 @@ from datasette import hookimpl from itsdangerous import BadSignature -import baseconv +from datasette.utils import baseconv import time diff --git a/datasette/utils/baseconv.py b/datasette/utils/baseconv.py new file mode 100644 index 00000000..27e4fb00 --- /dev/null +++ b/datasette/utils/baseconv.py @@ -0,0 +1,59 @@ +""" +Convert numbers from base 10 integers to base X strings and back again. + +Sample usage: + +>>> base20 = BaseConverter('0123456789abcdefghij') +>>> base20.from_decimal(1234) +'31e' +>>> base20.to_decimal('31e') +1234 + +Originally shared here: https://www.djangosnippets.org/snippets/1431/ +""" + + +class BaseConverter(object): + decimal_digits = "0123456789" + + def __init__(self, digits): + self.digits = digits + + def from_decimal(self, i): + return self.convert(i, self.decimal_digits, self.digits) + + def to_decimal(self, s): + return int(self.convert(s, self.digits, self.decimal_digits)) + + def convert(number, fromdigits, todigits): + # Based on http://code.activestate.com/recipes/111286/ + if str(number)[0] == "-": + number = str(number)[1:] + neg = 1 + else: + neg = 0 + + # make an integer out of the number + x = 0 + for digit in str(number): + x = x * len(fromdigits) + fromdigits.index(digit) + + # create the result in base 'len(todigits)' + if x == 0: + res = todigits[0] + else: + res = "" + while x > 0: + digit = x % len(todigits) + res = todigits[digit] + res + x = int(x / len(todigits)) + if neg: + res = "-" + res + return res + + convert = staticmethod(convert) + + +bin = BaseConverter("01") +hexconv = BaseConverter("0123456789ABCDEF") +base62 = BaseConverter("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz") diff --git a/docs/authentication.rst b/docs/authentication.rst index 24960733..685dab15 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -401,12 +401,12 @@ Including an expiry time ``ds_actor`` cookies can optionally include a signed expiry timestamp, after which the cookies will no longer be valid. Authentication plugins may chose to use this mechanism to limit the lifetime of the cookie. For example, if a plugin implements single-sign-on against another source it may decide to set short-lived cookies so that if the user is removed from the SSO system their existing Datasette cookies will stop working shortly afterwards. -To include an expiry, add a ``"e"`` key to the cookie value containing a `base62-encoded integer `__ representing the timestamp when the cookie should expire. For example, here's how to set a cookie that expires after 24 hours: +To include an expiry, add a ``"e"`` key to the cookie value containing a base62-encoded integer representing the timestamp when the cookie should expire. For example, here's how to set a cookie that expires after 24 hours: .. code-block:: python import time - import baseconv + from datasette.utils import baseconv expires_at = int(time.time()) + (24 * 60 * 60) diff --git a/setup.py b/setup.py index fcb43aa1..ca449f02 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,6 @@ setup( "PyYAML>=5.3,<7.0", "mergedeep>=1.1.1,<1.4.0", "itsdangerous>=1.1,<3.0", - "python-baseconv==1.2.2", ], entry_points=""" [console_scripts] diff --git a/tests/test_auth.py b/tests/test_auth.py index 974f89ea..4ef35a76 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,5 +1,5 @@ from .fixtures import app_client -import baseconv +from datasette.utils import baseconv import pytest import time From a29c1277896b6a7905ef5441c42a37bc15f67599 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 12:44:09 -0700 Subject: [PATCH 0010/1172] Rename to_decimal/from_decimal to decode/encode, refs #1734 --- datasette/utils/baseconv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/utils/baseconv.py b/datasette/utils/baseconv.py index 27e4fb00..c4b64908 100644 --- a/datasette/utils/baseconv.py +++ b/datasette/utils/baseconv.py @@ -19,10 +19,10 @@ class BaseConverter(object): def __init__(self, digits): self.digits = digits - def from_decimal(self, i): + def encode(self, i): return self.convert(i, self.decimal_digits, self.digits) - def to_decimal(self, s): + def decode(self, s): return int(self.convert(s, self.digits, self.decimal_digits)) def convert(number, fromdigits, todigits): From 3f00a29141bdea5be747f6d1c93871ccdb792167 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 13:15:27 -0700 Subject: [PATCH 0011/1172] Clean up compatibility with Pyodide (#1736) * Optional uvicorn import for Pyodide, refs #1733 * --setting num_sql_threads 0 to disable threading, refs #1735 --- datasette/app.py | 11 ++++++++--- datasette/database.py | 19 +++++++++++++++++++ docs/settings.rst | 2 ++ tests/test_internals_datasette.py | 14 +++++++++++++- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index a5330458..b7b84371 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -288,9 +288,12 @@ class Datasette: self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note - self.executor = futures.ThreadPoolExecutor( - max_workers=self.setting("num_sql_threads") - ) + if self.setting("num_sql_threads") == 0: + self.executor = None + else: + self.executor = futures.ThreadPoolExecutor( + max_workers=self.setting("num_sql_threads") + ) self.max_returned_rows = self.setting("max_returned_rows") self.sql_time_limit_ms = self.setting("sql_time_limit_ms") self.page_size = self.setting("default_page_size") @@ -862,6 +865,8 @@ class Datasette: ] def _threads(self): + if self.setting("num_sql_threads") == 0: + return {"num_threads": 0, "threads": []} threads = list(threading.enumerate()) d = { "num_threads": len(threads), diff --git a/datasette/database.py b/datasette/database.py index ba594a8c..44d32667 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -45,6 +45,9 @@ class Database: self._cached_table_counts = None self._write_thread = None self._write_queue = None + # These are used when in non-threaded mode: + self._read_connection = None + self._write_connection = None if not self.is_mutable and not self.is_memory: p = Path(path) self.hash = inspect_hash(p) @@ -134,6 +137,14 @@ class Database: return results async def execute_write_fn(self, fn, block=True): + if self.ds.executor is None: + # non-threaded mode + if self._write_connection is None: + self._write_connection = self.connect(write=True) + self.ds._prepare_connection(self._write_connection, self.name) + return fn(self._write_connection) + + # threaded mode task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") if self._write_queue is None: self._write_queue = queue.Queue() @@ -177,6 +188,14 @@ class Database: task.reply_queue.sync_q.put(result) async def execute_fn(self, fn): + if self.ds.executor is None: + # non-threaded mode + if self._read_connection is None: + self._read_connection = self.connect() + self.ds._prepare_connection(self._read_connection, self.name) + return fn(self._read_connection) + + # threaded mode def in_thread(): conn = getattr(connections, self.name, None) if not conn: diff --git a/docs/settings.rst b/docs/settings.rst index 60c4b36d..8437fb04 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -107,6 +107,8 @@ Maximum number of threads in the thread pool Datasette uses to execute SQLite qu datasette mydatabase.db --setting num_sql_threads 10 +Setting this to 0 turns off threaded SQL queries entirely - useful for environments that do not support threading such as `Pyodide `__. + .. _setting_allow_facet: allow_facet diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index cc200a2d..1dc14cab 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -1,7 +1,7 @@ """ Tests for the datasette.app.Datasette class """ -from datasette.app import Datasette +from datasette.app import Datasette, Database from itsdangerous import BadSignature from .fixtures import app_client import pytest @@ -63,3 +63,15 @@ async def test_datasette_constructor(): "hash": None, } ] + + +@pytest.mark.asyncio +async def test_num_sql_threads_zero(): + ds = Datasette([], memory=True, settings={"num_sql_threads": 0}) + db = ds.add_database(Database(ds, memory_name="test_num_sql_threads_zero")) + await db.execute_write("create table t(id integer primary key)") + await db.execute_write("insert into t (id) values (1)") + response = await ds.client.get("/-/threads.json") + assert response.json() == {"num_threads": 0, "threads": []} + response2 = await ds.client.get("/test_num_sql_threads_zero/t.json?_shape=array") + assert response2.json() == [{"id": 1}] From 943aa2e1f7341cb51e60332cde46bde650c64217 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 14:38:34 -0700 Subject: [PATCH 0012/1172] Release 0.62a0 Refs #1683, #1701, #1712, #1717, #1718, #1733 --- datasette/version.py | 2 +- docs/changelog.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 02451a1e..cf18c441 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.61.1" +__version__ = "0.62a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 03cf62b6..74814fcb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,20 @@ Changelog ========= +.. _v0_62a0: + +0.62a0 (2022-05-02) +------------------- + +- Datasette now runs some SQL queries in parallel. This has limited impact on performance, see `this research issue `__ for details. +- Datasette should now be compatible with Pyodide. (:issue:`1733`) +- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (`#1717 `__) +- Spaces in database names are now encoded as ``+`` rather than ``~20``. (:issue:`1701`) +- ```` is now displayed as ```` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`) +- Don't show the facet option in the cog menu if faceting is not allowed. (:issue:`1683`) +- Code examples in the documentation are now all formatted using Black. (:issue:`1718`) +- ``Request.fake()`` method is now documented, see :ref:`internals_request`. + .. _v0_61_1: 0.61.1 (2022-03-23) From 847d6b1aac38c3e776e8c600eed07ba4c9ac9942 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 16:32:24 -0700 Subject: [PATCH 0013/1172] Test wheel against Pyodide, refs #1737, #1733 --- .github/workflows/test-pyodide.yml | 28 ++++++++++++++++++ test-in-pyodide-with-shot-scraper.sh | 43 ++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 .github/workflows/test-pyodide.yml create mode 100755 test-in-pyodide-with-shot-scraper.sh diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml new file mode 100644 index 00000000..3715d055 --- /dev/null +++ b/.github/workflows/test-pyodide.yml @@ -0,0 +1,28 @@ +name: Test in Pyodide with shot-scraper + +on: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + cache: 'pip' + cache-dependency-path: '**/setup.py' + - name: Cache Playwright browsers + uses: actions/cache@v2 + with: + path: ~/.cache/ms-playwright/ + key: ${{ runner.os }}-browsers + - name: Install Playwright dependencies + run: | + pip install shot-scraper + shot-scraper install + - name: Run test + run: | + ./test-in-pyodide-with-shot-scraper.sh diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh new file mode 100755 index 00000000..0f29c0e0 --- /dev/null +++ b/test-in-pyodide-with-shot-scraper.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Build the wheel +python3 -m build + +# Find name of wheel +wheel=$(basename $(ls dist/*.whl)) +# strip off the dist/ + + +# Create a blank index page +echo ' + +' > dist/index.html + +# Run a server for that dist/ folder +cd dist +python3 -m http.server 8529 & +cd .. + +shot-scraper javascript http://localhost:8529/ " +async () => { + let pyodide = await loadPyodide(); + await pyodide.loadPackage(['micropip', 'ssl', 'setuptools']); + let output = await pyodide.runPythonAsync(\` + import micropip + await micropip.install('h11==0.12.0') + await micropip.install('http://localhost:8529/$wheel') + import ssl + import setuptools + from datasette.app import Datasette + ds = Datasette(memory=True, settings={'num_sql_threads': 0}) + (await ds.client.get('/_memory.json?sql=select+55+as+itworks&_shape=array')).text + \`); + if (JSON.parse(output)[0].itworks != 55) { + throw 'Got ' + output + ', expected itworks: 55'; + } + return 'Test passed!'; +} +" + +# Shut down the server +pkill -f 'http.server 8529' From c0cbcf2aba0d8393ba464acc515803ebf2eeda12 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 16:36:58 -0700 Subject: [PATCH 0014/1172] Tweaks to test scripts, refs #1737 --- .github/workflows/test-pyodide.yml | 2 +- test-in-pyodide-with-shot-scraper.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 3715d055..beb6a5fb 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -21,7 +21,7 @@ jobs: key: ${{ runner.os }}-browsers - name: Install Playwright dependencies run: | - pip install shot-scraper + pip install shot-scraper build shot-scraper install - name: Run test run: | diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh index 0f29c0e0..e5df7398 100755 --- a/test-in-pyodide-with-shot-scraper.sh +++ b/test-in-pyodide-with-shot-scraper.sh @@ -1,12 +1,12 @@ #!/bin/bash +set -e +# So the script fails if there are any errors # Build the wheel python3 -m build -# Find name of wheel +# Find name of wheel, strip off the dist/ wheel=$(basename $(ls dist/*.whl)) -# strip off the dist/ - # Create a blank index page echo ' From d60f163528f466b1127b2935c3b6869c34fd6545 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 2 May 2022 16:40:49 -0700 Subject: [PATCH 0015/1172] Run on push and PR, closes #1737 --- .github/workflows/test-pyodide.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index beb6a5fb..1b75aade 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -1,6 +1,8 @@ name: Test in Pyodide with shot-scraper on: + push: + pull_request: workflow_dispatch: jobs: From 280ff372ab30df244f6c54f6f3002da57334b3d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 May 2022 07:59:18 -0700 Subject: [PATCH 0016/1172] ETag support for .db downloads, closes #1739 --- datasette/utils/testing.py | 20 ++++++++++++++++++-- datasette/views/database.py | 7 +++++++ tests/test_html.py | 10 +++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index 94750b1f..640c94e6 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -55,10 +55,21 @@ class TestClient: @async_to_sync async def get( - self, path, follow_redirects=False, redirect_count=0, method="GET", cookies=None + self, + path, + follow_redirects=False, + redirect_count=0, + method="GET", + cookies=None, + if_none_match=None, ): return await self._request( - path, follow_redirects, redirect_count, method, cookies + path=path, + follow_redirects=follow_redirects, + redirect_count=redirect_count, + method=method, + cookies=cookies, + if_none_match=if_none_match, ) @async_to_sync @@ -110,6 +121,7 @@ class TestClient: headers=None, post_body=None, content_type=None, + if_none_match=None, ): return await self._request( path, @@ -120,6 +132,7 @@ class TestClient: headers=headers, post_body=post_body, content_type=content_type, + if_none_match=if_none_match, ) async def _request( @@ -132,10 +145,13 @@ class TestClient: headers=None, post_body=None, content_type=None, + if_none_match=None, ): headers = headers or {} if content_type: headers["content-type"] = content_type + if if_none_match: + headers["if-none-match"] = if_none_match httpx_response = await self.ds.client.request( method, path, diff --git a/datasette/views/database.py b/datasette/views/database.py index 9a8aca32..bc08ba05 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -183,6 +183,13 @@ class DatabaseDownload(DataView): headers = {} if self.ds.cors: add_cors_headers(headers) + if db.hash: + etag = '"{}"'.format(db.hash) + headers["Etag"] = etag + # Has user seen this already? + if_none_match = request.headers.get("if-none-match") + if if_none_match and if_none_match == etag: + return Response("", status=304) headers["Transfer-Encoding"] = "chunked" return AsgiFileDownload( filepath, diff --git a/tests/test_html.py b/tests/test_html.py index 42f1a3ee..409fec68 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -401,7 +401,7 @@ def test_database_download_for_immutable(): assert len(soup.findAll("a", {"href": re.compile(r"\.db$")})) # Check we can actually download it download_response = client.get("/fixtures.db") - assert 200 == download_response.status + assert download_response.status == 200 # Check the content-length header exists assert "content-length" in download_response.headers content_length = download_response.headers["content-length"] @@ -413,6 +413,14 @@ def test_database_download_for_immutable(): == 'attachment; filename="fixtures.db"' ) assert download_response.headers["transfer-encoding"] == "chunked" + # ETag header should be present and match db.hash + assert "etag" in download_response.headers + etag = download_response.headers["etag"] + assert etag == '"{}"'.format(client.ds.databases["fixtures"].hash) + # Try a second download with If-None-Match: current-etag + download_response2 = client.get("/fixtures.db", if_none_match=etag) + assert download_response2.body == b"" + assert download_response2.status == 304 def test_database_download_disallowed_for_mutable(app_client): From a5acfff4bd364d30ce8878e19f9839890371ef14 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 16 May 2022 17:06:40 -0700 Subject: [PATCH 0017/1172] Empty Datasette([]) list is no longer required --- docs/testing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 1bbaaac1..41046bfb 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -15,7 +15,7 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl @pytest.mark.asyncio async def test_plugin_is_installed(): - datasette = Datasette([], memory=True) + datasette = Datasette(memory=True) response = await datasette.client.get("/-/plugins.json") assert response.status_code == 200 installed_plugins = {p["name"] for p in response.json()} From 3508bf7875f8d62b2725222f3b07747974d54b97 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 17 May 2022 12:40:05 -0700 Subject: [PATCH 0018/1172] --nolock mode to ignore locked files, closes #1744 --- datasette/app.py | 2 ++ datasette/cli.py | 7 +++++++ datasette/database.py | 2 ++ docs/cli-reference.rst | 1 + docs/getting_started.rst | 4 +++- 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index b7b84371..f43700d4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -213,6 +213,7 @@ class Datasette: config_dir=None, pdb=False, crossdb=False, + nolock=False, ): assert config_dir is None or isinstance( config_dir, Path @@ -238,6 +239,7 @@ class Datasette: self.databases = collections.OrderedDict() self._refresh_schemas_lock = asyncio.Lock() self.crossdb = crossdb + self.nolock = nolock if memory or crossdb or not self.files: self.add_database(Database(self, is_memory=True), name="_memory") # memory_name is a random string so that each Datasette instance gets its own diff --git a/datasette/cli.py b/datasette/cli.py index 3c6e1b2c..8781747c 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -452,6 +452,11 @@ def uninstall(packages, yes): is_flag=True, help="Enable cross-database joins using the /_memory database", ) +@click.option( + "--nolock", + is_flag=True, + help="Ignore locking, open locked files in read-only mode", +) @click.option( "--ssl-keyfile", help="SSL key file", @@ -486,6 +491,7 @@ def serve( open_browser, create, crossdb, + nolock, ssl_keyfile, ssl_certfile, return_instance=False, @@ -545,6 +551,7 @@ def serve( version_note=version_note, pdb=pdb, crossdb=crossdb, + nolock=nolock, ) # if files is a single directory, use that as config_dir= diff --git a/datasette/database.py b/datasette/database.py index 44d32667..fa558045 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -89,6 +89,8 @@ class Database: # mode=ro or immutable=1? if self.is_mutable: qs = "?mode=ro" + if self.ds.nolock: + qs += "&nolock=1" else: qs = "?immutable=1" assert not (write and not self.is_mutable) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 2a6fbfc8..1c1aff15 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -115,6 +115,7 @@ datasette serve --help --create Create database files if they do not exist --crossdb Enable cross-database joins using the /_memory database + --nolock Ignore locking, open locked files in read-only mode --ssl-keyfile TEXT SSL key file --ssl-certfile TEXT SSL certificate file --help Show this message and exit. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 3e357afb..502a9e5a 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -56,7 +56,9 @@ like so: :: - datasette ~/Library/Application\ Support/Google/Chrome/Default/History + datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock + +The `--nolock` option ignores any file locks. This is safe as Datasette will open the file in read-only mode. Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data: From 5555bc8aef043f75d2200f66de90c54aeeaa08c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 17 May 2022 12:43:44 -0700 Subject: [PATCH 0019/1172] How to run cog, closes #1745 --- docs/contributing.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index c193ba49..bddceafe 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -211,6 +211,17 @@ For added productivity, you can use use `sphinx-autobuild `__. + +To update these pages, run the following command:: + + cog -r docs/*.rst + .. _contributing_continuous_deployment: Continuously deployed demo instances From b393e164dc9e962702546d6f1ad9c857b5788dc0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 17 May 2022 12:45:28 -0700 Subject: [PATCH 0020/1172] ReST fix --- docs/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 502a9e5a..af3a1385 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -58,7 +58,7 @@ like so: datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock -The `--nolock` option ignores any file locks. This is safe as Datasette will open the file in read-only mode. +The ``--nolock`` option ignores any file locks. This is safe as Datasette will open the file in read-only mode. Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data: From 7d1e004ff679b3fb4dca36d1d751a1ad16688fe6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 17 May 2022 12:59:28 -0700 Subject: [PATCH 0021/1172] Fix test I broke in #1744 --- tests/test_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index dca65f26..d0f6e26c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -150,6 +150,7 @@ def test_metadata_yaml(): help_settings=False, pdb=False, crossdb=False, + nolock=False, open_browser=False, create=False, ssl_keyfile=None, From 0e2f6f1f82f4445a63f1251470a7778a34f5c8b9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 May 2022 17:37:46 -0700 Subject: [PATCH 0022/1172] datasette-copyable is an example of register_output_renderer --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 3c9ae2e2..c0d88964 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -557,7 +557,7 @@ And here is an example ``can_render`` function which returns ``True`` only if th "atom_updated", }.issubset(columns) -Examples: `datasette-atom `_, `datasette-ics `_, `datasette-geojson `__ +Examples: `datasette-atom `_, `datasette-ics `_, `datasette-geojson `__, `datasette-copyable `__ .. _plugin_register_routes: From 18a6e05887abf1ac946a6e0d36ce662dfd8aeff1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 20 May 2022 12:05:33 -0700 Subject: [PATCH 0023/1172] Added "follow a tutorial" to getting started docs Closes #1747 --- docs/getting_started.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index af3a1385..00b753a9 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -1,6 +1,8 @@ Getting started =============== +.. _getting_started_demo: + Play with a live demo --------------------- @@ -9,6 +11,16 @@ The best way to experience Datasette for the first time is with a demo: * `global-power-plants.datasettes.com `__ provides a searchable database of power plants around the world, using data from the `World Resources Institude `__ rendered using the `datasette-cluster-map `__ plugin. * `fivethirtyeight.datasettes.com `__ shows Datasette running against over 400 datasets imported from the `FiveThirtyEight GitHub repository `__. +.. _getting_started_tutorial: + +Follow a tutorial +----------------- + +Datasette has several `tutorials `__ to help you get started with the tool. Try one of the following: + +- `Exploring a database with Datasette `__ shows how to use the Datasette web interface to explore a new database. +- `Learn SQL with Datasette `__ introduces SQL, and shows how to use that query language to ask questions of your data. + .. _getting_started_glitch: Try Datasette without installing anything using Glitch From 1465fea4798599eccfe7e8f012bd8d9adfac3039 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 20 May 2022 12:11:08 -0700 Subject: [PATCH 0024/1172] sphinx-copybutton for docs, closes #1748 --- docs/conf.py | 2 +- setup.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d114bc52..351cb1b1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc"] +extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc", "sphinx_copybutton"] extlinks = { "issue": ("https://github.com/simonw/datasette/issues/%s", "#"), diff --git a/setup.py b/setup.py index ca449f02..aad05840 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,13 @@ setup( """, setup_requires=["pytest-runner"], extras_require={ - "docs": ["sphinx_rtd_theme", "sphinx-autobuild", "codespell", "blacken-docs"], + "docs": [ + "sphinx_rtd_theme", + "sphinx-autobuild", + "codespell", + "blacken-docs", + "sphinx-copybutton", + ], "test": [ "pytest>=5.2.2,<7.2.0", "pytest-xdist>=2.2.1,<2.6", From 1d33fd03b3c211e0f48a8f3bde83880af89e4e69 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 20 May 2022 13:34:51 -0700 Subject: [PATCH 0025/1172] Switch docs theme to Furo, refs #1746 --- docs/_static/css/custom.css | 7 ++-- .../layout.html => _static/js/custom.js} | 34 ------------------- docs/_templates/base.html | 6 ++++ docs/_templates/sidebar/brand.html | 16 +++++++++ docs/_templates/sidebar/navigation.html | 11 ++++++ docs/conf.py | 24 +++---------- docs/installation.rst | 1 + docs/plugin_hooks.rst | 1 + setup.py | 2 +- 9 files changed, 45 insertions(+), 57 deletions(-) rename docs/{_templates/layout.html => _static/js/custom.js} (55%) create mode 100644 docs/_templates/base.html create mode 100644 docs/_templates/sidebar/brand.html create mode 100644 docs/_templates/sidebar/navigation.html diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index 4dabb725..0a6f8799 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -1,7 +1,8 @@ a.external { overflow-wrap: anywhere; } - -div .wy-side-nav-search > div.version { - color: rgba(0,0,0,0.75); +body[data-theme="dark"] .sidebar-logo-container { + background-color: white; + padding: 5px; + opacity: 0.6; } diff --git a/docs/_templates/layout.html b/docs/_static/js/custom.js similarity index 55% rename from docs/_templates/layout.html rename to docs/_static/js/custom.js index 785cdc7c..efca33ed 100644 --- a/docs/_templates/layout.html +++ b/docs/_static/js/custom.js @@ -1,35 +1,3 @@ -{%- extends "!layout.html" %} - -{% block htmltitle %} -{{ super() }} - -{% endblock %} - -{% block sidebartitle %} - - - - - -{% if theme_display_version %} - {%- set nav_version = version %} - {% if READTHEDOCS and current_version %} - {%- set nav_version = current_version %} - {% endif %} - {% if nav_version %} -
- {{ nav_version }} -
- {% endif %} -{% endif %} - -{% include "searchbox.html" %} - -{% endblock %} - -{% block footer %} -{{ super() }} - -{% endblock %} diff --git a/docs/_templates/base.html b/docs/_templates/base.html new file mode 100644 index 00000000..969de5ab --- /dev/null +++ b/docs/_templates/base.html @@ -0,0 +1,6 @@ +{%- extends "!base.html" %} + +{% block site_meta %} +{{ super() }} + +{% endblock %} diff --git a/docs/_templates/sidebar/brand.html b/docs/_templates/sidebar/brand.html new file mode 100644 index 00000000..8be9e8ee --- /dev/null +++ b/docs/_templates/sidebar/brand.html @@ -0,0 +1,16 @@ +
+ {% block brand_content %} +
+ +
+ {%- set nav_version = version %} + {% if READTHEDOCS and current_version %} + {%- set nav_version = current_version %} + {% endif %} + {% if nav_version %} +
+ {{ nav_version }} +
+ {% endif %} + {% endblock brand_content %} +
diff --git a/docs/_templates/sidebar/navigation.html b/docs/_templates/sidebar/navigation.html new file mode 100644 index 00000000..c460a17e --- /dev/null +++ b/docs/_templates/sidebar/navigation.html @@ -0,0 +1,11 @@ +
+ + {{ toctree( + collapse=True, + titles_only=False, + maxdepth=3, + includehidden=True, +) }} +
\ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 351cb1b1..25d2acfe 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -90,18 +90,15 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme" +html_theme = "furo" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - "logo_only": True, - "style_nav_header_background": "white", - "prev_next_buttons_location": "both", + "sidebar_hide_name": True, } - # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". @@ -112,20 +109,9 @@ html_logo = "datasette-logo.svg" html_css_files = [ "css/custom.css", ] - - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - "**": [ - "relations.html", # needs 'show_related': True theme option to display - "searchbox.html", - ] -} - +html_js_files = [ + "js/custom.js" +] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/docs/installation.rst b/docs/installation.rst index e8bef9cd..a4757736 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -13,6 +13,7 @@ If you want to start making contributions to the Datasette project by installing .. contents:: :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here .. _installation_basic: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index c0d88964..7d10fe37 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -20,6 +20,7 @@ For example, you can implement the ``render_cell`` plugin hook like this even th .. contents:: List of plugin hooks :local: + :class: this-will-duplicate-information-and-it-is-still-useful-here .. _plugin_hook_prepare_connection: diff --git a/setup.py b/setup.py index aad05840..d3fcdbd1 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "sphinx_rtd_theme", + "furo==2022.4.7", "sphinx-autobuild", "codespell", "blacken-docs", From 4446075334ea7231beb56b630bc7ec363afc2d08 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 20 May 2022 13:44:23 -0700 Subject: [PATCH 0026/1172] Append warning to the write element, refs #1746 --- docs/_static/js/custom.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/_static/js/custom.js b/docs/_static/js/custom.js index efca33ed..91c3e306 100644 --- a/docs/_static/js/custom.js +++ b/docs/_static/js/custom.js @@ -17,11 +17,7 @@ jQuery(function ($) { ` ); warning.find("a").attr("href", stableUrl); - var body = $("div.body"); - if (!body.length) { - body = $("div.document"); - } - body.prepend(warning); + $("article[role=main]").prepend(warning); } }); }); From b010af7bb85856aeb44f69e7e980f617c1fc0db1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 20 May 2022 15:23:09 -0700 Subject: [PATCH 0027/1172] Updated copyright years in documentation footer --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 25d2acfe..7ffeedd0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,7 @@ master_doc = "index" # General information about the project. project = "Datasette" -copyright = "2017-2021, Simon Willison" +copyright = "2017-2022, Simon Willison" author = "Simon Willison" # Disable -- turning into – From adedd85b68ec66e03b97fb62ff4da8987734436e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 28 May 2022 18:42:31 -0700 Subject: [PATCH 0028/1172] Clarify that request.headers names are converted to lowercase --- docs/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index 18822d47..da135282 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -26,7 +26,7 @@ The request object is passed to various plugin hooks. It represents an incoming The request scheme - usually ``https`` or ``http``. ``.headers`` - dictionary (str -> str) - A dictionary of incoming HTTP request headers. + A dictionary of incoming HTTP request headers. Header names have been converted to lowercase. ``.cookies`` - dictionary (str -> str) A dictionary of incoming cookies From 8dd816bc76937f1e37f86acce10dc2cb4fa31e52 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 May 2022 15:42:38 -0700 Subject: [PATCH 0029/1172] Applied Black --- docs/conf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7ffeedd0..4ef6b768 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -109,9 +109,7 @@ html_logo = "datasette-logo.svg" html_css_files = [ "css/custom.css", ] -html_js_files = [ - "js/custom.js" -] +html_js_files = ["js/custom.js"] # -- Options for HTMLHelp output ------------------------------------------ From 2e9751672d4fe329b3c359d5b7b1992283185820 Mon Sep 17 00:00:00 2001 From: Naveen <172697+naveensrinivasan@users.noreply.github.com> Date: Tue, 31 May 2022 14:28:40 -0500 Subject: [PATCH 0030/1172] chore: Set permissions for GitHub actions (#1740) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restrict the GitHub token permissions only to the required ones; this way, even if the attackers will succeed in compromising your workflow, they won’t be able to do much. - Included permissions for the action. https://github.com/ossf/scorecard/blob/main/docs/checks.md#token-permissions https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs [Keeping your GitHub Actions and workflows secure Part 1: Preventing pwn requests](https://securitylab.github.com/research/github-actions-preventing-pwn-requests/) Signed-off-by: naveen <172697+naveensrinivasan@users.noreply.github.com> --- .github/workflows/deploy-latest.yml | 3 +++ .github/workflows/prettier.yml | 3 +++ .github/workflows/publish.yml | 3 +++ .github/workflows/push_docker_tag.yml | 3 +++ .github/workflows/spellcheck.yml | 3 +++ .github/workflows/test-coverage.yml | 3 +++ .github/workflows/test-pyodide.yml | 3 +++ .github/workflows/test.yml | 3 +++ .github/workflows/tmate-mac.yml | 3 +++ .github/workflows/tmate.yml | 3 +++ 10 files changed, 30 insertions(+) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index a61f6629..2b94a7f1 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -5,6 +5,9 @@ on: branches: - main +permissions: + contents: read + jobs: deploy: runs-on: ubuntu-latest diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 9dfe7ee0..ded41040 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -2,6 +2,9 @@ name: Check JavaScript for conformance with Prettier on: [push] +permissions: + contents: read + jobs: prettier: runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3e4f8146..9ef09d2e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,6 +4,9 @@ on: release: types: [created] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index 9a3969f0..afe8d6b2 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -6,6 +6,9 @@ on: version_tag: description: Tag to build and push +permissions: + contents: read + jobs: deploy_docker: runs-on: ubuntu-latest diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 2e24d3eb..a2621ecc 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -2,6 +2,9 @@ name: Check spelling in documentation on: [push, pull_request] +permissions: + contents: read + jobs: spellcheck: runs-on: ubuntu-latest diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 1d1cf332..bd720664 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -7,6 +7,9 @@ on: pull_request: branches: - main +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index 1b75aade..bc9593a8 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -5,6 +5,9 @@ on: pull_request: workflow_dispatch: +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d916e49..90b6555e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,9 @@ name: Test on: [push, pull_request] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index 46be117e..fcee0f21 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -3,6 +3,9 @@ name: tmate session mac on: workflow_dispatch: +permissions: + contents: read + jobs: build: runs-on: macos-latest diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 02e7bd33..9792245d 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -3,6 +3,9 @@ name: tmate session on: workflow_dispatch: +permissions: + contents: read + jobs: build: runs-on: ubuntu-latest From e780b2f5d662ef3579d801d33567440055d4e84d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 20 Jun 2022 10:54:23 -0700 Subject: [PATCH 0031/1172] Trying out one-sentence-per-line As suggested here: https://sive.rs/1s Markdown and reStructuredText will display this as if it is a single paragraph, even though the sentences themselves are separated by newlines. This could result in more useful diffs. Trying it out on this page first. --- docs/facets.rst | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/docs/facets.rst b/docs/facets.rst index 0228aa84..2a2eb039 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -3,7 +3,9 @@ Facets ====== -Datasette facets can be used to add a faceted browse interface to any database table. With facets, tables are displayed along with a summary showing the most common values in specified columns. These values can be selected to further filter the table. +Datasette facets can be used to add a faceted browse interface to any database table. +With facets, tables are displayed along with a summary showing the most common values in specified columns. +These values can be selected to further filter the table. .. image:: facets.png @@ -12,11 +14,13 @@ Facets can be specified in two ways: using query string parameters, or in ``meta Facets in query strings ----------------------- -To turn on faceting for specific columns on a Datasette table view, add one or more ``_facet=COLUMN`` parameters to the URL. For example, if you want to turn on facets for the ``city_id`` and ``state`` columns, construct a URL that looks like this:: +To turn on faceting for specific columns on a Datasette table view, add one or more ``_facet=COLUMN`` parameters to the URL. +For example, if you want to turn on facets for the ``city_id`` and ``state`` columns, construct a URL that looks like this:: /dbname/tablename?_facet=state&_facet=city_id -This works for both the HTML interface and the ``.json`` view. When enabled, facets will cause a ``facet_results`` block to be added to the JSON output, looking something like this: +This works for both the HTML interface and the ``.json`` view. +When enabled, facets will cause a ``facet_results`` block to be added to the JSON output, looking something like this: .. code-block:: json @@ -86,7 +90,8 @@ This works for both the HTML interface and the ``.json`` view. When enabled, fac If Datasette detects that a column is a foreign key, the ``"label"`` property will be automatically derived from the detected label column on the referenced table. -The default number of facet results returned is 30, controlled by the :ref:`setting_default_facet_size` setting. You can increase this on an individual page by adding ``?_facet_size=100`` to the query string, up to a maximum of :ref:`setting_max_returned_rows` (which defaults to 1000). +The default number of facet results returned is 30, controlled by the :ref:`setting_default_facet_size` setting. +You can increase this on an individual page by adding ``?_facet_size=100`` to the query string, up to a maximum of :ref:`setting_max_returned_rows` (which defaults to 1000). .. _facets_metadata: @@ -137,12 +142,14 @@ For the currently filtered data are there any columns which, if applied as a fac * Will return less unique options than the total number of filtered rows * And the query used to evaluate this criteria can be completed in under 50ms -That last point is particularly important: Datasette runs a query for every column that is displayed on a page, which could get expensive - so to avoid slow load times it sets a time limit of just 50ms for each of those queries. This means suggested facets are unlikely to appear for tables with millions of records in them. +That last point is particularly important: Datasette runs a query for every column that is displayed on a page, which could get expensive - so to avoid slow load times it sets a time limit of just 50ms for each of those queries. +This means suggested facets are unlikely to appear for tables with millions of records in them. Speeding up facets with indexes ------------------------------- -The performance of facets can be greatly improved by adding indexes on the columns you wish to facet by. Adding indexes can be performed using the ``sqlite3`` command-line utility. Here's how to add an index on the ``state`` column in a table called ``Food_Trucks``:: +The performance of facets can be greatly improved by adding indexes on the columns you wish to facet by. +Adding indexes can be performed using the ``sqlite3`` command-line utility. Here's how to add an index on the ``state`` column in a table called ``Food_Trucks``:: $ sqlite3 mydatabase.db SQLite version 3.19.3 2017-06-27 16:48:08 @@ -169,6 +176,7 @@ Example here: `latest.datasette.io/fixtures/facetable?_facet_array=tags `__ From 00e59ec461dc0150772b999c7cc15fcb9b507d58 Mon Sep 17 00:00:00 2001 From: "M. Nasimul Haque" Date: Mon, 20 Jun 2022 19:05:44 +0100 Subject: [PATCH 0032/1172] Extract facet pieces of table.html into included templates Thanks, @nsmgr8 --- datasette/templates/_facet_results.html | 28 ++++++++++++++++++ datasette/templates/_suggested_facets.html | 3 ++ datasette/templates/table.html | 33 ++-------------------- 3 files changed, 33 insertions(+), 31 deletions(-) create mode 100644 datasette/templates/_facet_results.html create mode 100644 datasette/templates/_suggested_facets.html diff --git a/datasette/templates/_facet_results.html b/datasette/templates/_facet_results.html new file mode 100644 index 00000000..d0cbcf77 --- /dev/null +++ b/datasette/templates/_facet_results.html @@ -0,0 +1,28 @@ +
+ {% for facet_info in sorted_facet_results %} +
+

+ {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %} + {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }} + + {% if facet_info.hideable %} + ✖ + {% endif %} +

+
    + {% for facet_value in facet_info.results %} + {% if not facet_value.selected %} +
  • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
    • + {% else %} +
  • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
    • + {% endif %} + {% endfor %} + {% if facet_info.truncated %} +
  • {% if request.args._facet_size != "max" -%} + …{% else -%}…{% endif %} +
    • + {% endif %} +
+
+ {% endfor %} +
diff --git a/datasette/templates/_suggested_facets.html b/datasette/templates/_suggested_facets.html new file mode 100644 index 00000000..ec98fb36 --- /dev/null +++ b/datasette/templates/_suggested_facets.html @@ -0,0 +1,3 @@ +

+ Suggested facets: {% for facet in suggested_facets %}{{ facet.name }}{% if facet.type %} ({{ facet.type }}){% endif %}{% if not loop.last %}, {% endif %}{% endfor %} +

diff --git a/datasette/templates/table.html b/datasette/templates/table.html index a9e88330..a86398ea 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -142,9 +142,7 @@

This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}{% if display_rows %}, CSV (advanced){% endif %}

{% if suggested_facets %} -

- Suggested facets: {% for facet in suggested_facets %}{{ facet.name }}{% if facet.type %} ({{ facet.type }}){% endif %}{% if not loop.last %}, {% endif %}{% endfor %} -

+ {% include "_suggested_facets.html" %} {% endif %} {% if facets_timed_out %} @@ -152,34 +150,7 @@ {% endif %} {% if facet_results %} -
- {% for facet_info in sorted_facet_results %} -
-

- {{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %} - {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }} - - {% if facet_info.hideable %} - ✖ - {% endif %} -

-
    - {% for facet_value in facet_info.results %} - {% if not facet_value.selected %} -
  • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
    • - {% else %} -
  • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
    • - {% endif %} - {% endfor %} - {% if facet_info.truncated %} -
  • {% if request.args._facet_size != "max" -%} - …{% else -%}…{% endif %} -
    • - {% endif %} -
-
- {% endfor %} -
+ {% include "_facet_results.html" %} {% endif %} {% include custom_table_templates %} From 9f1eb0d4eac483b953392157bd9fd6cc4df37de7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jun 2022 10:40:24 -0700 Subject: [PATCH 0033/1172] Bump black from 22.1.0 to 22.6.0 (#1763) Bumps [black](https://github.com/psf/black) from 22.1.0 to 22.6.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.1.0...22.6.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d3fcdbd1..29cb77bf 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( "pytest-xdist>=2.2.1,<2.6", "pytest-asyncio>=0.17,<0.19", "beautifulsoup4>=4.8.1,<4.12.0", - "black==22.1.0", + "black==22.6.0", "blacken-docs==1.12.1", "pytest-timeout>=1.4.2,<2.2", "trustme>=0.7,<0.10", From 6373bb341457e5becfd5b67792ac2c8b9ed7c384 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Jul 2022 09:30:49 -0700 Subject: [PATCH 0034/1172] Expose current SQLite row to render_cell hook, closes #1300 --- datasette/hookspecs.py | 2 +- datasette/views/database.py | 1 + datasette/views/table.py | 1 + docs/plugin_hooks.rst | 9 ++++++--- tests/plugins/my_plugin.py | 3 ++- tests/test_plugins.py | 5 +++-- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 8f4fecab..c84db0a3 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -60,7 +60,7 @@ def publish_subcommand(publish): @hookspec -def render_cell(value, column, table, database, datasette): +def render_cell(row, value, column, table, database, datasette): """Customize rendering of HTML table cell values""" diff --git a/datasette/views/database.py b/datasette/views/database.py index bc08ba05..42058752 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -375,6 +375,7 @@ class QueryView(DataView): # pylint: disable=no-member plugin_display_value = None for candidate in pm.hook.render_cell( + row=row, value=value, column=column, table=None, diff --git a/datasette/views/table.py b/datasette/views/table.py index 23289b29..cd4be823 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -895,6 +895,7 @@ async def display_columns_and_rows( # pylint: disable=no-member plugin_display_value = None for candidate in pm.hook.render_cell( + row=row, value=value, column=column, table=table_name, diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 7d10fe37..f5c3ee83 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -373,12 +373,15 @@ Examples: `datasette-publish-fly Date: Sat, 9 Jul 2022 10:25:37 -0700 Subject: [PATCH 0035/1172] More than 90 plugins now --- docs/writing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 9aee70f6..01ee8c90 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -5,7 +5,7 @@ Writing plugins You can write one-off plugins that apply to just one Datasette instance, or you can write plugins which can be installed using ``pip`` and can be shipped to the Python Package Index (`PyPI `__) for other people to install. -Want to start by looking at an example? The `Datasette plugins directory `__ lists more than 50 open source plugins with code you can explore. The :ref:`plugin hooks ` page includes links to example plugins for each of the documented hooks. +Want to start by looking at an example? The `Datasette plugins directory `__ lists more than 90 open source plugins with code you can explore. The :ref:`plugin hooks ` page includes links to example plugins for each of the documented hooks. .. _writing_plugins_one_off: From 5d76c1f81b2d978f48b85c70d041a2142cf8ee26 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Jul 2022 15:03:33 -0700 Subject: [PATCH 0036/1172] Discord badge Refs https://github.com/simonw/datasette.io/issues/112 --- README.md | 1 + docs/index.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 557d9290..c57ee604 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE) [![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette) +[![discord](https://img.shields.io/discord/823971286308356157?label=Discord)](https://discord.gg/ktd74dm5mw) *An open source multi-tool for exploring and publishing data* diff --git a/docs/index.rst b/docs/index.rst index a2888822..62ed70f8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,8 @@ datasette| :target: https://github.com/simonw/datasette/blob/main/LICENSE .. |docker: datasette| image:: https://img.shields.io/badge/docker-datasette-blue :target: https://hub.docker.com/r/datasetteproject/datasette +.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=Discord + :target: https://discord.gg/ktd74dm5mw *An open source multi-tool for exploring and publishing data* From c133545fe9c7ac2d509e55bf4bf6164bfbe892ad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Jul 2022 15:04:38 -0700 Subject: [PATCH 0037/1172] Make discord badge lowercase Refs https://github.com/simonw/datasette.io/issues/112 --- README.md | 2 +- docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c57ee604..032180aa 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE) [![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette) -[![discord](https://img.shields.io/discord/823971286308356157?label=Discord)](https://discord.gg/ktd74dm5mw) +[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://discord.gg/ktd74dm5mw) *An open source multi-tool for exploring and publishing data* diff --git a/docs/index.rst b/docs/index.rst index 62ed70f8..051898b1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,7 +16,7 @@ datasette| :target: https://github.com/simonw/datasette/blob/main/LICENSE .. |docker: datasette| image:: https://img.shields.io/badge/docker-datasette-blue :target: https://hub.docker.com/r/datasetteproject/datasette -.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=Discord +.. |discord| image:: https://img.shields.io/discord/823971286308356157?label=discord :target: https://discord.gg/ktd74dm5mw *An open source multi-tool for exploring and publishing data* From 950cc7677f65aa2543067b3bbfc2b6acb98b62c8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 14 Jul 2022 15:18:28 -0700 Subject: [PATCH 0038/1172] Fix missing Discord image Refs https://github.com/simonw/datasette.io/issues/112 --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 051898b1..efe196b3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ Datasette ========= |PyPI| |Changelog| |Python 3.x| |Tests| |License| |docker: -datasette| +datasette| |discord| .. |PyPI| image:: https://img.shields.io/pypi/v/datasette.svg :target: https://pypi.org/project/datasette/ From 8188f55efc0fcca1be692b0d0c875f2d1ee99f17 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Jul 2022 15:24:16 -0700 Subject: [PATCH 0039/1172] Rename handle_500 to handle_exception, refs #1770 --- datasette/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f43700d4..43e60dbc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1275,7 +1275,7 @@ class DatasetteRouter: except NotFound as exception: return await self.handle_404(request, send, exception) except Exception as exception: - return await self.handle_500(request, send, exception) + return await self.handle_exception(request, send, exception) async def handle_404(self, request, send, exception=None): # If path contains % encoding, redirect to tilde encoding @@ -1354,7 +1354,7 @@ class DatasetteRouter: view_name="page", ) except NotFoundExplicit as e: - await self.handle_500(request, send, e) + await self.handle_exception(request, send, e) return # Pull content-type out into separate parameter content_type = "text/html; charset=utf-8" @@ -1369,9 +1369,9 @@ class DatasetteRouter: content_type=content_type, ) else: - await self.handle_500(request, send, exception or NotFound("404")) + await self.handle_exception(request, send, exception or NotFound("404")) - async def handle_500(self, request, send, exception): + async def handle_exception(self, request, send, exception): if self.ds.pdb: import pdb From c09c53f3455a7b9574cf7695478f2b87d20897db Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Jul 2022 16:24:39 -0700 Subject: [PATCH 0040/1172] New handle_exception plugin hook, refs #1770 Also refs: - https://github.com/simonw/datasette-sentry/issues/1 - https://github.com/simonw/datasette-show-errors/issues/2 --- datasette/app.py | 97 +++++++++-------------------------- datasette/forbidden.py | 20 ++++++++ datasette/handle_exception.py | 74 ++++++++++++++++++++++++++ datasette/hookspecs.py | 5 ++ datasette/plugins.py | 2 + docs/plugin_hooks.rst | 78 ++++++++++++++++++++-------- tests/fixtures.py | 1 + tests/plugins/my_plugin_2.py | 18 +++++++ tests/test_permissions.py | 1 + tests/test_plugins.py | 14 +++++ 10 files changed, 215 insertions(+), 95 deletions(-) create mode 100644 datasette/forbidden.py create mode 100644 datasette/handle_exception.py diff --git a/datasette/app.py b/datasette/app.py index 43e60dbc..edd05bb3 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -16,7 +16,6 @@ import re import secrets import sys import threading -import traceback import urllib.parse from concurrent import futures from pathlib import Path @@ -27,7 +26,7 @@ from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound -from .views.base import DatasetteError, ureg +from .views.base import ureg from .views.database import DatabaseDownload, DatabaseView from .views.index import IndexView from .views.special import ( @@ -49,7 +48,6 @@ from .utils import ( PrefixedUrlString, SPATIALITE_FUNCTIONS, StartupError, - add_cors_headers, async_call_with_supported_arguments, await_me_maybe, call_with_supported_arguments, @@ -87,11 +85,6 @@ from .tracer import AsgiTracer from .plugins import pm, DEFAULT_PLUGINS, get_plugins from .version import __version__ -try: - import rich -except ImportError: - rich = None - app_root = Path(__file__).parent.parent # https://github.com/simonw/datasette/issues/283#issuecomment-781591015 @@ -1274,6 +1267,16 @@ class DatasetteRouter: return except NotFound as exception: return await self.handle_404(request, send, exception) + except Forbidden as exception: + # Try the forbidden() plugin hook + for custom_response in pm.hook.forbidden( + datasette=self.ds, request=request, message=exception.args[0] + ): + custom_response = await await_me_maybe(custom_response) + assert ( + custom_response + ), "Default forbidden() hook should have been called" + return await custom_response.asgi_send(send) except Exception as exception: return await self.handle_exception(request, send, exception) @@ -1372,72 +1375,20 @@ class DatasetteRouter: await self.handle_exception(request, send, exception or NotFound("404")) async def handle_exception(self, request, send, exception): - if self.ds.pdb: - import pdb + responses = [] + for hook in pm.hook.handle_exception( + datasette=self.ds, + request=request, + exception=exception, + ): + response = await await_me_maybe(hook) + if response is not None: + responses.append(response) - pdb.post_mortem(exception.__traceback__) - - if rich is not None: - rich.get_console().print_exception(show_locals=True) - - title = None - if isinstance(exception, Forbidden): - status = 403 - info = {} - message = exception.args[0] - # Try the forbidden() plugin hook - for custom_response in pm.hook.forbidden( - datasette=self.ds, request=request, message=message - ): - custom_response = await await_me_maybe(custom_response) - if custom_response is not None: - await custom_response.asgi_send(send) - return - elif isinstance(exception, Base400): - status = exception.status - info = {} - message = exception.args[0] - elif isinstance(exception, DatasetteError): - status = exception.status - info = exception.error_dict - message = exception.message - if exception.message_is_html: - message = Markup(message) - title = exception.title - else: - status = 500 - info = {} - message = str(exception) - traceback.print_exc() - templates = [f"{status}.html", "error.html"] - info.update( - { - "ok": False, - "error": message, - "status": status, - "title": title, - } - ) - headers = {} - if self.ds.cors: - add_cors_headers(headers) - if request.path.split("?")[0].endswith(".json"): - await asgi_send_json(send, info, status=status, headers=headers) - else: - template = self.ds.jinja_env.select_template(templates) - await asgi_send_html( - send, - await template.render_async( - dict( - info, - urls=self.ds.urls, - app_css_hash=self.ds.app_css_hash(), - menu_links=lambda: [], - ) - ), - status=status, - headers=headers, - ) + assert responses, "Default exception handler should have returned something" + # Even if there are multiple responses use just the first one + response = responses[0] + await response.asgi_send(send) _cleaner_task_str_re = re.compile(r"\S*site-packages/") diff --git a/datasette/forbidden.py b/datasette/forbidden.py new file mode 100644 index 00000000..156a44d4 --- /dev/null +++ b/datasette/forbidden.py @@ -0,0 +1,20 @@ +from os import stat +from datasette import hookimpl, Response + + +@hookimpl(trylast=True) +def forbidden(datasette, request, message): + async def inner(): + return Response.html( + await datasette.render_template( + "error.html", + { + "title": "Forbidden", + "error": message, + }, + request=request, + ), + status=403, + ) + + return inner diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py new file mode 100644 index 00000000..8b7e83e3 --- /dev/null +++ b/datasette/handle_exception.py @@ -0,0 +1,74 @@ +from datasette import hookimpl, Response +from .utils import await_me_maybe, add_cors_headers +from .utils.asgi import ( + Base400, + Forbidden, +) +from .views.base import DatasetteError +from markupsafe import Markup +import pdb +import traceback +from .plugins import pm + +try: + import rich +except ImportError: + rich = None + + +@hookimpl(trylast=True) +def handle_exception(datasette, request, exception): + async def inner(): + if datasette.pdb: + pdb.post_mortem(exception.__traceback__) + + if rich is not None: + rich.get_console().print_exception(show_locals=True) + + title = None + if isinstance(exception, Base400): + status = exception.status + info = {} + message = exception.args[0] + elif isinstance(exception, DatasetteError): + status = exception.status + info = exception.error_dict + message = exception.message + if exception.message_is_html: + message = Markup(message) + title = exception.title + else: + status = 500 + info = {} + message = str(exception) + traceback.print_exc() + templates = [f"{status}.html", "error.html"] + info.update( + { + "ok": False, + "error": message, + "status": status, + "title": title, + } + ) + headers = {} + if datasette.cors: + add_cors_headers(headers) + if request.path.split("?")[0].endswith(".json"): + return Response.json(info, status=status, headers=headers) + else: + template = datasette.jinja_env.select_template(templates) + return Response.html( + await template.render_async( + dict( + info, + urls=datasette.urls, + app_css_hash=datasette.app_css_hash(), + menu_links=lambda: [], + ) + ), + status=status, + headers=headers, + ) + + return inner diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index c84db0a3..a5fb536f 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -138,3 +138,8 @@ def database_actions(datasette, actor, database, request): @hookspec def skip_csrf(datasette, scope): """Mechanism for skipping CSRF checks for certain requests""" + + +@hookspec +def handle_exception(datasette, request, exception): + """Handle an uncaught exception. Can return a Response or None.""" diff --git a/datasette/plugins.py b/datasette/plugins.py index 76b46a47..fef0c8e9 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -15,6 +15,8 @@ DEFAULT_PLUGINS = ( "datasette.default_magic_parameters", "datasette.blob_renderer", "datasette.default_menu_links", + "datasette.handle_exception", + "datasette.forbidden", ) pm = pluggy.PluginManager("datasette") diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f5c3ee83..6020a941 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -107,8 +107,8 @@ Extra template variables that should be made available in the rendered template ``view_name`` - string The name of the view being displayed. (``index``, ``database``, ``table``, and ``row`` are the most important ones.) -``request`` - object or None - The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available. +``request`` - :ref:`internals_request` or None + The current HTTP request. This can be ``None`` if the request object is not available. ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)`` @@ -504,7 +504,7 @@ When a request is received, the ``"render"`` callback function is called with ze The table or view, if one is being rendered. ``request`` - :ref:`internals_request` - The incoming HTTP request. + The current HTTP request. ``view_name`` - string The name of the current view being called. ``index``, ``database``, ``table``, and ``row`` are the most important ones. @@ -599,8 +599,8 @@ The optional view function arguments are as follows: ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. -``request`` - Request object - The current HTTP :ref:`internals_request`. +``request`` - :ref:`internals_request` + The current HTTP request. ``scope`` - dictionary The incoming ASGI scope dictionary. @@ -947,8 +947,8 @@ actor_from_request(datasette, request) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. -``request`` - object - The current HTTP :ref:`internals_request`. +``request`` - :ref:`internals_request` + The current HTTP request. This is part of Datasette's :ref:`authentication and permissions system `. The function should attempt to authenticate an actor (either a user or an API actor of some sort) based on information in the request. @@ -1010,8 +1010,8 @@ Example: `datasette-auth-tokens `__ and then renders a custom error page: + +.. code-block:: python + + from datasette import hookimpl, Response + import sentry_sdk + + + @hookimpl + def handle_exception(datasette, exception): + sentry_sdk.capture_exception(exception) + async def inner(): + return Response.html( + await datasette.render_template("custom_error.html", request=request) + ) + return inner + .. _plugin_hook_menu_links: menu_links(datasette, actor, request) @@ -1232,8 +1266,8 @@ menu_links(datasette, actor, request) ``actor`` - dictionary or None The currently authenticated :ref:`actor `. -``request`` - object or None - The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available. +``request`` - :ref:`internals_request` + The current HTTP request. This can be ``None`` if the request object is not available. This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon. @@ -1281,8 +1315,8 @@ table_actions(datasette, actor, database, table, request) ``table`` - string The name of the table. -``request`` - object - The current HTTP :ref:`internals_request`. This can be ``None`` if the request object is not available. +``request`` - :ref:`internals_request` + The current HTTP request. This can be ``None`` if the request object is not available. This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items. @@ -1325,8 +1359,8 @@ database_actions(datasette, actor, database, request) ``database`` - string The name of the database. -``request`` - object - The current HTTP :ref:`internals_request`. +``request`` - :ref:`internals_request` + The current HTTP request. This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. diff --git a/tests/fixtures.py b/tests/fixtures.py index e0e4ec7b..c145ac78 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -68,6 +68,7 @@ EXPECTED_PLUGINS = [ "canned_queries", "extra_js_urls", "extra_template_vars", + "handle_exception", "menu_links", "permission_allowed", "register_routes", diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index f5ce36b3..4df02343 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -185,3 +185,21 @@ def register_routes(datasette): # Also serves to demonstrate over-ride of default paths: (r"/(?P[^/]+)/(?P[^/]+?$)", new_table), ] + + +@hookimpl +def handle_exception(datasette, request, exception): + datasette._exception_hook_fired = (request, exception) + if request.args.get("_custom_error"): + return Response.text("_custom_error") + elif request.args.get("_custom_error_async"): + + async def inner(): + return Response.text("_custom_error_async") + + return inner + + +@hookimpl(specname="register_routes") +def register_triger_error(): + return ((r"/trigger-error", lambda: 1 / 0),) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index f4169dbe..2a519e76 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -332,6 +332,7 @@ def test_permissions_debug(app_client): assert checks == [ {"action": "permissions-debug", "result": True, "used_default": False}, {"action": "view-instance", "result": None, "used_default": True}, + {"action": "debug-menu", "result": False, "used_default": True}, {"action": "permissions-debug", "result": False, "used_default": True}, {"action": "view-instance", "result": None, "used_default": True}, ] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 4a7ad7c6..948a40b8 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -824,6 +824,20 @@ def test_hook_forbidden(restore_working_directory): assert "view-database" == client.ds._last_forbidden_message +def test_hook_handle_exception(app_client): + app_client.get("/trigger-error?x=123") + assert hasattr(app_client.ds, "_exception_hook_fired") + request, exception = app_client.ds._exception_hook_fired + assert request.url == "http://localhost/trigger-error?x=123" + assert isinstance(exception, ZeroDivisionError) + + +@pytest.mark.parametrize("param", ("_custom_error", "_custom_error_async")) +def test_hook_handle_exception_custom_response(app_client, param): + response = app_client.get("/trigger-error?{}=1".format(param)) + assert response.text == param + + def test_hook_menu_links(app_client): def get_menu_links(html): soup = Soup(html, "html.parser") From 58fd1e33ec7ac5ed85431d5c86d60600cd5280fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Jul 2022 16:30:58 -0700 Subject: [PATCH 0041/1172] Hint that you can render templates for these hooks, refs #1770 --- docs/plugin_hooks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 6020a941..b4869606 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1176,7 +1176,7 @@ forbidden(datasette, request, message) -------------------------------------- ``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to render templates or execute SQL queries. ``request`` - :ref:`internals_request` The current HTTP request. @@ -1224,7 +1224,7 @@ handle_exception(datasette, request, exception) ----------------------------------------------- ``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to render templates or execute SQL queries. ``request`` - :ref:`internals_request` The current HTTP request. From e543a095cc4c1ca895b082cfd1263ca25203a7c0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Jul 2022 17:57:41 -0700 Subject: [PATCH 0042/1172] Updated default plugins in docs, refs #1770 --- docs/plugins.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/plugins.rst b/docs/plugins.rst index f2ed02f7..29078054 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -172,6 +172,24 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "filters_from_request" ] }, + { + "name": "datasette.forbidden", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "forbidden" + ] + }, + { + "name": "datasette.handle_exception", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "handle_exception" + ] + }, { "name": "datasette.publish.cloudrun", "static": false, From 6d5e1955470424cf4faf5d35788d328ebdd6d463 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Jul 2022 17:59:20 -0700 Subject: [PATCH 0043/1172] Release 0.62a1 Refs #1300, #1739, #1744, #1746, #1748, #1759, #1770 --- datasette/version.py | 2 +- docs/changelog.rst | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index cf18c441..86f4cf7e 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.62a0" +__version__ = "0.62a1" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 74814fcb..3f105811 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,20 @@ Changelog ========= +.. _v0_62a1: + +0.62a1 (2022-07-17) +------------------- + +- New plugin hook: :ref:`handle_exception() `, for custom handling of exceptions caught by Datasette. (:issue:`1770`) +- The :ref:`render_cell() ` plugin hook is now also passed a ``row`` argument, representing the ``sqlite3.Row`` object that is being rendered. (:issue:`1300`) +- New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`) +- Documentation now uses the `Furo `__ Sphinx theme. (:issue:`1746`) +- Datasette now has a `Discord community `__. +- Database file downloads now implement conditional GET using ETags. (:issue:`1739`) +- Examples in the documentation now include a copy-to-clipboard button. (:issue:`1748`) +- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (`#1759 `__) + .. _v0_62a0: 0.62a0 (2022-05-02) From ed1ebc0f1d4153e3e0934f2af19f82e5fdf137d3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Jul 2022 18:03:33 -0700 Subject: [PATCH 0044/1172] Run blacken-docs, refs #1770 --- docs/plugin_hooks.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index b4869606..aec1df56 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1213,7 +1213,9 @@ The function can alternatively return an awaitable function if it needs to make def forbidden(datasette): async def inner(): return Response.html( - await datasette.render_template("render_message.html", request=request) + await datasette.render_template( + "render_message.html", request=request + ) ) return inner @@ -1249,10 +1251,14 @@ This example logs an error to `Sentry `__ and then renders a @hookimpl def handle_exception(datasette, exception): sentry_sdk.capture_exception(exception) + async def inner(): return Response.html( - await datasette.render_template("custom_error.html", request=request) + await datasette.render_template( + "custom_error.html", request=request + ) ) + return inner .. _plugin_hook_menu_links: From ea6161f8475d9fa41c4879049511c58f692cce04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 17 Jul 2022 18:06:26 -0700 Subject: [PATCH 0045/1172] Bump furo from 2022.4.7 to 2022.6.21 (#1760) Bumps [furo](https://github.com/pradyunsg/furo) from 2022.4.7 to 2022.6.21. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2022.04.07...2022.06.21) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 29cb77bf..558b5c87 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "furo==2022.4.7", + "furo==2022.6.21", "sphinx-autobuild", "codespell", "blacken-docs", From 22354c48ce4d514d7a1b321e5651c7f1340e3f5e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 17 Jul 2022 18:06:37 -0700 Subject: [PATCH 0046/1172] Update pytest-asyncio requirement from <0.19,>=0.17 to >=0.17,<0.20 (#1769) Updates the requirements on [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases) - [Changelog](https://github.com/pytest-dev/pytest-asyncio/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.17.0...v0.19.0) --- updated-dependencies: - dependency-name: pytest-asyncio dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 558b5c87..a1c51d0b 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ setup( "test": [ "pytest>=5.2.2,<7.2.0", "pytest-xdist>=2.2.1,<2.6", - "pytest-asyncio>=0.17,<0.19", + "pytest-asyncio>=0.17,<0.20", "beautifulsoup4>=4.8.1,<4.12.0", "black==22.6.0", "blacken-docs==1.12.1", From 01369176b0a8943ab45292ffc6f9c929b80a00e8 Mon Sep 17 00:00:00 2001 From: Chris Amico Date: Sun, 17 Jul 2022 21:12:45 -0400 Subject: [PATCH 0047/1172] Keep track of datasette.config_dir (#1766) Thanks, @eyeseast - closes #1764 --- datasette/app.py | 1 + tests/test_config_dir.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index edd05bb3..1a9afc10 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -211,6 +211,7 @@ class Datasette: assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" + self.config_dir = config_dir self.pdb = pdb self._secret = secret or secrets.token_hex(32) self.files = tuple(files or []) + tuple(immutables or []) diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 015c6ace..fe927c42 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -1,4 +1,5 @@ import json +import pathlib import pytest from datasette.app import Datasette @@ -150,3 +151,11 @@ def test_metadata_yaml(tmp_path_factory, filename): response = client.get("/-/metadata.json") assert 200 == response.status assert {"title": "Title from metadata"} == response.json + + +def test_store_config_dir(config_dir_client): + ds = config_dir_client.ds + + assert hasattr(ds, "config_dir") + assert ds.config_dir is not None + assert isinstance(ds.config_dir, pathlib.Path) From 7af67b54b7d9bca43e948510fc62f6db2b748fa8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 18 Jul 2022 14:31:09 -0700 Subject: [PATCH 0048/1172] How to register temporary plugins in tests, closes #903 --- docs/testing_plugins.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 41046bfb..d02003a9 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -219,3 +219,39 @@ Here's a test for that plugin that mocks the HTTPX outbound request: assert ( outbound_request.url == "https://www.example.com/" ) + +.. _testing_plugins_register_in_test: + +Registering a plugin for the duration of a test +----------------------------------------------- + +When writing tests for plugins you may find it useful to register a test plugin just for the duration of a single test. You can do this using ``pm.register()`` and ``pm.unregister()`` like this: + +.. code-block:: python + + from datasette import hookimpl + from datasette.app import Datasette + from datasette.plugins import pm + import pytest + + + @pytest.mark.asyncio + async def test_using_test_plugin(): + class TestPlugin: + __name__ = "TestPlugin" + + # Use hookimpl and method names to register hooks + @hookimpl + def register_routes(self): + return [ + (r"^/error$", lambda: 1/0), + ] + + pm.register(TestPlugin(), name="undo") + try: + # The test implementation goes here + datasette = Datasette() + response = await datasette.client.get("/error") + assert response.status_code == 500 + finally: + pm.unregister(name="undo") From bca2d95d0228f80a108e13408f8e72b2c06c2c7b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 2 Aug 2022 16:38:02 -0700 Subject: [PATCH 0049/1172] Configure readthedocs/readthedocs-preview --- .github/workflows/documentation-links.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/documentation-links.yml diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml new file mode 100644 index 00000000..e7062a46 --- /dev/null +++ b/.github/workflows/documentation-links.yml @@ -0,0 +1,16 @@ +name: Read the Docs Pull Request Preview +on: + pull_request_target: + types: + - opened + +permissions: + pull-requests: write + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/readthedocs-preview@main + with: + project-slug: "datasette" From 8cfc72336878dd846d149658e99cc598e835b661 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Aug 2022 11:21:53 -0700 Subject: [PATCH 0050/1172] Ran blacken-docs --- docs/testing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index d02003a9..992b4b0e 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -244,7 +244,7 @@ When writing tests for plugins you may find it useful to register a test plugin @hookimpl def register_routes(self): return [ - (r"^/error$", lambda: 1/0), + (r"^/error$", lambda: 1 / 0), ] pm.register(TestPlugin(), name="undo") From 05d9c682689a0f1d23cbb502e027364ab3363910 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 08:16:53 -0700 Subject: [PATCH 0051/1172] Promote Discord more in the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 032180aa..7ebbca57 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover * Comprehensive documentation: https://docs.datasette.io/ * Examples: https://datasette.io/examples * Live demo of current main: https://latest.datasette.io/ -* Support questions, feedback? Join our [GitHub Discussions forum](https://github.com/simonw/datasette/discussions) +* Questions, feedback or want to talk about the project? Join our [Discord](https://discord.gg/ktd74dm5mw) Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. From db00c00f6397287749331e8042fe998ee7f3b919 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 08:19:30 -0700 Subject: [PATCH 0052/1172] Promote Datasette Lite in the README, refs #1781 --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ebbca57..1af20129 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover * Latest [Datasette News](https://datasette.io/news) * Comprehensive documentation: https://docs.datasette.io/ * Examples: https://datasette.io/examples -* Live demo of current main: https://latest.datasette.io/ +* Live demo of current `main` branch: https://latest.datasette.io/ * Questions, feedback or want to talk about the project? Join our [Discord](https://discord.gg/ktd74dm5mw) Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem. @@ -85,3 +85,7 @@ Or: This will create a docker image containing both the datasette application and the specified SQLite database files. It will then deploy that image to Heroku or Cloud Run and give you a URL to access the resulting website and API. See [Publishing data](https://docs.datasette.io/en/stable/publish.html) in the documentation for more details. + +## Datasette Lite + +[Datasette Lite](https://lite.datasette.io/) is Datasette packaged using WebAssembly so that it runs entirely in your browser, no Python web application server required. Read more about that in the [Datasette Lite documentation](https://github.com/simonw/datasette-lite/blob/main/README.md). From 8eb699de7becdefc6d72555d9fb17c9f06235dc4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 08:24:39 -0700 Subject: [PATCH 0053/1172] Datasette Lite in Getting Started docs, closes #1781 --- docs/getting_started.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 00b753a9..571540cf 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -21,6 +21,17 @@ Datasette has several `tutorials `__ to help you - `Exploring a database with Datasette `__ shows how to use the Datasette web interface to explore a new database. - `Learn SQL with Datasette `__ introduces SQL, and shows how to use that query language to ask questions of your data. +.. _getting_started_datasette_lite: + +Datasette in your browser with Datasette Lite +--------------------------------------------- + +`Datasette Lite `__ is Datasette packaged using WebAssembly so that it runs entirely in your browser, no Python web application server required. + +You can pass a URL to a CSV, SQLite or raw SQL file directly to Datasette Lite to explore that data in your browser. + +This `example link `__ opens Datasette Lite and loads the SQL Murder Mystery example database from `Northwestern University Knight Lab `__. + .. _getting_started_glitch: Try Datasette without installing anything using Glitch From df4fd2d7ddca8956d8a51c72ce007b8c75227f32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 08:44:02 -0700 Subject: [PATCH 0054/1172] _sort= works even if sort column not selected, closes #1773 --- datasette/views/table.py | 22 +++++++++++++++++++++- tests/test_table_api.py | 2 ++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index cd4be823..94d2673b 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -630,7 +630,27 @@ class TableView(DataView): next_value = path_from_row_pks(rows[-2], pks, use_rowid) # If there's a sort or sort_desc, add that value as a prefix if (sort or sort_desc) and not is_view: - prefix = rows[-2][sort or sort_desc] + try: + prefix = rows[-2][sort or sort_desc] + except IndexError: + # sort/sort_desc column missing from SELECT - look up value by PK instead + prefix_where_clause = " and ".join( + "[{}] = :pk{}".format(pk, i) for i, pk in enumerate(pks) + ) + prefix_lookup_sql = "select [{}] from [{}] where {}".format( + sort or sort_desc, table_name, prefix_where_clause + ) + prefix = ( + await db.execute( + prefix_lookup_sql, + { + **{ + "pk{}".format(i): rows[-2][pk] + for i, pk in enumerate(pks) + } + }, + ) + ).single_value() if isinstance(prefix, dict) and "value" in prefix: prefix = prefix["value"] if prefix is None: diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 9db383c3..e56a72b5 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -288,6 +288,8 @@ def test_paginate_compound_keys_with_extra_filters(app_client): ), # text column contains '$null' - ensure it doesn't confuse pagination: ("_sort=text", lambda row: row["text"], "sorted by text"), + # Still works if sort column removed using _col= + ("_sort=text&_col=content", lambda row: row["text"], "sorted by text"), ], ) def test_sortable(app_client, query_string, sort_key, human_description_en): From 668415df9f6334bd255c22ab02018bed5bc14edd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 08:47:17 -0700 Subject: [PATCH 0055/1172] Upgrade Docker baes to 3.10.6-slim-bullseye - refs #1768 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 42f5529b..ee7ed957 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.7-slim-bullseye as build +FROM python:3.10.6-slim-bullseye as build # Version of Datasette to install, e.g. 0.55 # docker build . -t datasette --build-arg VERSION=0.55 From 080d4b3e065d78faf977c6ded6ead31aae24e2ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 08:49:14 -0700 Subject: [PATCH 0056/1172] Switch to python:3.10.6-slim-bullseye for datasette publish - refs #1768 --- datasette/utils/__init__.py | 2 +- demos/apache-proxy/Dockerfile | 2 +- docs/publish.rst | 2 +- tests/test_package.py | 2 +- tests/test_publish_cloudrun.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 77768112..d148cc2c 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -390,7 +390,7 @@ def make_dockerfile( "SQLITE_EXTENSIONS" ] = "/usr/lib/x86_64-linux-gnu/mod_spatialite.so" return """ -FROM python:3.8 +FROM python:3.10.6-slim-bullseye COPY . /app WORKDIR /app {apt_get_extras} diff --git a/demos/apache-proxy/Dockerfile b/demos/apache-proxy/Dockerfile index 6c921963..70b33bec 100644 --- a/demos/apache-proxy/Dockerfile +++ b/demos/apache-proxy/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.7-slim-bullseye +FROM python:3.10.6-slim-bullseye RUN apt-get update && \ apt-get install -y apache2 supervisor && \ diff --git a/docs/publish.rst b/docs/publish.rst index 166f2883..9c7c99cc 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -144,7 +144,7 @@ Here's example output for the package command:: $ datasette package parlgov.db --extra-options="--setting sql_time_limit_ms 2500" Sending build context to Docker daemon 4.459MB - Step 1/7 : FROM python:3 + Step 1/7 : FROM python:3.10.6-slim-bullseye ---> 79e1dc9af1c1 Step 2/7 : COPY . /app ---> Using cache diff --git a/tests/test_package.py b/tests/test_package.py index 02ed1775..ac15e61e 100644 --- a/tests/test_package.py +++ b/tests/test_package.py @@ -12,7 +12,7 @@ class CaptureDockerfile: EXPECTED_DOCKERFILE = """ -FROM python:3.8 +FROM python:3.10.6-slim-bullseye COPY . /app WORKDIR /app diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 3427f4f7..60079ab3 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -223,7 +223,7 @@ def test_publish_cloudrun_plugin_secrets( ) expected = textwrap.dedent( r""" - FROM python:3.8 + FROM python:3.10.6-slim-bullseye COPY . /app WORKDIR /app @@ -290,7 +290,7 @@ def test_publish_cloudrun_apt_get_install( ) expected = textwrap.dedent( r""" - FROM python:3.8 + FROM python:3.10.6-slim-bullseye COPY . /app WORKDIR /app From 1563c22a8c65e6cff5194aa07df54d0ab8d4eecb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 09:13:12 -0700 Subject: [PATCH 0057/1172] Don't duplicate _sort_desc, refs #1738 --- datasette/views/table.py | 2 +- tests/test_table_html.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 94d2673b..49c30c9c 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -710,7 +710,7 @@ class TableView(DataView): for key in request.args: if ( key.startswith("_") - and key not in ("_sort", "_search", "_next") + and key not in ("_sort", "_sort_desc", "_search", "_next") and "__" not in key ): for value in request.args.getlist(key): diff --git a/tests/test_table_html.py b/tests/test_table_html.py index d3cb3e17..f3808ea3 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -828,6 +828,7 @@ def test_other_hidden_form_fields(app_client, path, expected_hidden): [ ("/fixtures/searchable?_search=terry", []), ("/fixtures/searchable?_sort=text2", []), + ("/fixtures/searchable?_sort_desc=text2", []), ("/fixtures/searchable?_sort=text2&_where=1", [("_where", "1")]), ], ) From c1396bf86033a7bd99fa0c0431f585475391a11a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 09:34:31 -0700 Subject: [PATCH 0058/1172] Don't allow canned write queries on immutable DBs, closes #1728 --- datasette/templates/query.html | 6 ++++- datasette/views/database.py | 4 ++++ tests/test_canned_queries.py | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 8c920527..cee779fc 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -28,6 +28,10 @@ {% block content %} +{% if canned_write and db_is_immutable %} +

This query cannot be executed because the database is immutable.

+{% endif %} +

{{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

{% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} @@ -61,7 +65,7 @@

{% if not hide_sql %}{% endif %} {% if canned_write %}{% endif %} - + {{ show_hide_hidden }} {% if canned_query and edit_sql_url %}Edit SQL{% endif %}

diff --git a/datasette/views/database.py b/datasette/views/database.py index 42058752..77632b9d 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -273,6 +273,9 @@ class QueryView(DataView): # Execute query - as write or as read if write: if request.method == "POST": + # If database is immutable, return an error + if not db.is_mutable: + raise Forbidden("Database is immutable") body = await request.post_body() body = body.decode("utf-8").strip() if body.startswith("{") and body.endswith("}"): @@ -326,6 +329,7 @@ class QueryView(DataView): async def extra_template(): return { "request": request, + "db_is_immutable": not db.is_mutable, "path_with_added_args": path_with_added_args, "path_with_removed_args": path_with_removed_args, "named_parameter_values": named_parameter_values, diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index 5abffdcc..976aa0db 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -53,6 +53,26 @@ def canned_write_client(tmpdir): yield client +@pytest.fixture +def canned_write_immutable_client(): + with make_app_client( + is_immutable=True, + metadata={ + "databases": { + "fixtures": { + "queries": { + "add": { + "sql": "insert into sortable (text) values (:text)", + "write": True, + }, + } + } + } + }, + ) as client: + yield client + + def test_canned_query_with_named_parameter(app_client): response = app_client.get("/fixtures/neighborhood_search.json?text=town") assert [ @@ -373,3 +393,23 @@ def test_canned_write_custom_template(canned_write_client): response.headers["link"] == 'http://localhost/data/update_name.json; rel="alternate"; type="application/json+datasette"' ) + + +def test_canned_write_query_disabled_for_immutable_database( + canned_write_immutable_client, +): + response = canned_write_immutable_client.get("/fixtures/add") + assert response.status == 200 + assert ( + "This query cannot be executed because the database is immutable." + in response.text + ) + assert '' in response.text + # Submitting form should get a forbidden error + response = canned_write_immutable_client.post( + "/fixtures/add", + {"text": "text"}, + csrftoken_from=True, + ) + assert response.status == 403 + assert "Database is immutable" in response.text From 82167105ee699c850cc106ea927de1ad09276cfe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 10:07:30 -0700 Subject: [PATCH 0059/1172] --min-instances and --max-instances Cloud Run publish options, closes #1779 --- datasette/publish/cloudrun.py | 26 +++++++++++++++++--- docs/cli-reference.rst | 2 ++ tests/test_publish_cloudrun.py | 43 ++++++++++++++++++++++++---------- 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/datasette/publish/cloudrun.py b/datasette/publish/cloudrun.py index 50b2b2fd..77274eb0 100644 --- a/datasette/publish/cloudrun.py +++ b/datasette/publish/cloudrun.py @@ -52,6 +52,16 @@ def publish_subcommand(publish): multiple=True, help="Additional packages to apt-get install", ) + @click.option( + "--max-instances", + type=int, + help="Maximum Cloud Run instances", + ) + @click.option( + "--min-instances", + type=int, + help="Minimum Cloud Run instances", + ) def cloudrun( files, metadata, @@ -79,6 +89,8 @@ def publish_subcommand(publish): cpu, timeout, apt_get_extras, + max_instances, + min_instances, ): "Publish databases to Datasette running on Cloud Run" fail_if_publish_binary_not_installed( @@ -168,12 +180,20 @@ def publish_subcommand(publish): ), shell=True, ) + extra_deploy_options = [] + for option, value in ( + ("--memory", memory), + ("--cpu", cpu), + ("--max-instances", max_instances), + ("--min-instances", min_instances), + ): + if value: + extra_deploy_options.append("{} {}".format(option, value)) check_call( - "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}{}".format( + "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format( image_id, service, - " --memory {}".format(memory) if memory else "", - " --cpu {}".format(cpu) if cpu else "", + " " + " ".join(extra_deploy_options) if extra_deploy_options else "", ), shell=True, ) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 1c1aff15..415af13c 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -251,6 +251,8 @@ datasette publish cloudrun --help --cpu [1|2|4] Number of vCPUs to allocate in Cloud Run --timeout INTEGER Build timeout in seconds --apt-get-install TEXT Additional packages to apt-get install + --max-instances INTEGER Maximum Cloud Run instances + --min-instances INTEGER Minimum Cloud Run instances --help Show this message and exit. diff --git a/tests/test_publish_cloudrun.py b/tests/test_publish_cloudrun.py index 60079ab3..e64534d2 100644 --- a/tests/test_publish_cloudrun.py +++ b/tests/test_publish_cloudrun.py @@ -105,19 +105,36 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory): @mock.patch("datasette.publish.cloudrun.check_output") @mock.patch("datasette.publish.cloudrun.check_call") @pytest.mark.parametrize( - "memory,cpu,timeout,expected_gcloud_args", + "memory,cpu,timeout,min_instances,max_instances,expected_gcloud_args", [ - ["1Gi", None, None, "--memory 1Gi"], - ["2G", None, None, "--memory 2G"], - ["256Mi", None, None, "--memory 256Mi"], - ["4", None, None, None], - ["GB", None, None, None], - [None, 1, None, "--cpu 1"], - [None, 2, None, "--cpu 2"], - [None, 3, None, None], - [None, 4, None, "--cpu 4"], - ["2G", 4, None, "--memory 2G --cpu 4"], - [None, None, 1800, "--timeout 1800"], + ["1Gi", None, None, None, None, "--memory 1Gi"], + ["2G", None, None, None, None, "--memory 2G"], + ["256Mi", None, None, None, None, "--memory 256Mi"], + [ + "4", + None, + None, + None, + None, + None, + ], + [ + "GB", + None, + None, + None, + None, + None, + ], + [None, 1, None, None, None, "--cpu 1"], + [None, 2, None, None, None, "--cpu 2"], + [None, 3, None, None, None, None], + [None, 4, None, None, None, "--cpu 4"], + ["2G", 4, None, None, None, "--memory 2G --cpu 4"], + [None, None, 1800, None, None, "--timeout 1800"], + [None, None, None, 2, None, "--min-instances 2"], + [None, None, None, 2, 4, "--min-instances 2 --max-instances 4"], + [None, 2, None, None, 4, "--cpu 2 --max-instances 4"], ], ) def test_publish_cloudrun_memory_cpu( @@ -127,6 +144,8 @@ def test_publish_cloudrun_memory_cpu( memory, cpu, timeout, + min_instances, + max_instances, expected_gcloud_args, tmp_path_factory, ): From 5e6c5c9e3191a80f17a91c5205d9d69efdebb73f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 10:18:47 -0700 Subject: [PATCH 0060/1172] Document datasette.config_dir, refs #1766 --- docs/internals.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/internals.rst b/docs/internals.rst index da135282..20797e98 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -260,6 +260,7 @@ Constructor parameters include: - ``files=[...]`` - a list of database files to open - ``immutables=[...]`` - a list of database files to open in immutable mode - ``metadata={...}`` - a dictionary of :ref:`metadata` +- ``config_dir=...`` - the :ref:`configuration directory ` to use, stored in ``datasette.config_dir`` .. _datasette_databases: From 815162cf029fab9f1c9308c1d6ecdba7ee369ebe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 10:32:42 -0700 Subject: [PATCH 0061/1172] Release 0.62 Refs #903, #1300, #1683, #1701, #1712, #1717, #1718, #1728, #1733, #1738, #1739, #1744, #1746, #1748, #1759, #1766, #1768, #1770, #1773, #1779 Closes #1782 --- datasette/version.py | 2 +- docs/changelog.rst | 53 ++++++++++++++++++++++++++++++-------------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index 86f4cf7e..0453346c 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.62a1" +__version__ = "0.62" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3f105811..1225c63f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,33 +4,52 @@ Changelog ========= -.. _v0_62a1: +.. _v0_62: -0.62a1 (2022-07-17) +0.62 (2022-08-14) ------------------- +Datasette can now run entirely in your browser using WebAssembly. Try out `Datasette Lite `__, take a look `at the code `__ or read more about it in `Datasette Lite: a server-side Python web application running in a browser `__. + +Datasette now has a `Discord community `__ for questions and discussions about Datasette and its ecosystem of projects. + +Features +~~~~~~~~ + +- Datasette is now compatible with `Pyodide `__. This is the enabling technology behind `Datasette Lite `__. (:issue:`1733`) +- Database file downloads now implement conditional GET using ETags. (:issue:`1739`) +- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (`#1759 `__) +- Datasette now runs some SQL queries in parallel. This has limited impact on performance, see `this research issue `__ for details. +- New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`) +- Spaces in the database names in URLs are now encoded as ``+`` rather than ``~20``. (:issue:`1701`) +- ```` is now displayed as ```` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`) +- The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image `__ has been upgraded to ``3.10.6-slim-bullseye``. (:issue:`1768`) +- Canned writable queries against immutable databases now show a warning message. (:issue:`1728`) +- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (`#1717 `__) +- ``datasette publish cloudrun`` has new ``--min-instances`` and ``--max-instances`` options. (:issue:`1779`) + +Plugin hooks +~~~~~~~~~~~~ + - New plugin hook: :ref:`handle_exception() `, for custom handling of exceptions caught by Datasette. (:issue:`1770`) - The :ref:`render_cell() ` plugin hook is now also passed a ``row`` argument, representing the ``sqlite3.Row`` object that is being rendered. (:issue:`1300`) -- New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`) -- Documentation now uses the `Furo `__ Sphinx theme. (:issue:`1746`) -- Datasette now has a `Discord community `__. -- Database file downloads now implement conditional GET using ETags. (:issue:`1739`) -- Examples in the documentation now include a copy-to-clipboard button. (:issue:`1748`) -- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (`#1759 `__) +- The :ref:`configuration directory ` is now stored in ``datasette.config_dir``, making it available to plugins. Thanks, Chris Amico. (`#1766 `__) -.. _v0_62a0: +Bug fixes +~~~~~~~~~ -0.62a0 (2022-05-02) -------------------- - -- Datasette now runs some SQL queries in parallel. This has limited impact on performance, see `this research issue `__ for details. -- Datasette should now be compatible with Pyodide. (:issue:`1733`) -- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (`#1717 `__) -- Spaces in database names are now encoded as ``+`` rather than ``~20``. (:issue:`1701`) -- ```` is now displayed as ```` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`) - Don't show the facet option in the cog menu if faceting is not allowed. (:issue:`1683`) +- ``?_sort`` and ``?_sort_desc`` now work if the column that is being sorted has been excluded from the query using ``?_col=`` or ``?_nocol=``. (:issue:`1773`) +- Fixed bug where ``?_sort_desc`` was duplicated in the URL every time the Apply button was clicked. (:issue:`1738`) + +Documentation +~~~~~~~~~~~~~ + +- Examples in the documentation now include a copy-to-clipboard button. (:issue:`1748`) +- Documentation now uses the `Furo `__ Sphinx theme. (:issue:`1746`) - Code examples in the documentation are now all formatted using Black. (:issue:`1718`) - ``Request.fake()`` method is now documented, see :ref:`internals_request`. +- New documentation for plugin authors: :ref:`testing_plugins_register_in_test`. (:issue:`903`) .. _v0_61_1: From a107e3a028923c1ab3911c0f880011283f93f368 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 16:07:46 -0700 Subject: [PATCH 0062/1172] datasette-sentry is an example of handle_exception --- docs/plugin_hooks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index aec1df56..c6f35d06 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1261,6 +1261,8 @@ This example logs an error to `Sentry `__ and then renders a return inner +Example: `datasette-sentry `_ + .. _plugin_hook_menu_links: menu_links(datasette, actor, request) From 481eb96d85291cdfa5767a83884a1525dfc382d8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 15 Aug 2022 13:17:28 -0700 Subject: [PATCH 0063/1172] https://datasette.io/tutorials/clean-data tutorial Refs #1783 --- docs/getting_started.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 571540cf..a9eaa404 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -20,6 +20,7 @@ Datasette has several `tutorials `__ to help you - `Exploring a database with Datasette `__ shows how to use the Datasette web interface to explore a new database. - `Learn SQL with Datasette `__ introduces SQL, and shows how to use that query language to ask questions of your data. +- `Cleaning data with sqlite-utils and Datasette `__ guides you through using `sqlite-utils `__ to turn a CSV file into a database that you can explore using Datasette. .. _getting_started_datasette_lite: From a3e6f1b16757fb2d39e7ddba4e09eda2362508bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Aug 2022 09:06:02 -0700 Subject: [PATCH 0064/1172] Increase height of non-JS textarea to fit query Closes #1786 --- datasette/templates/query.html | 3 ++- tests/test_html.py | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index cee779fc..a35e3afe 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -45,7 +45,8 @@ {% endif %} {% if not hide_sql %} {% if editable and allow_execute_sql %} -

+

{% else %} 
{% if query %}{{ query.sql }}{% endif %}
{% endif %} diff --git a/tests/test_html.py b/tests/test_html.py index 409fec68..be21bd84 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -695,10 +695,8 @@ def test_query_error(app_client): response = app_client.get("/fixtures?sql=select+*+from+notatable") html = response.text assert '

no such table: notatable

' in html - assert ( - '' - in html - ) + assert '" in html assert "0 results" not in html From 09a41662e70b788469157bb58ed9ca4acdf2f904 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Aug 2022 09:10:48 -0700 Subject: [PATCH 0065/1172] Fix typo --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index c6f35d06..30bd75b7 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -874,7 +874,7 @@ canned_queries(datasette, database, actor) ``actor`` - dictionary or None The currently authenticated :ref:`actor `. -Ues this hook to return a dictionary of additional :ref:`canned query ` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query ` documentation. +Use this hook to return a dictionary of additional :ref:`canned query ` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query ` documentation. .. code-block:: python From 6c0ba7c00c2ae3ecbb5309efa59079cea1c850b3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Aug 2022 14:52:04 -0700 Subject: [PATCH 0066/1172] Improved CLI reference documentation, refs #1787 --- datasette/cli.py | 2 +- docs/changelog.rst | 2 +- docs/cli-reference.rst | 325 ++++++++++++++++++++++++++++++--------- docs/getting_started.rst | 50 ------ docs/index.rst | 2 +- docs/publish.rst | 2 + 6 files changed, 259 insertions(+), 124 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 8781747c..f2a03d53 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -282,7 +282,7 @@ def package( port, **extra_metadata, ): - """Package specified SQLite files into a new datasette Docker container""" + """Package SQLite files into a Datasette Docker container""" if not shutil.which("docker"): click.secho( ' The package command requires "docker" to be installed and configured ', diff --git a/docs/changelog.rst b/docs/changelog.rst index 1225c63f..f9dcc980 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -621,7 +621,7 @@ See also `Datasette 0.49: The annotated release notes `__ for conversations about the project that go beyond just bug reports and issues. - Datasette can now be installed on macOS using Homebrew! Run ``brew install simonw/datasette/datasette``. See :ref:`installation_homebrew`. (:issue:`335`) - Two new commands: ``datasette install name-of-plugin`` and ``datasette uninstall name-of-plugin``. These are equivalent to ``pip install`` and ``pip uninstall`` but automatically run in the same virtual environment as Datasette, so users don't have to figure out where that virtual environment is - useful for installations created using Homebrew or ``pipx``. See :ref:`plugins_installing`. (:issue:`925`) -- A new command-line option, ``datasette --get``, accepts a path to a URL within the Datasette instance. It will run that request through Datasette (without starting a web server) and print out the response. See :ref:`getting_started_datasette_get` for an example. (:issue:`926`) +- A new command-line option, ``datasette --get``, accepts a path to a URL within the Datasette instance. It will run that request through Datasette (without starting a web server) and print out the response. See :ref:`cli_datasette_get` for an example. (:issue:`926`) .. _v0_46: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 415af13c..a1e56774 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -4,44 +4,34 @@ CLI reference =============== -This page lists the ``--help`` for every ``datasette`` CLI command. +The ``datasette`` CLI tool provides a number of commands. + +Running ``datasette`` without specifying a command runs the default command, ``datasette serve``. See :ref:`cli_help_serve___help` for the full list of options for that command. .. [[[cog from datasette import cli from click.testing import CliRunner import textwrap - commands = [ - ["--help"], - ["serve", "--help"], - ["serve", "--help-settings"], - ["plugins", "--help"], - ["publish", "--help"], - ["publish", "cloudrun", "--help"], - ["publish", "heroku", "--help"], - ["package", "--help"], - ["inspect", "--help"], - ["install", "--help"], - ["uninstall", "--help"], - ] - cog.out("\n") - for command in commands: - title = "datasette " + " ".join(command) - ref = "_cli_help_" + ("_".join(command).replace("-", "_")) - cog.out(".. {}:\n\n".format(ref)) - cog.out(title + "\n") - cog.out(("=" * len(title)) + "\n\n") + def help(args): + title = "datasette " + " ".join(args) cog.out("::\n\n") - result = CliRunner().invoke(cli.cli, command) + result = CliRunner().invoke(cli.cli, args) output = result.output.replace("Usage: cli ", "Usage: datasette ") cog.out(textwrap.indent(output, ' ')) cog.out("\n\n") .. ]]] +.. [[[end]]] .. _cli_help___help: datasette --help ================ +Running ``datasette --help`` shows a list of all of the available commands. + +.. [[[cog + help(["--help"]) +.. ]]] :: Usage: datasette [OPTIONS] COMMAND [ARGS]... @@ -59,17 +49,34 @@ datasette --help serve* Serve up specified SQLite database files with a web UI inspect Generate JSON summary of provided database files install Install plugins and packages from PyPI into the same... - package Package specified SQLite files into a new datasette Docker... + package Package SQLite files into a Datasette Docker container plugins List currently installed plugins publish Publish specified SQLite database files to the internet along... uninstall Uninstall plugins and Python packages from the Datasette... +.. [[[end]]] + +Additional commands added by plugins that use the :ref:`plugin_hook_register_commands` hook will be listed here as well. + .. _cli_help_serve___help: -datasette serve --help -====================== +datasette serve +=============== +This command starts the Datasette web application running on your machine:: + + datasette serve mydatabase.db + +Or since this is the default command you can run this instead:: + + datasette mydatabase.db + +Once started you can access it at ``http://localhost:8001`` + +.. [[[cog + help(["serve", "--help"]) +.. ]]] :: Usage: datasette serve [OPTIONS] [FILES]... @@ -121,11 +128,75 @@ datasette serve --help --help Show this message and exit. +.. [[[end]]] + + +.. _cli_datasette_get: + +datasette --get +--------------- + +The ``--get`` option to ``datasette serve`` (or just ``datasette``) specifies the path to a page within Datasette and causes Datasette to output the content from that path without starting the web server. + +This means that all of Datasette's functionality can be accessed directly from the command-line. + +For example:: + + $ datasette --get '/-/versions.json' | jq . + { + "python": { + "version": "3.8.5", + "full": "3.8.5 (default, Jul 21 2020, 10:48:26) \n[Clang 11.0.3 (clang-1103.0.32.62)]" + }, + "datasette": { + "version": "0.46+15.g222a84a.dirty" + }, + "asgi": "3.0", + "uvicorn": "0.11.8", + "sqlite": { + "version": "3.32.3", + "fts_versions": [ + "FTS5", + "FTS4", + "FTS3" + ], + "extensions": { + "json1": null + }, + "compile_options": [ + "COMPILER=clang-11.0.3", + "ENABLE_COLUMN_METADATA", + "ENABLE_FTS3", + "ENABLE_FTS3_PARENTHESIS", + "ENABLE_FTS4", + "ENABLE_FTS5", + "ENABLE_GEOPOLY", + "ENABLE_JSON1", + "ENABLE_PREUPDATE_HOOK", + "ENABLE_RTREE", + "ENABLE_SESSION", + "MAX_VARIABLE_NUMBER=250000", + "THREADSAFE=1" + ] + } + } + +The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. + +This lets you use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions. + .. _cli_help_serve___help_settings: datasette serve --help-settings -=============================== +------------------------------- +This command outputs all of the available Datasette :ref:`settings `. + +These can be passed to ``datasette serve`` using ``datasette serve --setting name value``. + +.. [[[cog + help(["--help-settings"]) +.. ]]] :: Settings: @@ -170,11 +241,18 @@ datasette serve --help-settings +.. [[[end]]] + .. _cli_help_plugins___help: -datasette plugins --help -======================== +datasette plugins +================= +Output JSON showing all currently installed plugins, their versions, whether they include static files or templates and which :ref:`plugin_hooks` they use. + +.. [[[cog + help(["plugins", "--help"]) +.. ]]] :: Usage: datasette plugins [OPTIONS] @@ -187,11 +265,110 @@ datasette plugins --help --help Show this message and exit. +.. [[[end]]] + +Example output: + +.. code-block:: json + + [ + { + "name": "datasette-geojson", + "static": false, + "templates": false, + "version": "0.3.1", + "hooks": [ + "register_output_renderer" + ] + }, + { + "name": "datasette-geojson-map", + "static": true, + "templates": false, + "version": "0.4.0", + "hooks": [ + "extra_body_script", + "extra_css_urls", + "extra_js_urls" + ] + }, + { + "name": "datasette-leaflet", + "static": true, + "templates": false, + "version": "0.2.2", + "hooks": [ + "extra_body_script", + "extra_template_vars" + ] + } + ] + + +.. _cli_help_install___help: + +datasette install +================= + +Install new Datasette plugins. This command works like ``pip install`` but ensures that your plugins will be installed into the same environment as Datasette. + +This command:: + + datasette install datasette-cluster-map + +Would install the `datasette-cluster-map `__ plugin. + +.. [[[cog + help(["install", "--help"]) +.. ]]] +:: + + Usage: datasette install [OPTIONS] PACKAGES... + + Install plugins and packages from PyPI into the same environment as Datasette + + Options: + -U, --upgrade Upgrade packages to latest version + --help Show this message and exit. + + +.. [[[end]]] + +.. _cli_help_uninstall___help: + +datasette uninstall +=================== + +Uninstall one or more plugins. + +.. [[[cog + help(["uninstall", "--help"]) +.. ]]] +:: + + Usage: datasette uninstall [OPTIONS] PACKAGES... + + Uninstall plugins and Python packages from the Datasette environment + + Options: + -y, --yes Don't ask for confirmation + --help Show this message and exit. + + +.. [[[end]]] + .. _cli_help_publish___help: -datasette publish --help -======================== +datasette publish +================= +Shows a list of available deployment targets for :ref:`publishing data ` with Datasette. + +Additional deployment targets can be added by plugins that use the :ref:`plugin_hook_publish_subcommand` hook. + +.. [[[cog + help(["publish", "--help"]) +.. ]]] :: Usage: datasette publish [OPTIONS] COMMAND [ARGS]... @@ -207,11 +384,19 @@ datasette publish --help heroku Publish databases to Datasette running on Heroku +.. [[[end]]] + + .. _cli_help_publish_cloudrun___help: -datasette publish cloudrun --help -================================= +datasette publish cloudrun +========================== +See :ref:`publish_cloud_run`. + +.. [[[cog + help(["publish", "cloudrun", "--help"]) +.. ]]] :: Usage: datasette publish cloudrun [OPTIONS] [FILES]... @@ -256,11 +441,19 @@ datasette publish cloudrun --help --help Show this message and exit. +.. [[[end]]] + + .. _cli_help_publish_heroku___help: -datasette publish heroku --help -=============================== +datasette publish heroku +======================== +See :ref:`publish_heroku`. + +.. [[[cog + help(["publish", "heroku", "--help"]) +.. ]]] :: Usage: datasette publish heroku [OPTIONS] [FILES]... @@ -297,16 +490,23 @@ datasette publish heroku --help --help Show this message and exit. +.. [[[end]]] + .. _cli_help_package___help: -datasette package --help -======================== +datasette package +================= +Package SQLite files into a Datasette Docker container, see :ref:`cli_package`. + +.. [[[cog + help(["package", "--help"]) +.. ]]] :: Usage: datasette package [OPTIONS] FILES... - Package specified SQLite files into a new datasette Docker container + Package SQLite files into a Datasette Docker container Options: -t, --tag TEXT Name for the resulting Docker container, can @@ -335,11 +535,26 @@ datasette package --help --help Show this message and exit. +.. [[[end]]] + + .. _cli_help_inspect___help: -datasette inspect --help -======================== +datasette inspect +================= +Outputs JSON representing introspected data about one or more SQLite database files. + +If you are opening an immutable database, you can pass this file to the ``--inspect-data`` option to improve Datasette's performance by allowing it to skip running row counts against the database when it first starts running:: + + datasette inspect mydatabase.db > inspect-data.json + datasette serve -i mydatabase.db --inspect-file inspect-data.json + +This performance optimization is used automatically by some of the ``datasette publish`` commands. You are unlikely to need to apply this optimization manually. + +.. [[[cog + help(["inspect", "--help"]) +.. ]]] :: Usage: datasette inspect [OPTIONS] [FILES]... @@ -355,36 +570,4 @@ datasette inspect --help --help Show this message and exit. -.. _cli_help_install___help: - -datasette install --help -======================== - -:: - - Usage: datasette install [OPTIONS] PACKAGES... - - Install plugins and packages from PyPI into the same environment as Datasette - - Options: - -U, --upgrade Upgrade packages to latest version - --help Show this message and exit. - - -.. _cli_help_uninstall___help: - -datasette uninstall --help -========================== - -:: - - Usage: datasette uninstall [OPTIONS] PACKAGES... - - Uninstall plugins and Python packages from the Datasette environment - - Options: - -y, --yes Don't ask for confirmation - --help Show this message and exit. - - .. [[[end]]] diff --git a/docs/getting_started.rst b/docs/getting_started.rst index a9eaa404..6515ef8d 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -138,53 +138,3 @@ JSON in a more convenient format: } ] } - -.. _getting_started_datasette_get: - -datasette --get ---------------- - -The ``--get`` option can specify the path to a page within Datasette and cause Datasette to output the content from that path without starting the web server. This means that all of Datasette's functionality can be accessed directly from the command-line. For example:: - - $ datasette --get '/-/versions.json' | jq . - { - "python": { - "version": "3.8.5", - "full": "3.8.5 (default, Jul 21 2020, 10:48:26) \n[Clang 11.0.3 (clang-1103.0.32.62)]" - }, - "datasette": { - "version": "0.46+15.g222a84a.dirty" - }, - "asgi": "3.0", - "uvicorn": "0.11.8", - "sqlite": { - "version": "3.32.3", - "fts_versions": [ - "FTS5", - "FTS4", - "FTS3" - ], - "extensions": { - "json1": null - }, - "compile_options": [ - "COMPILER=clang-11.0.3", - "ENABLE_COLUMN_METADATA", - "ENABLE_FTS3", - "ENABLE_FTS3_PARENTHESIS", - "ENABLE_FTS4", - "ENABLE_FTS5", - "ENABLE_GEOPOLY", - "ENABLE_JSON1", - "ENABLE_PREUPDATE_HOOK", - "ENABLE_RTREE", - "ENABLE_SESSION", - "MAX_VARIABLE_NUMBER=250000", - "THREADSAFE=1" - ] - } - } - -The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. This means you can use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions. - -Running ``datasette`` without specifying a command runs the default command, ``datasette serve``. See :ref:`cli_help_serve___help` for the full list of options for that command. diff --git a/docs/index.rst b/docs/index.rst index efe196b3..5a9cc7ed 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,6 +40,7 @@ Contents getting_started installation ecosystem + cli-reference pages publish deploying @@ -61,6 +62,5 @@ Contents plugin_hooks testing_plugins internals - cli-reference contributing changelog diff --git a/docs/publish.rst b/docs/publish.rst index 9c7c99cc..dd8566ed 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -56,6 +56,8 @@ Cloud Run provides a URL on the ``.run.app`` domain, but you can also point your See :ref:`cli_help_publish_cloudrun___help` for the full list of options for this command. +.. _publish_heroku: + Publishing to Heroku -------------------- From aff3df03d4fe0806ce432d1818f6643cdb2a854e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Aug 2022 14:55:08 -0700 Subject: [PATCH 0067/1172] Ignore ro which stands for read only Refs #1787 where it caused tests to break --- docs/codespell-ignore-words.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt index a625cde5..d6744d05 100644 --- a/docs/codespell-ignore-words.txt +++ b/docs/codespell-ignore-words.txt @@ -1 +1 @@ -AddWordsToIgnoreHere +ro From 0d9d33955b503c88a2c712144d97f094baa5d46d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Aug 2022 16:06:12 -0700 Subject: [PATCH 0068/1172] Clarify you can publish multiple files, closes #1788 --- docs/publish.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/publish.rst b/docs/publish.rst index dd8566ed..d817ed31 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -31,7 +31,7 @@ Publishing to Google Cloud Run You will first need to install and configure the Google Cloud CLI tools by following `these instructions `__. -You can then publish a database to Google Cloud Run using the following command:: +You can then publish one or more SQLite database files to Google Cloud Run using the following command:: datasette publish cloudrun mydatabase.db --service=my-database @@ -63,7 +63,7 @@ Publishing to Heroku To publish your data using `Heroku `__, first create an account there and install and configure the `Heroku CLI tool `_. -You can publish a database to Heroku using the following command:: +You can publish one or more databases to Heroku using the following command:: datasette publish heroku mydatabase.db @@ -138,7 +138,7 @@ If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plug datasette package ================= -If you have docker installed (e.g. using `Docker for Mac `_) you can use the ``datasette package`` command to create a new Docker image in your local repository containing the datasette app bundled together with your selected SQLite databases:: +If you have docker installed (e.g. using `Docker for Mac `_) you can use the ``datasette package`` command to create a new Docker image in your local repository containing the datasette app bundled together with one or more SQLite databases:: datasette package mydatabase.db From 663ac431fe7202c85967568d82b2034f92b9aa43 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sat, 20 Aug 2022 02:04:16 +0200 Subject: [PATCH 0069/1172] Use Read the Docs action v1 (#1778) Read the Docs repository was renamed from `readthedocs/readthedocs-preview` to `readthedocs/actions/`. Now, the `preview` action is under `readthedocs/actions/preview` and is tagged as `v1` --- .github/workflows/documentation-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index e7062a46..a54bd83a 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -11,6 +11,6 @@ jobs: documentation-links: runs-on: ubuntu-latest steps: - - uses: readthedocs/readthedocs-preview@main + - uses: readthedocs/actions/preview@v1 with: project-slug: "datasette" From 1d64c9a8dac45b9a3452acf8e76dfadea2b0bc49 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Tue, 23 Aug 2022 11:34:30 -0700 Subject: [PATCH 0070/1172] Add new entrypoint option to --load-extensions. (#1789) Thanks, @asg017 --- .gitignore | 6 ++++ datasette/app.py | 8 ++++- datasette/cli.py | 4 ++- datasette/utils/__init__.py | 11 ++++++ tests/ext.c | 48 ++++++++++++++++++++++++++ tests/test_load_extensions.py | 65 +++++++++++++++++++++++++++++++++++ 6 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 tests/ext.c create mode 100644 tests/test_load_extensions.py diff --git a/.gitignore b/.gitignore index 066009f0..277ff653 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,9 @@ ENV/ .DS_Store node_modules .*.swp + +# In case someone compiled tests/ext.c for test_load_extensions, don't +# include it in source control. +tests/*.dylib +tests/*.so +tests/*.dll \ No newline at end of file diff --git a/datasette/app.py b/datasette/app.py index 1a9afc10..bb9232c9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -559,7 +559,13 @@ class Datasette: if self.sqlite_extensions: conn.enable_load_extension(True) for extension in self.sqlite_extensions: - conn.execute("SELECT load_extension(?)", [extension]) + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. + if isinstance(extension, tuple): + path, entrypoint = extension + conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) + else: + conn.execute("SELECT load_extension(?)", [extension]) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member diff --git a/datasette/cli.py b/datasette/cli.py index f2a03d53..6eb42712 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -21,6 +21,7 @@ from .app import ( pm, ) from .utils import ( + LoadExtension, StartupError, check_connection, find_spatialite, @@ -128,9 +129,10 @@ def sqlite_extensions(fn): return click.option( "sqlite_extensions", "--load-extension", + type=LoadExtension(), envvar="SQLITE_EXTENSIONS", multiple=True, - help="Path to a SQLite extension to load", + help="Path to a SQLite extension to load, and optional entrypoint", )(fn) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index d148cc2c..0fc87d51 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -833,6 +833,17 @@ class StaticMount(click.ParamType): self.fail(f"{value} is not a valid directory path", param, ctx) return path, dirpath +# The --load-extension parameter can optionally include a specific entrypoint. +# This is done by appending ":entrypoint_name" after supplying the path to the extension +class LoadExtension(click.ParamType): + name = "path:entrypoint?" + + def convert(self, value, param, ctx): + if ":" not in value: + return value + path, entrypoint = value.split(":", 1) + return path, entrypoint + def format_bytes(bytes): current = float(bytes) diff --git a/tests/ext.c b/tests/ext.c new file mode 100644 index 00000000..5fe970d9 --- /dev/null +++ b/tests/ext.c @@ -0,0 +1,48 @@ +/* +** This file implements a SQLite extension with multiple entrypoints. +** +** The default entrypoint, sqlite3_ext_init, has a single function "a". +** The 1st alternate entrypoint, sqlite3_ext_b_init, has a single function "b". +** The 2nd alternate entrypoint, sqlite3_ext_c_init, has a single function "c". +** +** Compiling instructions: +** https://www.sqlite.org/loadext.html#compiling_a_loadable_extension +** +*/ + +#include "sqlite3ext.h" + +SQLITE_EXTENSION_INIT1 + +// SQL function that returns back the value supplied during sqlite3_create_function() +static void func(sqlite3_context *context, int argc, sqlite3_value **argv) { + sqlite3_result_text(context, (char *) sqlite3_user_data(context), -1, SQLITE_STATIC); +} + + +// The default entrypoint, since it matches the "ext.dylib"/"ext.so" name +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "a", 0, 0, "a", func, 0, 0); +} + +// Alternate entrypoint #1 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_b_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "b", 0, 0, "b", func, 0, 0); +} + +// Alternate entrypoint #2 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_c_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "c", 0, 0, "c", func, 0, 0); +} diff --git a/tests/test_load_extensions.py b/tests/test_load_extensions.py new file mode 100644 index 00000000..360bc8f3 --- /dev/null +++ b/tests/test_load_extensions.py @@ -0,0 +1,65 @@ +from datasette.app import Datasette +import pytest +from pathlib import Path + +# not necessarily a full path - the full compiled path looks like "ext.dylib" +# or another suffix, but sqlite will, under the hood, decide which file +# extension to use based on the operating system (apple=dylib, windows=dll etc) +# this resolves to "./ext", which is enough for SQLite to calculate the rest +COMPILED_EXTENSION_PATH = str(Path(__file__).parent / "ext") + +# See if ext.c has been compiled, based off the different possible suffixes. +def has_compiled_ext(): + for ext in ["dylib", "so", "dll"]: + path = Path(__file__).parent / f"ext.{ext}" + if path.is_file(): + return True + return False + + +@pytest.mark.asyncio +@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c") +async def test_load_extension_default_entrypoint(): + + # The default entrypoint only loads a() and NOT b() or c(), so those + # should fail. + ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH]) + + response = await ds.client.get("/_memory.json?sql=select+a()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "a" + + response = await ds.client.get("/_memory.json?sql=select+b()") + assert response.status_code == 400 + assert response.json()["error"] == "no such function: b" + + response = await ds.client.get("/_memory.json?sql=select+c()") + assert response.status_code == 400 + assert response.json()["error"] == "no such function: c" + + +@pytest.mark.asyncio +@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c") +async def test_load_extension_multiple_entrypoints(): + + # Load in the default entrypoint and the other 2 custom entrypoints, now + # all a(), b(), and c() should run successfully. + ds = Datasette( + sqlite_extensions=[ + COMPILED_EXTENSION_PATH, + (COMPILED_EXTENSION_PATH, "sqlite3_ext_b_init"), + (COMPILED_EXTENSION_PATH, "sqlite3_ext_c_init"), + ] + ) + + response = await ds.client.get("/_memory.json?sql=select+a()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "a" + + response = await ds.client.get("/_memory.json?sql=select+b()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "b" + + response = await ds.client.get("/_memory.json?sql=select+c()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "c" From fd1086c6867f3e3582b1eca456e4ea95f6cecf8b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Aug 2022 11:35:41 -0700 Subject: [PATCH 0071/1172] Applied Black, refs #1789 --- datasette/app.py | 4 ++-- datasette/utils/__init__.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index bb9232c9..f2a6763a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -559,8 +559,8 @@ class Datasette: if self.sqlite_extensions: conn.enable_load_extension(True) for extension in self.sqlite_extensions: - # "extension" is either a string path to the extension - # or a 2-item tuple that specifies which entrypoint to load. + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. if isinstance(extension, tuple): path, entrypoint = extension conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 0fc87d51..bbaa0510 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -833,6 +833,7 @@ class StaticMount(click.ParamType): self.fail(f"{value} is not a valid directory path", param, ctx) return path, dirpath + # The --load-extension parameter can optionally include a specific entrypoint. # This is done by appending ":entrypoint_name" after supplying the path to the extension class LoadExtension(click.ParamType): From 456dc155d491a009942ace71a4e1827cddc6b93d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Aug 2022 11:40:36 -0700 Subject: [PATCH 0072/1172] Ran cog, refs #1789 --- docs/cli-reference.rst | 95 +++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index a1e56774..f8419d58 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -84,48 +84,53 @@ Once started you can access it at ``http://localhost:8001`` Serve up specified SQLite database files with a web UI Options: - -i, --immutable PATH Database files to open in immutable mode - -h, --host TEXT Host for server. Defaults to 127.0.0.1 which means - only connections from the local machine will be - allowed. Use 0.0.0.0 to listen to all IPs and allow - access from other machines. - -p, --port INTEGER RANGE Port for server, defaults to 8001. Use -p 0 to - automatically assign an available port. - [0<=x<=65535] - --uds TEXT Bind to a Unix domain socket - --reload Automatically reload if code or metadata change - detected - useful for development - --cors Enable CORS by serving Access-Control-Allow-Origin: - * - --load-extension TEXT Path to a SQLite extension to load - --inspect-file TEXT Path to JSON file created using "datasette inspect" - -m, --metadata FILENAME Path to JSON/YAML file containing license/source - metadata - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... - --memory Make /_memory database available - --config CONFIG Deprecated: set config option using - configname:value. Use --setting instead. - --setting SETTING... Setting, see - docs.datasette.io/en/stable/settings.html - --secret TEXT Secret used for signing secure values, such as - signed cookies - --root Output URL that sets a cookie authenticating the - root user - --get TEXT Run an HTTP GET request against this path, print - results and exit - --version-note TEXT Additional note to show on /-/versions - --help-settings Show available settings - --pdb Launch debugger on any errors - -o, --open Open Datasette in your web browser - --create Create database files if they do not exist - --crossdb Enable cross-database joins using the /_memory - database - --nolock Ignore locking, open locked files in read-only mode - --ssl-keyfile TEXT SSL key file - --ssl-certfile TEXT SSL certificate file - --help Show this message and exit. + -i, --immutable PATH Database files to open in immutable mode + -h, --host TEXT Host for server. Defaults to 127.0.0.1 which + means only connections from the local machine + will be allowed. Use 0.0.0.0 to listen to all + IPs and allow access from other machines. + -p, --port INTEGER RANGE Port for server, defaults to 8001. Use -p 0 to + automatically assign an available port. + [0<=x<=65535] + --uds TEXT Bind to a Unix domain socket + --reload Automatically reload if code or metadata + change detected - useful for development + --cors Enable CORS by serving Access-Control-Allow- + Origin: * + --load-extension PATH:ENTRYPOINT? + Path to a SQLite extension to load, and + optional entrypoint + --inspect-file TEXT Path to JSON file created using "datasette + inspect" + -m, --metadata FILENAME Path to JSON/YAML file containing + license/source metadata + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static MOUNT:DIRECTORY Serve static files from this directory at + /MOUNT/... + --memory Make /_memory database available + --config CONFIG Deprecated: set config option using + configname:value. Use --setting instead. + --setting SETTING... Setting, see + docs.datasette.io/en/stable/settings.html + --secret TEXT Secret used for signing secure values, such as + signed cookies + --root Output URL that sets a cookie authenticating + the root user + --get TEXT Run an HTTP GET request against this path, + print results and exit + --version-note TEXT Additional note to show on /-/versions + --help-settings Show available settings + --pdb Launch debugger on any errors + -o, --open Open Datasette in your web browser + --create Create database files if they do not exist + --crossdb Enable cross-database joins using the /_memory + database + --nolock Ignore locking, open locked files in read-only + mode + --ssl-keyfile TEXT SSL key file + --ssl-certfile TEXT SSL certificate file + --help Show this message and exit. .. [[[end]]] @@ -566,8 +571,10 @@ This performance optimization is used automatically by some of the ``datasette p Options: --inspect-file TEXT - --load-extension TEXT Path to a SQLite extension to load - --help Show this message and exit. + --load-extension PATH:ENTRYPOINT? + Path to a SQLite extension to load, and + optional entrypoint + --help Show this message and exit. .. [[[end]]] From ba35105eee2d3ba620e4f230028a02b2e2571df2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Aug 2022 17:11:45 -0700 Subject: [PATCH 0073/1172] Test `--load-extension` in GitHub Actions (#1792) * Run the --load-extension test, refs #1789 * Ran cog, refs #1789 --- .github/workflows/test.yml | 3 +++ tests/test_api.py | 2 +- tests/test_html.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90b6555e..e38d5ee9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,9 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- + - name: Build extension for --load-extension test + run: |- + (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | pip install -e '.[test]' diff --git a/tests/test_api.py b/tests/test_api.py index 253c1718..f6db2f9d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -36,7 +36,7 @@ def test_homepage(app_client): # 4 hidden FTS tables + no_primary_key (hidden in metadata) assert d["hidden_tables_count"] == 6 # 201 in no_primary_key, plus 6 in other hidden tables: - assert d["hidden_table_rows_sum"] == 207 + assert d["hidden_table_rows_sum"] == 207, response.json assert d["views_count"] == 4 diff --git a/tests/test_html.py b/tests/test_html.py index be21bd84..d6e969ad 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -115,7 +115,7 @@ def test_database_page(app_client): assert fragment in response.text # And views - views_ul = soup.find("h2", text="Views").find_next_sibling("ul") + views_ul = soup.find("h2", string="Views").find_next_sibling("ul") assert views_ul is not None assert [ ("/fixtures/paginated_view", "paginated_view"), @@ -128,7 +128,7 @@ def test_database_page(app_client): ] == sorted([(a["href"], a.text) for a in views_ul.find_all("a")]) # And a list of canned queries - queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul") + queries_ul = soup.find("h2", string="Queries").find_next_sibling("ul") assert queries_ul is not None assert [ ("/fixtures/from_async_hook", "from_async_hook"), From 51030df1869b3b574dd3584d1563415776b9cd4e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 11:35:40 -0700 Subject: [PATCH 0074/1172] Don't use upper bound dependencies any more See https://iscinumpy.dev/post/bound-version-constraints/ for the rationale behind this change. Closes #1800 --- setup.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/setup.py b/setup.py index a1c51d0b..b2e50b38 100644 --- a/setup.py +++ b/setup.py @@ -42,21 +42,21 @@ setup( include_package_data=True, python_requires=">=3.7", install_requires=[ - "asgiref>=3.2.10,<3.6.0", - "click>=7.1.1,<8.2.0", + "asgiref>=3.2.10", + "click>=7.1.1", "click-default-group-wheel>=1.2.2", - "Jinja2>=2.10.3,<3.1.0", - "hupper~=1.9", + "Jinja2>=2.10.3", + "hupper>=1.9", "httpx>=0.20", - "pint~=0.9", - "pluggy>=1.0,<1.1", - "uvicorn~=0.11", - "aiofiles>=0.4,<0.9", - "janus>=0.6.2,<1.1", + "pint>=0.9", + "pluggy>=1.0", + "uvicorn>=0.11", + "aiofiles>=0.4", + "janus>=0.6.2", "asgi-csrf>=0.9", - "PyYAML>=5.3,<7.0", - "mergedeep>=1.1.1,<1.4.0", - "itsdangerous>=1.1,<3.0", + "PyYAML>=5.3", + "mergedeep>=1.1.1", + "itsdangerous>=1.1", ], entry_points=""" [console_scripts] @@ -72,14 +72,14 @@ setup( "sphinx-copybutton", ], "test": [ - "pytest>=5.2.2,<7.2.0", - "pytest-xdist>=2.2.1,<2.6", - "pytest-asyncio>=0.17,<0.20", - "beautifulsoup4>=4.8.1,<4.12.0", + "pytest>=5.2.2", + "pytest-xdist>=2.2.1", + "pytest-asyncio>=0.17", + "beautifulsoup4>=4.8.1", "black==22.6.0", "blacken-docs==1.12.1", - "pytest-timeout>=1.4.2,<2.2", - "trustme>=0.7,<0.10", + "pytest-timeout>=1.4.2", + "trustme>=0.7", "cogapp>=3.3.0", ], "rich": ["rich"], From 294ecd45f7801971dbeef383d0c5456ee95ab839 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Sep 2022 11:51:51 -0700 Subject: [PATCH 0075/1172] Bump black from 22.6.0 to 22.8.0 (#1797) Bumps [black](https://github.com/psf/black) from 22.6.0 to 22.8.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.6.0...22.8.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b2e50b38..92fa60d0 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==22.6.0", + "black==22.8.0", "blacken-docs==1.12.1", "pytest-timeout>=1.4.2", "trustme>=0.7", From b91e17280c05bbb9cf97432081bdcea8665879f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 16:50:53 -0700 Subject: [PATCH 0076/1172] Run tests in serial, refs #1802 --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e38d5ee9..9c8c48ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,8 +33,7 @@ jobs: pip freeze - name: Run tests run: | - pytest -n auto -m "not serial" - pytest -m "serial" + pytest - name: Check if cog needs to be run run: | cog --check docs/*.rst From b2b901e8c4b939e50ee1117ffcd2881ed8a8e3bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 17:05:23 -0700 Subject: [PATCH 0077/1172] Skip SpatiaLite test if no conn.enable_load_extension() Ran into this problem while working on #1802 --- tests/test_spatialite.py | 2 ++ tests/utils.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/tests/test_spatialite.py b/tests/test_spatialite.py index 8b98c5d6..c07a30e8 100644 --- a/tests/test_spatialite.py +++ b/tests/test_spatialite.py @@ -1,5 +1,6 @@ from datasette.app import Datasette from datasette.utils import find_spatialite, SpatialiteNotFound, SPATIALITE_FUNCTIONS +from .utils import has_load_extension import pytest @@ -13,6 +14,7 @@ def has_spatialite(): @pytest.mark.asyncio @pytest.mark.skipif(not has_spatialite(), reason="Requires SpatiaLite") +@pytest.mark.skipif(not has_load_extension(), reason="Requires enable_load_extension") async def test_spatialite_version_info(): ds = Datasette(sqlite_extensions=["spatialite"]) response = await ds.client.get("/-/versions.json") diff --git a/tests/utils.py b/tests/utils.py index 972300db..191ead9b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,3 +1,6 @@ +from datasette.utils.sqlite import sqlite3 + + def assert_footer_links(soup): footer_links = soup.find("footer").findAll("a") assert 4 == len(footer_links) @@ -22,3 +25,8 @@ def inner_html(soup): # This includes the parent tag - so remove that inner_html = html.split(">", 1)[1].rsplit("<", 1)[0] return inner_html.strip() + + +def has_load_extension(): + conn = sqlite3.connect(":memory:") + return hasattr(conn, "enable_load_extension") From 1c29b925d300d1ee17047504473f2517767aa05b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 17:10:52 -0700 Subject: [PATCH 0078/1172] Run tests in serial again Because this didn't fix the issue I'm seeing in #1802 Revert "Run tests in serial, refs #1802" This reverts commit b91e17280c05bbb9cf97432081bdcea8665879f9. --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c8c48ef..e38d5ee9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,8 @@ jobs: pip freeze - name: Run tests run: | - pytest + pytest -n auto -m "not serial" + pytest -m "serial" - name: Check if cog needs to be run run: | cog --check docs/*.rst From 64288d827f7ff97f825e10f714da3f781ecf9345 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 17:40:19 -0700 Subject: [PATCH 0079/1172] Workaround for test failure: RuntimeError: There is no current event loop (#1803) * Remove ensure_eventloop hack * Hack to recover from intermittent RuntimeError calling asyncio.Lock() --- datasette/app.py | 10 +++++++++- tests/test_cli.py | 27 ++++++++++----------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f2a6763a..c6bbdaf0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -231,7 +231,15 @@ class Datasette: self.inspect_data = inspect_data self.immutables = set(immutables or []) self.databases = collections.OrderedDict() - self._refresh_schemas_lock = asyncio.Lock() + try: + self._refresh_schemas_lock = asyncio.Lock() + except RuntimeError as rex: + # Workaround for intermittent test failure, see: + # https://github.com/simonw/datasette/issues/1802 + if "There is no current event loop in thread" in str(rex): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + self._refresh_schemas_lock = asyncio.Lock() self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: diff --git a/tests/test_cli.py b/tests/test_cli.py index d0f6e26c..f0d28037 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -22,13 +22,6 @@ from unittest import mock import urllib -@pytest.fixture -def ensure_eventloop(): - # Workaround for "Event loop is closed" error - if asyncio.get_event_loop().is_closed(): - asyncio.set_event_loop(asyncio.new_event_loop()) - - def test_inspect_cli(app_client): runner = CliRunner() result = runner.invoke(cli, ["inspect", "fixtures.db"]) @@ -72,7 +65,7 @@ def test_serve_with_inspect_file_prepopulates_table_counts_cache(): ), ) def test_spatialite_error_if_attempt_to_open_spatialite( - ensure_eventloop, spatialite_paths, should_suggest_load_extension + spatialite_paths, should_suggest_load_extension ): with mock.patch("datasette.utils.SPATIALITE_PATHS", spatialite_paths): runner = CliRunner() @@ -199,14 +192,14 @@ def test_version(): @pytest.mark.parametrize("invalid_port", ["-1", "0.5", "dog", "65536"]) -def test_serve_invalid_ports(ensure_eventloop, invalid_port): +def test_serve_invalid_ports(invalid_port): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--port", invalid_port]) assert result.exit_code == 2 assert "Invalid value for '-p'" in result.stderr -def test_setting(ensure_eventloop): +def test_setting(): runner = CliRunner() result = runner.invoke( cli, ["--setting", "default_page_size", "5", "--get", "/-/settings.json"] @@ -215,14 +208,14 @@ def test_setting(ensure_eventloop): assert json.loads(result.output)["default_page_size"] == 5 -def test_setting_type_validation(ensure_eventloop): +def test_setting_type_validation(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--setting", "default_page_size", "dog"]) assert result.exit_code == 2 assert '"default_page_size" should be an integer' in result.stderr -def test_config_deprecated(ensure_eventloop): +def test_config_deprecated(): # The --config option should show a deprecation message runner = CliRunner(mix_stderr=False) result = runner.invoke( @@ -233,14 +226,14 @@ def test_config_deprecated(ensure_eventloop): assert "will be deprecated in" in result.stderr -def test_sql_errors_logged_to_stderr(ensure_eventloop): +def test_sql_errors_logged_to_stderr(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--get", "/_memory.json?sql=select+blah"]) assert result.exit_code == 1 assert "sql = 'select blah', params = {}: no such column: blah\n" in result.stderr -def test_serve_create(ensure_eventloop, tmpdir): +def test_serve_create(tmpdir): runner = CliRunner() db_path = tmpdir / "does_not_exist_yet.db" assert not db_path.exists() @@ -258,7 +251,7 @@ def test_serve_create(ensure_eventloop, tmpdir): assert db_path.exists() -def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): +def test_serve_duplicate_database_names(tmpdir): "'datasette db.db nested/db.db' should attach two databases, /db and /db_2" runner = CliRunner() db_1_path = str(tmpdir / "db.db") @@ -273,7 +266,7 @@ def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): assert {db["name"] for db in databases} == {"db", "db_2"} -def test_serve_deduplicate_same_database_path(ensure_eventloop, tmpdir): +def test_serve_deduplicate_same_database_path(tmpdir): "'datasette db.db db.db' should only attach one database, /db" runner = CliRunner() db_path = str(tmpdir / "db.db") @@ -287,7 +280,7 @@ def test_serve_deduplicate_same_database_path(ensure_eventloop, tmpdir): @pytest.mark.parametrize( "filename", ["test-database (1).sqlite", "database (1).sqlite"] ) -def test_weird_database_names(ensure_eventloop, tmpdir, filename): +def test_weird_database_names(tmpdir, filename): # https://github.com/simonw/datasette/issues/1181 runner = CliRunner() db_path = str(tmpdir / filename) From c9d1943aede436fa3413fd49bc56335cbda4ad07 Mon Sep 17 00:00:00 2001 From: Daniel Rech Date: Tue, 6 Sep 2022 02:45:41 +0200 Subject: [PATCH 0080/1172] Fix word break in facets by adding ul.tight-bullets li word-break: break-all (#1794) Thanks, @dmr --- datasette/static/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index af3e14d5..712b9925 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -260,6 +260,7 @@ ul.bullets li { ul.tight-bullets li { list-style-type: disc; margin-bottom: 0; + word-break: break-all; } a.not-underlined { text-decoration: none; From d80775a48d20917633792fdc9525f075d3bc2c7a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 17:44:44 -0700 Subject: [PATCH 0081/1172] Raise error if it's not about loops, refs #1802 --- datasette/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index c6bbdaf0..aeb81687 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -240,6 +240,8 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() + else: + raise self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: From 8430c3bc7dd22b173c1a8c6cd7180e3b31240cd1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 08:59:19 -0700 Subject: [PATCH 0082/1172] table facet_size in metadata, refs #1804 --- datasette/facets.py | 14 +++++++++++--- tests/test_facets.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index b15a758c..e70d42df 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -102,11 +102,19 @@ class Facet: def get_facet_size(self): facet_size = self.ds.setting("default_facet_size") max_returned_rows = self.ds.setting("max_returned_rows") + table_facet_size = None + if self.table: + tables_metadata = self.ds.metadata("tables", database=self.database) or {} + table_metadata = tables_metadata.get(self.table) or {} + if table_metadata: + table_facet_size = table_metadata.get("facet_size") custom_facet_size = self.request.args.get("_facet_size") - if custom_facet_size == "max": - facet_size = max_returned_rows - elif custom_facet_size and custom_facet_size.isdigit(): + if custom_facet_size and custom_facet_size.isdigit(): facet_size = int(custom_facet_size) + elif table_facet_size: + facet_size = table_facet_size + if facet_size == "max": + facet_size = max_returned_rows return min(facet_size, max_returned_rows) async def suggest(self): diff --git a/tests/test_facets.py b/tests/test_facets.py index c28dc43c..cbee23b0 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -581,6 +581,23 @@ async def test_facet_size(): ) data5 = response5.json() assert len(data5["facet_results"]["city"]["results"]) == 20 + # Now try messing with facet_size in the table metadata + ds._metadata_local = { + "databases": { + "test_facet_size": {"tables": {"neighbourhoods": {"facet_size": 6}}} + } + } + response6 = await ds.client.get("/test_facet_size/neighbourhoods.json?_facet=city") + data6 = response6.json() + assert len(data6["facet_results"]["city"]["results"]) == 6 + # Setting it to max bumps it up to 50 again + ds._metadata_local["databases"]["test_facet_size"]["tables"]["neighbourhoods"][ + "facet_size" + ] = "max" + data7 = ( + await ds.client.get("/test_facet_size/neighbourhoods.json?_facet=city") + ).json() + assert len(data7["facet_results"]["city"]["results"]) == 20 def test_other_types_of_facet_in_metadata(): From 303c6c733d95a6133558ec1b468f5bea5827d0d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 11:05:00 -0700 Subject: [PATCH 0083/1172] Fix for incorrectly handled _facet_size=max, refs #1804 --- datasette/facets.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index e70d42df..7fb0c68b 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -109,12 +109,19 @@ class Facet: if table_metadata: table_facet_size = table_metadata.get("facet_size") custom_facet_size = self.request.args.get("_facet_size") - if custom_facet_size and custom_facet_size.isdigit(): - facet_size = int(custom_facet_size) - elif table_facet_size: - facet_size = table_facet_size - if facet_size == "max": - facet_size = max_returned_rows + if custom_facet_size: + if custom_facet_size == "max": + facet_size = max_returned_rows + elif custom_facet_size.isdigit(): + facet_size = int(custom_facet_size) + else: + # Invalid value, ignore it + custom_facet_size = None + if table_facet_size and not custom_facet_size: + if table_facet_size == "max": + facet_size = max_returned_rows + else: + facet_size = table_facet_size return min(facet_size, max_returned_rows) async def suggest(self): From 0a7815d2038255a0834c955066a2a16c01f707b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 11:06:49 -0700 Subject: [PATCH 0084/1172] Documentation for facet_size in metadata, closes #1804 --- docs/facets.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/facets.rst b/docs/facets.rst index 2a2eb039..6c9d99bd 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -129,6 +129,22 @@ You can specify :ref:`array ` or :ref:`date ] } +You can change the default facet size (the number of results shown for each facet) for a table using ``facet_size``: + +.. code-block:: json + + { + "databases": { + "sf-trees": { + "tables": { + "Street_Tree_List": { + "facets": ["qLegalStatus"], + "facet_size": 10 + } + } + } + } + } Suggested facets ---------------- From d0476897e10249bb4867473722270d02491c2c1f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 11:24:30 -0700 Subject: [PATCH 0085/1172] Fixed Sphinx warning about language = None --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 4ef6b768..8965974a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -71,7 +71,7 @@ release = "" # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From ff9c87197dde8b09f9787ee878804cb6842ea5dc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 11:26:21 -0700 Subject: [PATCH 0086/1172] Fixed Sphinx warnings on cli-reference page --- docs/cli-reference.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index f8419d58..4a8465cb 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -14,7 +14,7 @@ Running ``datasette`` without specifying a command runs the default command, ``d import textwrap def help(args): title = "datasette " + " ".join(args) - cog.out("::\n\n") + cog.out("\n::\n\n") result = CliRunner().invoke(cli.cli, args) output = result.output.replace("Usage: cli ", "Usage: datasette ") cog.out(textwrap.indent(output, ' ')) @@ -32,6 +32,7 @@ Running ``datasette --help`` shows a list of all of the available commands. .. [[[cog help(["--help"]) .. ]]] + :: Usage: datasette [OPTIONS] COMMAND [ARGS]... @@ -77,6 +78,7 @@ Once started you can access it at ``http://localhost:8001`` .. [[[cog help(["serve", "--help"]) .. ]]] + :: Usage: datasette serve [OPTIONS] [FILES]... @@ -202,6 +204,7 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam .. [[[cog help(["--help-settings"]) .. ]]] + :: Settings: @@ -258,6 +261,7 @@ Output JSON showing all currently installed plugins, their versions, whether the .. [[[cog help(["plugins", "--help"]) .. ]]] + :: Usage: datasette plugins [OPTIONS] @@ -326,6 +330,7 @@ Would install the `datasette-cluster-map Date: Tue, 6 Sep 2022 16:50:43 -0700 Subject: [PATCH 0087/1172] truncate_cells_html now affects URLs too, refs #1805 --- datasette/utils/__init__.py | 10 ++++++++++ datasette/views/database.py | 11 ++++++++--- datasette/views/table.py | 8 ++++++-- tests/fixtures.py | 9 +++++---- tests/test_api.py | 2 +- tests/test_table_api.py | 11 +++++++---- tests/test_table_html.py | 11 +++++++++++ tests/test_utils.py | 20 ++++++++++++++++++++ 8 files changed, 68 insertions(+), 14 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index bbaa0510..2bdea673 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1167,3 +1167,13 @@ def resolve_routes(routes, path): if match is not None: return match, view return None, None + + +def truncate_url(url, length): + if (not length) or (len(url) <= length): + return url + bits = url.rsplit(".", 1) + if len(bits) == 2 and 1 <= len(bits[1]) <= 4 and "/" not in bits[1]: + rest, ext = bits + return rest[: length - 1 - len(ext)] + "…." + ext + return url[: length - 1] + "…" diff --git a/datasette/views/database.py b/datasette/views/database.py index 77632b9d..fc344245 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -20,6 +20,7 @@ from datasette.utils import ( path_with_format, path_with_removed_args, sqlite3, + truncate_url, InvalidSql, ) from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden @@ -371,6 +372,7 @@ class QueryView(DataView): async def extra_template(): display_rows = [] + truncate_cells = self.ds.setting("truncate_cells_html") for row in results.rows if results else []: display_row = [] for column, value in zip(results.columns, row): @@ -396,9 +398,12 @@ class QueryView(DataView): if value in ("", None): display_value = Markup(" ") elif is_url(str(display_value).strip()): - display_value = Markup( - '{url}'.format( - url=escape(value.strip()) + display_value = markupsafe.Markup( + '{truncated_url}'.format( + url=markupsafe.escape(value.strip()), + truncated_url=markupsafe.escape( + truncate_url(value.strip(), truncate_cells) + ), ) ) elif isinstance(display_value, bytes): diff --git a/datasette/views/table.py b/datasette/views/table.py index 49c30c9c..60c092f9 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -24,6 +24,7 @@ from datasette.utils import ( path_with_removed_args, path_with_replaced_args, to_css_class, + truncate_url, urlsafe_components, value_as_boolean, ) @@ -966,8 +967,11 @@ async def display_columns_and_rows( display_value = markupsafe.Markup(" ") elif is_url(str(value).strip()): display_value = markupsafe.Markup( - '{url}'.format( - url=markupsafe.escape(value.strip()) + '{truncated_url}'.format( + url=markupsafe.escape(value.strip()), + truncated_url=markupsafe.escape( + truncate_url(value.strip(), truncate_cells) + ), ) ) elif column in table_metadata.get("units", {}) and value != "": diff --git a/tests/fixtures.py b/tests/fixtures.py index c145ac78..82d8452e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -598,23 +598,24 @@ CREATE TABLE roadside_attractions ( pk integer primary key, name text, address text, + url text, latitude real, longitude real ); INSERT INTO roadside_attractions VALUES ( - 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", + 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", 37.0167, -122.0024 ); INSERT INTO roadside_attractions VALUES ( - 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", + 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", 37.3184, -121.9511 ); INSERT INTO roadside_attractions VALUES ( - 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", + 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, 37.5793, -122.3442 ); INSERT INTO roadside_attractions VALUES ( - 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", + 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", 37.0414, -122.0725 ); diff --git a/tests/test_api.py b/tests/test_api.py index f6db2f9d..7a2bf91f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -339,7 +339,7 @@ def test_database_page(app_client): }, { "name": "roadside_attractions", - "columns": ["pk", "name", "address", "latitude", "longitude"], + "columns": ["pk", "name", "address", "url", "latitude", "longitude"], "primary_keys": ["pk"], "count": 4, "hidden": False, diff --git a/tests/test_table_api.py b/tests/test_table_api.py index e56a72b5..0db04434 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -615,11 +615,12 @@ def test_table_through(app_client): response = app_client.get( '/fixtures/roadside_attractions.json?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}' ) - assert [ + assert response.json["rows"] == [ [ 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", + None, 37.5793, -122.3442, ], @@ -627,13 +628,15 @@ def test_table_through(app_client): 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", + "https://www.bigfootdiscoveryproject.com/", 37.0414, -122.0725, ], - ] == response.json["rows"] + ] + assert ( - 'where roadside_attraction_characteristics.characteristic_id = "1"' - == response.json["human_description_en"] + response.json["human_description_en"] + == 'where roadside_attraction_characteristics.characteristic_id = "1"' ) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index f3808ea3..8e37468f 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -69,6 +69,17 @@ def test_table_cell_truncation(): td.string for td in table.findAll("td", {"class": "col-neighborhood-b352a7"}) ] + # URLs should be truncated too + response2 = client.get("/fixtures/roadside_attractions") + assert response2.status == 200 + table = Soup(response2.body, "html.parser").find("table") + tds = table.findAll("td", {"class": "col-url"}) + assert [str(td) for td in tds] == [ + 'http…', + 'http…', + '\xa0', + 'http…', + ] def test_add_filter_redirects(app_client): diff --git a/tests/test_utils.py b/tests/test_utils.py index df788767..d71a612d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -626,3 +626,23 @@ def test_tilde_encoding(original, expected): assert actual == expected # And test round-trip assert original == utils.tilde_decode(actual) + + +@pytest.mark.parametrize( + "url,length,expected", + ( + ("https://example.com/", 5, "http…"), + ("https://example.com/foo/bar", 15, "https://exampl…"), + ("https://example.com/foo/bar/baz.jpg", 30, "https://example.com/foo/ba….jpg"), + # Extensions longer than 4 characters are not treated specially: + ("https://example.com/foo/bar/baz.jpeg2", 30, "https://example.com/foo/bar/b…"), + ( + "https://example.com/foo/bar/baz.jpeg2", + None, + "https://example.com/foo/bar/baz.jpeg2", + ), + ), +) +def test_truncate_url(url, length, expected): + actual = utils.truncate_url(url, length) + assert actual == expected From 5aa359b86907d11b3ee601510775a85a90224da8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 16:58:30 -0700 Subject: [PATCH 0088/1172] Apply cell truncation on query page too, refs #1805 --- datasette/views/database.py | 7 ++++++- tests/test_html.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index fc344245..affbc540 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -428,7 +428,12 @@ class QueryView(DataView): "" if len(value) == 1 else "s", ) ) - + else: + display_value = str(value) + if truncate_cells and len(display_value) > truncate_cells: + display_value = ( + display_value[:truncate_cells] + "\u2026" + ) display_row.append(display_value) display_rows.append(display_row) diff --git a/tests/test_html.py b/tests/test_html.py index d6e969ad..bf915247 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -186,6 +186,25 @@ def test_row_page_does_not_truncate(): ] +def test_query_page_truncates(): + with make_app_client(settings={"truncate_cells_html": 5}) as client: + response = client.get( + "/fixtures?" + + urllib.parse.urlencode( + { + "sql": "select 'this is longer than 5' as a, 'https://example.com/' as b" + } + ) + ) + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + tds = table.findAll("td") + assert [str(td) for td in tds] == [ + 'this …', + 'http…', + ] + + @pytest.mark.parametrize( "path,expected_classes", [ From bf8d84af5422606597be893cedd375020cb2b369 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 20:34:59 -0700 Subject: [PATCH 0089/1172] word-wrap: anywhere on links in cells, refs #1805 --- datasette/static/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 712b9925..08b724f6 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -446,6 +446,7 @@ th { } table a:link { text-decoration: none; + word-wrap: anywhere; } .rows-and-columns td:before { display: block; From fb7e70d5e72a951efe4b29ad999d8915c032d021 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Sep 2022 09:19:20 -0700 Subject: [PATCH 0090/1172] Database(is_mutable=) now defaults to True, closes #1808 Refs https://github.com/simonw/datasette-upload-dbs/issues/6 --- datasette/database.py | 3 +-- docs/internals.rst | 9 +++++---- tests/test_internals_database.py | 1 + tests/test_internals_datasette.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index fa558045..44467370 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -28,7 +28,7 @@ AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) class Database: def __init__( - self, ds, path=None, is_mutable=False, is_memory=False, memory_name=None + self, ds, path=None, is_mutable=True, is_memory=False, memory_name=None ): self.name = None self.route = None @@ -39,7 +39,6 @@ class Database: self.memory_name = memory_name if memory_name is not None: self.is_memory = True - self.is_mutable = True self.hash = None self.cached_size = None self._cached_table_counts = None diff --git a/docs/internals.rst b/docs/internals.rst index 20797e98..adeec1d8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -426,12 +426,13 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database` Database( datasette, path="path/to/my-new-database.db", - is_mutable=True, ) ) This will add a mutable database and serve it at ``/my-new-database``. +Use ``is_mutable=False`` to add an immutable database. + ``.add_database()`` returns the Database instance, with its name set as the ``database.name`` attribute. Any time you are working with a newly added database you should use the return value of ``.add_database()``, for example: .. code-block:: python @@ -671,8 +672,8 @@ Instances of the ``Database`` class can be used to execute queries against attac .. _database_constructor: -Database(ds, path=None, is_mutable=False, is_memory=False, memory_name=None) ----------------------------------------------------------------------------- +Database(ds, path=None, is_mutable=True, is_memory=False, memory_name=None) +--------------------------------------------------------------------------- The ``Database()`` constructor can be used by plugins, in conjunction with :ref:`datasette_add_database`, to create and register new databases. @@ -685,7 +686,7 @@ The arguments are as follows: Path to a SQLite database file on disk. ``is_mutable`` - boolean - Set this to ``True`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior. + Set this to ``False`` to cause Datasette to open the file in immutable mode. ``is_memory`` - boolean Use this to create non-shared memory connections. diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 551f67e1..9e81c1d6 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -499,6 +499,7 @@ def test_mtime_ns_is_none_for_memory(app_client): def test_is_mutable(app_client): + assert Database(app_client.ds, is_memory=True).is_mutable is True assert Database(app_client.ds, is_memory=True, is_mutable=True).is_mutable is True assert Database(app_client.ds, is_memory=True, is_mutable=False).is_mutable is False diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 1dc14cab..249920fe 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -58,7 +58,7 @@ async def test_datasette_constructor(): "route": "_memory", "path": None, "size": 0, - "is_mutable": False, + "is_mutable": True, "is_memory": True, "hash": None, } From 610425460b519e9c16d386cb81aa081c9d730ef0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Sep 2022 14:24:26 -0700 Subject: [PATCH 0091/1172] Add --nolock to the README Chrome demo Refs #1744 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1af20129..af95b85e 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ This will start a web server on port 8001 - visit http://localhost:8001/ to acce Use Chrome on OS X? You can run datasette against your browser history like so: - datasette ~/Library/Application\ Support/Google/Chrome/Default/History + datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data: From b40872f5e5ae5dad331c58f75451e2d206565196 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 14 Sep 2022 14:31:54 -0700 Subject: [PATCH 0092/1172] prepare_jinja2_environment(datasette) argument, refs #1809 --- datasette/app.py | 2 +- datasette/hookspecs.py | 2 +- docs/plugin_hooks.rst | 9 +++++++-- tests/plugins/my_plugin.py | 3 ++- tests/test_plugins.py | 5 +++-- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index aeb81687..db686670 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -345,7 +345,7 @@ class Datasette: self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class # pylint: disable=no-member - pm.hook.prepare_jinja2_environment(env=self.jinja_env) + pm.hook.prepare_jinja2_environment(env=self.jinja_env, datasette=self) self._register_renderers() self._permission_checks = collections.deque(maxlen=200) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index a5fb536f..34e19664 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -26,7 +26,7 @@ def prepare_connection(conn, database, datasette): @hookspec -def prepare_jinja2_environment(env): +def prepare_jinja2_environment(env, datasette): """Modify Jinja2 template environment e.g. register custom template tags""" diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 30bd75b7..62ec5c90 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -61,12 +61,15 @@ Examples: `datasette-jellyfish `_, for @@ -85,6 +88,8 @@ You can now use this filter in your custom templates like so:: Table name: {{ table|uppercase }} +Examples: `datasette-edit-templates `_ + .. _plugin_hook_extra_template_vars: extra_template_vars(template, database, table, columns, view_name, request, datasette) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 53613b7d..d49a7a34 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -142,8 +142,9 @@ def extra_template_vars( @hookimpl -def prepare_jinja2_environment(env): +def prepare_jinja2_environment(env, datasette): env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}" + env.filters["to_hello"] = lambda s: datasette._HELLO @hookimpl diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 948a40b8..590d88f6 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -545,11 +545,12 @@ def test_hook_register_output_renderer_can_render(app_client): @pytest.mark.asyncio async def test_hook_prepare_jinja2_environment(app_client): + app_client.ds._HELLO = "HI" template = app_client.ds.jinja_env.from_string( - "Hello there, {{ a|format_numeric }}", {"a": 3412341} + "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}", {"a": 3412341} ) rendered = await app_client.ds.render_template(template) - assert "Hello there, 3,412,341" == rendered + assert "Hello there, 3,412,341, HI" == rendered def test_hook_publish_subcommand(): From 2ebcffe2226ece2a5a86722790d486a480338632 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Sep 2022 12:50:52 -0700 Subject: [PATCH 0093/1172] Bump furo from 2022.6.21 to 2022.9.15 (#1812) Bumps [furo](https://github.com/pradyunsg/furo) from 2022.6.21 to 2022.9.15. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2022.06.21...2022.09.15) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 92fa60d0..afcba1f0 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "furo==2022.6.21", + "furo==2022.9.15", "sphinx-autobuild", "codespell", "blacken-docs", From ddc999ad1296e8c69cffede3e367dda059b8adad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Sep 2022 20:38:15 -0700 Subject: [PATCH 0094/1172] Async support for prepare_jinja2_environment, closes #1809 --- datasette/app.py | 22 ++++++++++++++--- datasette/utils/testing.py | 1 + docs/plugin_hooks.rst | 2 ++ docs/testing_plugins.rst | 30 ++++++++++++++++++++++++ tests/fixtures.py | 1 + tests/plugins/my_plugin.py | 10 ++++++-- tests/plugins/my_plugin_2.py | 6 +++++ tests/test_internals_datasette_client.py | 6 +++-- tests/test_plugins.py | 6 +++-- tests/test_routes.py | 1 + 10 files changed, 76 insertions(+), 9 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index db686670..ea3e7b43 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -208,6 +208,7 @@ class Datasette: crossdb=False, nolock=False, ): + self._startup_invoked = False assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" @@ -344,9 +345,6 @@ class Datasette: self.jinja_env.filters["quote_plus"] = urllib.parse.quote_plus self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class - # pylint: disable=no-member - pm.hook.prepare_jinja2_environment(env=self.jinja_env, datasette=self) - self._register_renderers() self._permission_checks = collections.deque(maxlen=200) self._root_token = secrets.token_hex(32) @@ -389,8 +387,16 @@ class Datasette: return Urls(self) async def invoke_startup(self): + # This must be called for Datasette to be in a usable state + if self._startup_invoked: + return + for hook in pm.hook.prepare_jinja2_environment( + env=self.jinja_env, datasette=self + ): + await await_me_maybe(hook) for hook in pm.hook.startup(datasette=self): await await_me_maybe(hook) + self._startup_invoked = True def sign(self, value, namespace="default"): return URLSafeSerializer(self._secret, namespace).dumps(value) @@ -933,6 +939,8 @@ class Datasette: async def render_template( self, templates, context=None, request=None, view_name=None ): + if not self._startup_invoked: + raise Exception("render_template() called before await ds.invoke_startup()") context = context or {} if isinstance(templates, Template): template = templates @@ -1495,34 +1503,42 @@ class DatasetteClient: return path async def get(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.get(self._fix(path), **kwargs) async def options(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.options(self._fix(path), **kwargs) async def head(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.head(self._fix(path), **kwargs) async def post(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.post(self._fix(path), **kwargs) async def put(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.put(self._fix(path), **kwargs) async def patch(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.patch(self._fix(path), **kwargs) async def delete(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.delete(self._fix(path), **kwargs) async def request(self, method, path, **kwargs): + await self.ds.invoke_startup() avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) async with httpx.AsyncClient(app=self.app) as client: return await client.request( diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index 640c94e6..b28fc575 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -147,6 +147,7 @@ class TestClient: content_type=None, if_none_match=None, ): + await self.ds.invoke_startup() headers = headers or {} if content_type: headers["content-type"] = content_type diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 62ec5c90..f208e727 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -88,6 +88,8 @@ You can now use this filter in your custom templates like so:: Table name: {{ table|uppercase }} +This function can return an awaitable function if it needs to run any async code. + Examples: `datasette-edit-templates `_ .. _plugin_hook_extra_template_vars: diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 992b4b0e..41f50e56 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -52,6 +52,36 @@ Then run the tests using pytest like so:: pytest +.. _testing_plugins_datasette_test_instance: + +Setting up a Datasette test instance +------------------------------------ + +The above example shows the easiest way to start writing tests against a Datasette instance: + +.. code-block:: python + + from datasette.app import Datasette + import pytest + + + @pytest.mark.asyncio + async def test_plugin_is_installed(): + datasette = Datasette(memory=True) + response = await datasette.client.get("/-/plugins.json") + assert response.status_code == 200 + +Creating a ``Datasette()`` instance like this as useful shortcut in tests, but there is one detail you need to be aware of. It's important to ensure that the async method ``.invoke_startup()`` is called on that instance. You can do that like this: + +.. code-block:: python + + datasette = Datasette(memory=True) + await datasette.invoke_startup() + +This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls. + +If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - those method calls ensure that ``.invoke_startup()`` has been called for you. + .. _testing_plugins_pdb: Using pdb for errors thrown inside Datasette diff --git a/tests/fixtures.py b/tests/fixtures.py index 82d8452e..5a875cd2 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -71,6 +71,7 @@ EXPECTED_PLUGINS = [ "handle_exception", "menu_links", "permission_allowed", + "prepare_jinja2_environment", "register_routes", "render_cell", "startup", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index d49a7a34..1a41de38 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -143,8 +143,14 @@ def extra_template_vars( @hookimpl def prepare_jinja2_environment(env, datasette): - env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}" - env.filters["to_hello"] = lambda s: datasette._HELLO + async def select_times_three(s): + db = datasette.get_database() + return (await db.execute("select 3 * ?", [int(s)])).first()[0] + + async def inner(): + env.filters["select_times_three"] = select_times_three + + return inner @hookimpl diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 4df02343..cee80703 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -126,6 +126,12 @@ def permission_allowed(datasette, actor, action): return inner +@hookimpl +def prepare_jinja2_environment(env, datasette): + env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}" + env.filters["to_hello"] = lambda s: datasette._HELLO + + @hookimpl def startup(datasette): async def inner(): diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py index 8c5b5bd3..497bf475 100644 --- a/tests/test_internals_datasette_client.py +++ b/tests/test_internals_datasette_client.py @@ -1,10 +1,12 @@ from .fixtures import app_client import httpx import pytest +import pytest_asyncio -@pytest.fixture -def datasette(app_client): +@pytest_asyncio.fixture +async def datasette(app_client): + await app_client.ds.invoke_startup() return app_client.ds diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 590d88f6..0ae3abf3 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -546,11 +546,13 @@ def test_hook_register_output_renderer_can_render(app_client): @pytest.mark.asyncio async def test_hook_prepare_jinja2_environment(app_client): app_client.ds._HELLO = "HI" + await app_client.ds.invoke_startup() template = app_client.ds.jinja_env.from_string( - "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}", {"a": 3412341} + "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}, {{ b|select_times_three }}", + {"a": 3412341, "b": 5}, ) rendered = await app_client.ds.render_template(template) - assert "Hello there, 3,412,341, HI" == rendered + assert "Hello there, 3,412,341, HI, 15" == rendered def test_hook_publish_subcommand(): diff --git a/tests/test_routes.py b/tests/test_routes.py index 5ae55d21..d467abe1 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -59,6 +59,7 @@ def test_routes(routes, path, expected_class, expected_matches): @pytest_asyncio.fixture async def ds_with_route(): ds = Datasette() + await ds.invoke_startup() ds.remove_database("_memory") db = Database(ds, is_memory=True, memory_name="route-name-db") ds.add_database(db, name="original-name", route="custom-route-name") From df851c117db031dec50dd4ef1ca34745920ac77a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Sep 2022 16:46:39 -0700 Subject: [PATCH 0095/1172] Validate settings.json keys on startup, closes #1816 Refs #1814 --- datasette/app.py | 4 ++++ tests/test_config_dir.py | 20 ++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index ea3e7b43..8873ce28 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -292,6 +292,10 @@ class Datasette: raise StartupError("config.json should be renamed to settings.json") if config_dir and (config_dir / "settings.json").exists() and not settings: settings = json.loads((config_dir / "settings.json").read_text()) + # Validate those settings + for key in settings: + if key not in DEFAULT_SETTINGS: + raise StartupError("Invalid setting '{key}' in settings.json") self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index fe927c42..e365515b 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -5,6 +5,7 @@ import pytest from datasette.app import Datasette from datasette.cli import cli from datasette.utils.sqlite import sqlite3 +from datasette.utils import StartupError from .fixtures import TestClient as _TestClient from click.testing import CliRunner @@ -27,9 +28,8 @@ body { margin-top: 3em} @pytest.fixture(scope="session") -def config_dir_client(tmp_path_factory): +def config_dir(tmp_path_factory): config_dir = tmp_path_factory.mktemp("config-dir") - plugins_dir = config_dir / "plugins" plugins_dir.mkdir() (plugins_dir / "hooray.py").write_text(PLUGIN, "utf-8") @@ -77,7 +77,23 @@ def config_dir_client(tmp_path_factory): ), "utf-8", ) + return config_dir + +def test_invalid_settings(config_dir): + previous = (config_dir / "settings.json").read_text("utf-8") + (config_dir / "settings.json").write_text( + json.dumps({"invalid": "invalid-setting"}), "utf-8" + ) + try: + with pytest.raises(StartupError): + ds = Datasette([], config_dir=config_dir) + finally: + (config_dir / "settings.json").write_text(previous, "utf-8") + + +@pytest.fixture(scope="session") +def config_dir_client(config_dir): ds = Datasette([], config_dir=config_dir) yield _TestClient(ds) From cb1e093fd361b758120aefc1a444df02462389a3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Sep 2022 18:15:40 -0700 Subject: [PATCH 0096/1172] Fixed error message, closes #1816 --- datasette/app.py | 4 +++- tests/test_config_dir.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8873ce28..03d1dacc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -295,7 +295,9 @@ class Datasette: # Validate those settings for key in settings: if key not in DEFAULT_SETTINGS: - raise StartupError("Invalid setting '{key}' in settings.json") + raise StartupError( + "Invalid setting '{}' in settings.json".format(key) + ) self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index e365515b..f5ecf0d6 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -86,8 +86,9 @@ def test_invalid_settings(config_dir): json.dumps({"invalid": "invalid-setting"}), "utf-8" ) try: - with pytest.raises(StartupError): + with pytest.raises(StartupError) as ex: ds = Datasette([], config_dir=config_dir) + assert ex.value.args[0] == "Invalid setting 'invalid' in settings.json" finally: (config_dir / "settings.json").write_text(previous, "utf-8") From 212137a90b4291db9605e039f198564dae59c5d0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 14:14:25 -0700 Subject: [PATCH 0097/1172] Release 0.63a0 Refs #1786, #1787, #1789, #1794, #1800, #1804, #1805, #1808, #1809, #1816 --- datasette/version.py | 2 +- docs/changelog.rst | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 0453346c..e5ad585f 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.62" +__version__ = "0.63a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index f9dcc980..bd93f4cb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,23 @@ Changelog ========= +.. _v0_63a0: + +0.63a0 (2022-09-26) +------------------- + +- The :ref:`plugin_hook_prepare_jinja2_environment` plugin hook now accepts an optional ``datasette`` argument. Hook implementations can also now return an ``async`` function which will be awaited automatically. (:issue:`1809`) +- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 `__) +- New tutorial: `Cleaning data with sqlite-utils and Datasette `__. +- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) +- ``truncate_cells_html`` setting now also affects long URLs in columns. (:issue:`1805`) +- ``Database(is_mutable=)`` now defaults to ``True``. (:issue:`1808`) +- Non-JavaScript textarea now increases height to fit the SQL query. (:issue:`1786`) +- More detailed command descriptions on the :ref:`CLI reference ` page. (:issue:`1787`) +- Datasette no longer enforces upper bounds on its depenedencies. (:issue:`1800`) +- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 `__) +- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) + .. _v0_62: 0.62 (2022-08-14) From 5f9f567acbc58c9fcd88af440e68034510fb5d2b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 16:06:01 -0700 Subject: [PATCH 0098/1172] Show SQL query when reporting time limit error, closes #1819 --- datasette/database.py | 5 ++++- datasette/views/base.py | 21 +++++++++++++-------- tests/test_api.py | 12 +++++++++++- tests/test_html.py | 10 +++++++--- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 44467370..46094bd7 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -476,7 +476,10 @@ class WriteTask: class QueryInterrupted(Exception): - pass + def __init__(self, e, sql, params): + self.e = e + self.sql = sql + self.params = params class MultipleValues(Exception): diff --git a/datasette/views/base.py b/datasette/views/base.py index 221e1882..67aa3a42 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -1,10 +1,12 @@ import asyncio import csv import hashlib -import re import sys +import textwrap import time import urllib +from markupsafe import escape + import pint @@ -24,11 +26,9 @@ from datasette.utils import ( path_with_removed_args, path_with_format, sqlite3, - HASH_LENGTH, ) from datasette.utils.asgi import ( AsgiStream, - Forbidden, NotFound, Response, BadRequest, @@ -371,13 +371,18 @@ class DataView(BaseView): ) = response_or_template_contexts else: data, extra_template_data, templates = response_or_template_contexts - except QueryInterrupted: + except QueryInterrupted as ex: raise DatasetteError( - """ - SQL query took too long. The time limit is controlled by the + textwrap.dedent( + """ +

SQL query took too long. The time limit is controlled by the sql_time_limit_ms - configuration option. - """, + configuration option.

+ 
{}
+ """.format( + escape(ex.sql) + ) + ).strip(), title="SQL Interrupted", status=400, message_is_html=True, diff --git a/tests/test_api.py b/tests/test_api.py index 7a2bf91f..ad74d16e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -656,7 +656,17 @@ def test_custom_sql(app_client): def test_sql_time_limit(app_client_shorter_time_limit): response = app_client_shorter_time_limit.get("/fixtures.json?sql=select+sleep(0.5)") assert 400 == response.status - assert "SQL Interrupted" == response.json["title"] + assert response.json == { + "ok": False, + "error": ( + "

SQL query took too long. The time limit is controlled by the\n" + 'sql_time_limit_ms\n' + "configuration option.

\n" + "
select sleep(0.5)
" + ), + "status": 400, + "title": "SQL Interrupted", + } def test_custom_sql_time_limit(app_client): diff --git a/tests/test_html.py b/tests/test_html.py index bf915247..a99b0b6c 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -168,10 +168,14 @@ def test_disallowed_custom_sql_pragma(app_client): def test_sql_time_limit(app_client_shorter_time_limit): response = app_client_shorter_time_limit.get("/fixtures?sql=select+sleep(0.5)") assert 400 == response.status - expected_html_fragment = """ + expected_html_fragments = [ + """ sql_time_limit_ms - """.strip() - assert expected_html_fragment in response.text + """.strip(), + "
select sleep(0.5)
", + ] + for expected_html_fragment in expected_html_fragments: + assert expected_html_fragment in response.text def test_row_page_does_not_truncate(): From 7fb4ea4e39a15e1f7d3202949794d98af1cfa272 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Sep 2022 21:06:40 -0700 Subject: [PATCH 0099/1172] Update note about render_cell signature, refs #1826 --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f208e727..c9cab8ab 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -9,7 +9,7 @@ Each plugin can implement one or more hooks using the ``@hookimpl`` decorator ag When you implement a plugin hook you can accept any or all of the parameters that are documented as being passed to that hook. -For example, you can implement the ``render_cell`` plugin hook like this even though the full documented hook signature is ``render_cell(value, column, table, database, datasette)``: +For example, you can implement the ``render_cell`` plugin hook like this even though the full documented hook signature is ``render_cell(row, value, column, table, database, datasette)``: .. code-block:: python From 984b1df12cf19a6731889fc0665bb5f622e07b7c Mon Sep 17 00:00:00 2001 From: Adam Simpson Date: Wed, 28 Sep 2022 00:21:36 -0400 Subject: [PATCH 0100/1172] Add documentation for serving via OpenRC (#1825) * Add documentation for serving via OpenRC --- docs/deploying.rst | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index d4ad8836..c8552758 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -74,18 +74,30 @@ Once the service has started you can confirm that Datasette is running on port 8 curl 127.0.0.1:8000/-/versions.json # Should output JSON showing the installed version -Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx``. +Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx`` - see :ref:`deploying_proxy`. -Ubuntu offer `a tutorial on installing nginx `__. Once it is installed you can add configuration to proxy traffic through to Datasette that looks like this:: +.. _deploying_openrc: - server { - server_name mysubdomain.myhost.net; +Running Datasette using OpenRC +=============================== +OpenRC is the service manager on non-systemd Linux distributions like `Alpine Linux `__ and `Gentoo `__. - location / { - proxy_pass http://127.0.0.1:8000/; - proxy_set_header Host $host; - } - } +Create an init script at ``/etc/init.d/datasette`` with the following contents: + +.. code-block:: sh + + #!/sbin/openrc-run + + name="datasette" + command="datasette" + command_args="serve -h 0.0.0.0 /path/to/db.db" + command_background=true + pidfile="/run/${RC_SVCNAME}.pid" + +You then need to configure the service to run at boot and start it:: + + rc-update add datasette + rc-service datasette start .. _deploying_buildpacks: From 34defdc10aa293294ca01cfab70780755447e1d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 28 Sep 2022 17:39:36 -0700 Subject: [PATCH 0101/1172] Browse the plugins directory --- docs/writing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 01ee8c90..a3fc88ec 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -234,7 +234,7 @@ To avoid accidentally conflicting with a database file that may be loaded into D - ``/-/upload-excel`` -Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `datasette-plugin topic `__ on GitHub. +Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `plugins directory `. If your plugin includes functionality that relates to a specific database you could also register a URL route like this: From c92c4318e9892101f75fa158410c0a12c1d80b6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Sep 2022 10:55:40 -0700 Subject: [PATCH 0102/1172] Bump furo from 2022.9.15 to 2022.9.29 (#1827) Bumps [furo](https://github.com/pradyunsg/furo) from 2022.9.15 to 2022.9.29. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2022.09.15...2022.09.29) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index afcba1f0..fe258adb 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "furo==2022.9.15", + "furo==2022.9.29", "sphinx-autobuild", "codespell", "blacken-docs", From 883e326dd6ef95f854f7750ef2d4b0e17082fa96 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 2 Oct 2022 14:26:16 -0700 Subject: [PATCH 0103/1172] Drop word-wrap: anywhere, refs #1828, #1805 --- datasette/static/app.css | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 08b724f6..712b9925 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -446,7 +446,6 @@ th { } table a:link { text-decoration: none; - word-wrap: anywhere; } .rows-and-columns td:before { display: block; From 4218c9cd742b79b1e3cb80878e42b7e39d16ded2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Oct 2022 11:45:36 -0700 Subject: [PATCH 0104/1172] reST markup fix --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index c9cab8ab..832a76b0 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -268,7 +268,7 @@ you have one: def extra_js_urls(): return ["/-/static-plugins/your-plugin/app.js"] -Note that `your-plugin` here should be the hyphenated plugin name - the name that is displayed in the list on the `/-/plugins` debug page. +Note that ``your-plugin`` here should be the hyphenated plugin name - the name that is displayed in the list on the ``/-/plugins`` debug page. If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details. From b6ba117b7978b58b40e3c3c2b723b92c3010ed53 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Oct 2022 18:25:52 -0700 Subject: [PATCH 0105/1172] Clarify request or None for two hooks --- docs/plugin_hooks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 832a76b0..b61f953a 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1281,7 +1281,7 @@ menu_links(datasette, actor, request) ``actor`` - dictionary or None The currently authenticated :ref:`actor `. -``request`` - :ref:`internals_request` +``request`` - :ref:`internals_request` or None The current HTTP request. This can be ``None`` if the request object is not available. This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon. @@ -1330,7 +1330,7 @@ table_actions(datasette, actor, database, table, request) ``table`` - string The name of the table. -``request`` - :ref:`internals_request` +``request`` - :ref:`internals_request` or None The current HTTP request. This can be ``None`` if the request object is not available. This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items. From bbf33a763537a1d913180b22bd3b5fe4a5e5b252 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Oct 2022 21:32:11 -0700 Subject: [PATCH 0106/1172] Test for bool(results), closes #1832 --- tests/test_internals_database.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 9e81c1d6..4e33beed 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -30,6 +30,14 @@ async def test_results_first(db): assert isinstance(row, sqlite3.Row) +@pytest.mark.asyncio +@pytest.mark.parametrize("expected", (True, False)) +async def test_results_bool(db, expected): + where = "" if expected else "where pk = 0" + results = await db.execute("select * from facetable {}".format(where)) + assert bool(results) is expected + + @pytest.mark.parametrize( "query,expected", [ From eff112498ecc499323c26612d707908831446d25 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Thu, 6 Oct 2022 16:06:06 -0400 Subject: [PATCH 0107/1172] Useuse inspect data for hash and file size on startup Thanks, @fgregg Closes #1834 --- datasette/database.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 46094bd7..d75bd70c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -48,9 +48,13 @@ class Database: self._read_connection = None self._write_connection = None if not self.is_mutable and not self.is_memory: - p = Path(path) - self.hash = inspect_hash(p) - self.cached_size = p.stat().st_size + if self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.hash = self.ds.inspect_data[self.name]["hash"] + self.cached_size = self.ds.inspect_data[self.name]["size"] + else: + p = Path(path) + self.hash = inspect_hash(p) + self.cached_size = p.stat().st_size @property def cached_table_counts(self): From b7fec7f9020b79c1fe60cc5a2def86b50eeb5af9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 7 Oct 2022 16:03:09 -0700 Subject: [PATCH 0108/1172] .sqlite/.sqlite3 extensions for config directory mode Closes #1646 --- datasette/app.py | 5 ++++- docs/settings.rst | 2 +- tests/test_config_dir.py | 11 +++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 03d1dacc..32a911c2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -217,7 +217,10 @@ class Datasette: self._secret = secret or secrets.token_hex(32) self.files = tuple(files or []) + tuple(immutables or []) if config_dir: - self.files += tuple([str(p) for p in config_dir.glob("*.db")]) + db_files = [] + for ext in ("db", "sqlite", "sqlite3"): + db_files.extend(config_dir.glob("*.{}".format(ext))) + self.files += tuple(str(f) for f in db_files) if ( config_dir and (config_dir / "inspect-data.json").exists() diff --git a/docs/settings.rst b/docs/settings.rst index 8437fb04..a6d50543 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -46,7 +46,7 @@ Datasette will detect the files in that directory and automatically configure it The files that can be included in this directory are as follows. All are optional. -* ``*.db`` - SQLite database files that will be served by Datasette +* ``*.db`` (or ``*.sqlite3`` or ``*.sqlite``) - SQLite database files that will be served by Datasette * ``metadata.json`` - :ref:`metadata` for those databases - ``metadata.yaml`` or ``metadata.yml`` can be used as well * ``inspect-data.json`` - the result of running ``datasette inspect *.db --inspect-file=inspect-data.json`` from the configuration directory - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running * ``settings.json`` - settings that would normally be passed using ``--setting`` - here they should be stored as a JSON object of key/value pairs diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index f5ecf0d6..c2af3836 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -49,7 +49,7 @@ def config_dir(tmp_path_factory): (config_dir / "metadata.json").write_text(json.dumps(METADATA), "utf-8") (config_dir / "settings.json").write_text(json.dumps(SETTINGS), "utf-8") - for dbname in ("demo.db", "immutable.db"): + for dbname in ("demo.db", "immutable.db", "j.sqlite3", "k.sqlite"): db = sqlite3.connect(str(config_dir / dbname)) db.executescript( """ @@ -151,12 +151,11 @@ def test_databases(config_dir_client): response = config_dir_client.get("/-/databases.json") assert 200 == response.status databases = response.json - assert 2 == len(databases) + assert 4 == len(databases) databases.sort(key=lambda d: d["name"]) - assert "demo" == databases[0]["name"] - assert databases[0]["is_mutable"] - assert "immutable" == databases[1]["name"] - assert not databases[1]["is_mutable"] + for db, expected_name in zip(databases, ("demo", "immutable", "j", "k")): + assert expected_name == db["name"] + assert db["is_mutable"] == (expected_name != "immutable") @pytest.mark.parametrize("filename", ("metadata.yml", "metadata.yaml")) From 1a5e5f2aa951e5bd731067a49819efba68fbe8ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Oct 2022 14:42:52 -0700 Subject: [PATCH 0109/1172] Refactor breadcrumbs to respect permissions, refs #1831 --- datasette/app.py | 40 ++++++++++++++++++++++ datasette/templates/_crumbs.html | 15 ++++++++ datasette/templates/base.html | 4 +-- datasette/templates/database.html | 9 ----- datasette/templates/error.html | 7 ---- datasette/templates/logout.html | 7 ---- datasette/templates/permissions_debug.html | 7 ---- datasette/templates/query.html | 8 ++--- datasette/templates/row.html | 9 ++--- datasette/templates/show_json.html | 7 ---- datasette/templates/table.html | 8 ++--- tests/test_permissions.py | 1 + tests/test_plugins.py | 2 +- 13 files changed, 65 insertions(+), 59 deletions(-) create mode 100644 datasette/templates/_crumbs.html diff --git a/datasette/app.py b/datasette/app.py index 32a911c2..5fa4955c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -631,6 +631,44 @@ class Datasette: else: return [] + async def _crumb_items(self, request, table=None, database=None): + crumbs = [] + # Top-level link + if await self.permission_allowed( + actor=request.actor, action="view-instance", default=True + ): + crumbs.append({"href": self.urls.instance(), "label": "home"}) + # Database link + if database: + if await self.permission_allowed( + actor=request.actor, + action="view-database", + resource=database, + default=True, + ): + crumbs.append( + { + "href": self.urls.database(database), + "label": database, + } + ) + # Table link + if table: + assert database, "table= requires database=" + if await self.permission_allowed( + actor=request.actor, + action="view-table", + resource=(database, table), + default=True, + ): + crumbs.append( + { + "href": self.urls.table(database, table), + "label": table, + } + ) + return crumbs + async def permission_allowed(self, actor, action, resource=None, default=False): """Check permissions using the permissions_allowed plugin hook""" result = None @@ -1009,6 +1047,8 @@ class Datasette: template_context = { **context, **{ + "request": request, + "crumb_items": self._crumb_items, "urls": self.urls, "actor": request.actor if request else None, "menu_links": menu_links, diff --git a/datasette/templates/_crumbs.html b/datasette/templates/_crumbs.html new file mode 100644 index 00000000..bd1ff0da --- /dev/null +++ b/datasette/templates/_crumbs.html @@ -0,0 +1,15 @@ +{% macro nav(request, database=None, table=None) -%} +{% if crumb_items is defined %} + {% set items=crumb_items(request=request, database=database, table=table) %} + {% if items %} +

+ {% for item in items %} + {{ item.label }} + {% if not loop.last %} + / + {% endif %} + {% endfor %} +

+ {% endif %} +{% endif %} +{%- endmacro %} diff --git a/datasette/templates/base.html b/datasette/templates/base.html index c3a71acb..87c939ac 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -1,4 +1,4 @@ - +{% import "_crumbs.html" as crumbs with context %} {% block title %}{% endblock %} @@ -17,7 +17,7 @@
-
+ + + + +{% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} +{% include "_execute_write_analysis_scripts.html" %} + + + +{% endblock %} diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index fa4859b1..a8c9a391 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -205,32 +205,32 @@

Queries

-
-

- - - {% if filters.is_write %}{% endif %} - {% if filters.is_private %}{% endif %} - {% if filters.source %}{% endif %} - {% if filters.owner_id %}{% endif %} - -

-
- - - {% if queries %} +
+

+ + + {% if filters.is_write %}{% endif %} + {% if filters.is_private %}{% endif %} + {% if filters.source %}{% endif %} + {% if filters.owner_id %}{% endif %} + +

+
+ + +
diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index f30a30bc..9efe3f81 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -436,6 +436,35 @@ async def _query_create_form_context( } +async def _query_edit_form_context( + datasette, + request, + db, + existing: StoredQuery, + *, + sql=None, + title=None, + description=None, + is_private=None, +): + sql = existing.sql if sql is None else sql + title = existing.title if title is None else title + description = existing.description if description is None else description + is_private = existing.is_private if is_private is None else is_private + analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor) + return { + "database": db.name, + "database_color": db.color, + "name": existing.name, + "sql": sql, + "title": title or "", + "description": description or "", + "is_private": is_private, + "query_url": datasette.urls.table(db.name, existing.name), + **analysis_data, + } + + async def _inserted_row_url(datasette, db, analysis, cursor): if cursor.rowcount != 1: return None diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 8c4e849e..2753f876 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -18,6 +18,7 @@ from .query_helpers import ( _query_create_analysis_data, _query_create_form_context, _query_create_form_error_message, + _query_edit_form_context, _query_list_limit, ) @@ -464,13 +465,164 @@ class QueryUpdateView(BaseView): return Response.json({"ok": True}) -class QueryDeleteView(BaseView): - name = "query-delete" +class QueryEditView(BaseView): + name = "query-edit" + has_json_alternate = False - async def post(self, request): + async def _load(self, request): db = await self.ds.resolve_database(request) query_name = tilde_decode(request.url_vars["query"]) existing = await self.ds.get_query(db.name, query_name) + return db, query_name, existing + + async def _render_form( + self, + request, + db, + existing, + *, + sql=None, + title=None, + description=None, + is_private=None, + status=200, + ): + response = await self.render( + ["query_edit.html"], + request, + await _query_edit_form_context( + self.ds, + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + ), + ) + response.status = status + return response + + async def get(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + await self.ds.ensure_permission( + action="update-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ) + if existing.is_trusted: + return _error(["Trusted queries cannot be edited"], 403) + return await self._render_form(request, db, existing) + + async def post(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + if not await self.ds.allowed( + action="update-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ): + return _error(["Permission denied: need update-query"], 403) + if existing.is_trusted: + return _error(["Trusted queries cannot be edited"], 403) + + data, _ = await _json_or_form_payload(request) + if not isinstance(data, dict): + return _error(["Invalid form submission"], 400) + sql = data.get("sql") + sql = existing.sql if sql is None else sql.strip() + title = data.get("title") or "" + description = data.get("description") or "" + is_private = _as_bool(data.get("is_private")) + + update = { + "title": title, + "description": description, + "is_private": is_private, + } + if sql != existing.sql: + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + self.ds.add_message( + request, + "Permission denied: need execute-sql to change the SQL", + self.ds.ERROR, + ) + return await self._render_form( + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + status=403, + ) + update["sql"] = sql + + try: + update_kwargs = await _prepare_query_update( + self.ds, request, db, existing, update + ) + except QueryValidationError as ex: + self.ds.add_message(request, ex.message, self.ds.ERROR) + return await self._render_form( + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + status=ex.status, + ) + + await self.ds.update_query(db.name, query_name, **update_kwargs) + self.ds.add_message(request, "Query updated", self.ds.INFO) + return Response.redirect( + self.ds.urls.path(self.ds.urls.table(db.name, query_name)) + ) + + +class QueryDeleteView(BaseView): + name = "query-delete" + has_json_alternate = False + + async def _load(self, request): + db = await self.ds.resolve_database(request) + query_name = tilde_decode(request.url_vars["query"]) + existing = await self.ds.get_query(db.name, query_name) + return db, query_name, existing + + async def get(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + await self.ds.ensure_permission( + action="delete-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ) + return await self.render( + ["query_delete.html"], + request, + { + "database": db.name, + "database_color": db.color, + "query": stored_query_to_dict(existing), + "query_url": self.ds.urls.table(db.name, query_name), + }, + ) + + async def post(self, request): + db, query_name, existing = await self._load(request) if existing is None: return _error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( @@ -479,5 +631,14 @@ class QueryDeleteView(BaseView): actor=request.actor, ): return _error(["Permission denied: need delete-query"], 403) + + data, is_json = await _json_or_form_payload(request) await self.ds.remove_query(db.name, query_name) - return Response.json({"ok": True}) + if is_json: + return Response.json({"ok": True}) + self.ds.add_message( + request, + "Query “{}” deleted".format(existing.title or query_name), + self.ds.INFO, + ) + return Response.redirect(self.ds.urls.path(self.ds.urls.database(db.name))) diff --git a/docs/changelog.rst b/docs/changelog.rst index d5f8fa14..75e4f3e8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v1_0_a33: + +1.0a33 (unreleased) +------------------- + +- Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) + .. _v1_0_a32: 1.0a32 (2026-05-31) diff --git a/docs/plugins.rst b/docs/plugins.rst index d578e9e2..c2eb282a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -271,6 +271,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "register_token_handler" ] }, + { + "name": "datasette.default_query_actions", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "query_actions" + ] + }, { "name": "datasette.events", "static": false, diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index c0ba67f0..371348fb 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -142,6 +142,15 @@ Datasette stores both configured queries and user-created queries in the ``queri Stored queries created by users default to private. Private stored queries can only be viewed, updated or deleted by the actor that created them. Broad ``view-query``, ``update-query`` or ``delete-query`` permission grants still do not allow other actors to access another actor's private stored queries. +Editing and deleting stored queries ++++++++++++++++++++++++++++++++++++ + +The page for a stored query includes a "Query actions" menu with **Edit this query** and **Delete this query** links for actors who have permission to use them. + +The owner of a stored query can always edit and delete it. For queries that are not private, any actor granted the ``update-query`` or ``delete-query`` permission can edit or delete the query, even if they did not create it. Private queries can only be edited or deleted by their owner, regardless of any broad permission grants. + +Editing a query lets you change its title, description, SQL and whether it is private. Changing the SQL also requires the ``execute-sql`` permission (and the relevant write permissions for writable queries). The same operations are available through the JSON API by sending a ``POST`` to ``///-/update`` or ``///-/delete``. Trusted stored queries cannot be edited or deleted through the web interface or the JSON API. + Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. SQL functions are allowed and are not separately restricted by Datasette permissions. .. _trusted_stored_queries: diff --git a/tests/test_docs.py b/tests/test_docs.py index 9cf39f41..51caf595 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -77,6 +77,7 @@ def documented_views(): "QueryCreateAnalyzeView", "QueryDeleteView", "QueryDefinitionView", + "QueryEditView", "QueryListView", "QueryParametersView", "QueryStoreView", diff --git a/tests/test_queries.py b/tests/test_queries.py index 25e423d4..6e9bcbdb 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -3,6 +3,7 @@ import re from html import unescape import pytest +from bs4 import BeautifulSoup as Soup from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource @@ -712,6 +713,10 @@ async def test_query_list_search_filter_and_html(): "/data/-/queries?is_private=1", actor={"id": "root"}, ) + no_results_response = await ds.client.get( + "/data/-/queries?q=nope", + actor={"id": "root"}, + ) assert html_response.status_code == 200 assert "Demo query 02" in html_response.text @@ -799,6 +804,13 @@ async def test_query_list_search_filter_and_html(): 'Not private0' not in filtered_private_response.text ) + assert no_results_response.status_code == 200 + assert "No queries found." in no_results_response.text + assert 'class="query-list-filters core"' not in no_results_response.text + assert 'id="query-search"' not in no_results_response.text + assert 'class="query-list-facets"' not in no_results_response.text + assert "

Mode

" not in no_results_response.text + assert "

Visibility

" not in no_results_response.text @pytest.mark.asyncio @@ -1114,6 +1126,227 @@ async def test_query_update_api_rejects_trusted_queries_but_internal_update_allo assert query.title == "Internal" +async def _make_ds_with_user_query(name, *, is_private=False, owner_id="owner"): + ds = Datasette(memory=True, settings={"default_allow_sql": True}) + db = ds.add_memory_database(name, name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "saved", + "select * from dogs", + title="Saved query", + description="A saved query", + source="user", + owner_id=owner_id, + is_private=is_private, + ) + return ds + + +@pytest.mark.asyncio +async def test_query_edit_form_renders_and_updates_for_owner(): + ds = await _make_ds_with_user_query("query_edit_owner") + actor = {"id": "owner"} + + # GET renders the form pre-filled with existing values + get_response = await ds.client.get("/data/saved/-/edit", actor=actor) + assert get_response.status_code == 200 + assert 'value="Saved query"' in get_response.text + assert ">A saved query" in get_response.text + assert "select * from dogs" in get_response.text + # URL slug is shown but not editable + assert 'name="name"' not in get_response.text + + # POST updates the query and redirects back to the query page + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=actor, + data={ + "title": "Updated title", + "description": "Updated description", + "sql": "select id from dogs", + "is_private": "1", + }, + ) + assert post_response.status_code == 302 + assert post_response.headers["location"] == "/data/saved" + + query = await ds.get_query("data", "saved") + assert query.title == "Updated title" + assert query.description == "Updated description" + assert query.sql == "select id from dogs" + assert query.is_private is True + + +@pytest.mark.asyncio +async def test_query_edit_metadata_only_does_not_require_execute_sql(): + # An owner who can no longer execute SQL can still edit title/description + ds = await _make_ds_with_user_query("query_edit_metadata_only") + actor = {"id": "owner"} + + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=actor, + data={ + "title": "Renamed", + "description": "A saved query", + "sql": "select * from dogs", + }, + ) + assert post_response.status_code == 302 + query = await ds.get_query("data", "saved") + assert query.title == "Renamed" + + +@pytest.mark.asyncio +async def test_private_query_edit_delete_restricted_to_owner(): + ds = await _make_ds_with_user_query( + "query_edit_private", is_private=True, owner_id="owner" + ) + + # A different actor cannot view, edit or delete the private query + other = {"id": "intruder"} + assert (await ds.client.get("/data/saved/-/edit", actor=other)).status_code == 403 + assert (await ds.client.get("/data/saved/-/delete", actor=other)).status_code == 403 + delete_attempt = await ds.client.post( + "/data/saved/-/delete", + actor=other, + data={}, + ) + assert delete_attempt.status_code == 403 + assert await ds.get_query("data", "saved") is not None + + # The owner can edit and delete + owner = {"id": "owner"} + assert (await ds.client.get("/data/saved/-/edit", actor=owner)).status_code == 200 + + +@pytest.mark.asyncio +async def test_non_private_query_editable_by_permitted_non_owner(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "execute-sql": {"id": "editor"}, + "update-query": {"id": "editor"}, + "delete-query": {"id": "editor"}, + } + } + } + }, + ) + db = ds.add_memory_database("query_non_private_editor", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "saved", + "select * from dogs", + title="Shared", + source="user", + owner_id="owner", + is_private=False, + ) + + editor = {"id": "editor"} + # Editor (not the owner) can edit because the query is not private + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=editor, + data={ + "title": "Edited by editor", + "description": "", + "sql": "select * from dogs", + }, + ) + assert post_response.status_code == 302 + query = await ds.get_query("data", "saved") + assert query.title == "Edited by editor" + + # Editor can also delete it + delete_response = await ds.client.post( + "/data/saved/-/delete", + actor=editor, + data={}, + ) + assert delete_response.status_code == 302 + assert await ds.get_query("data", "saved") is None + + +@pytest.mark.asyncio +async def test_query_delete_confirmation_and_form_delete(): + ds = await _make_ds_with_user_query("query_delete_form") + actor = {"id": "owner"} + + get_response = await ds.client.get("/data/saved/-/delete", actor=actor) + assert get_response.status_code == 200 + assert "Are you sure" in get_response.text + assert "select * from dogs" in get_response.text + soup = Soup(get_response.text, "html.parser") + form = soup.select_one("form.query-delete-form") + assert form is not None + assert "core" in form["class"] + assert form.select_one('input[type="submit"][value="Delete query"]') is not None + + post_response = await ds.client.post( + "/data/saved/-/delete", + actor=actor, + data={}, + ) + assert post_response.status_code == 302 + assert post_response.headers["location"] == "/data" + assert await ds.get_query("data", "saved") is None + + +@pytest.mark.asyncio +async def test_query_action_menu_shows_edit_and_delete_for_owner(): + ds = await _make_ds_with_user_query("query_action_menu") + + owner_response = await ds.client.get("/data/saved", actor={"id": "owner"}) + assert owner_response.status_code == 200 + assert "/data/saved/-/edit" in owner_response.text + assert "/data/saved/-/delete" in owner_response.text + + # A different actor (the query is public) cannot edit/delete by default + other_response = await ds.client.get("/data/saved", actor={"id": "stranger"}) + assert other_response.status_code == 200 + assert "/data/saved/-/edit" not in other_response.text + assert "/data/saved/-/delete" not in other_response.text + + +@pytest.mark.asyncio +async def test_query_edit_rejected_for_trusted_query(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "execute-sql": {"id": "editor"}, + "update-query": {"id": "editor"}, + }, + "queries": {"trusted_report": {"sql": "select 1 as one"}}, + } + } + }, + ) + ds.add_memory_database("query_edit_trusted", name="data") + await ds.invoke_startup() + + response = await ds.client.get( + "/data/trusted_report/-/edit", actor={"id": "editor"} + ) + assert response.status_code == 403 + # Edit/delete links should not appear on a trusted/config query page + page = await ds.client.get("/data/trusted_report", actor={"id": "editor"}) + assert "/data/trusted_report/-/edit" not in page.text + + @pytest.mark.asyncio async def test_query_store_api_rejects_magic_parameters(): ds = Datasette(memory=True, default_deny=True) From 03f1ffdf8fbf0ed7da46be48f3fcc3f4698e1e21 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:45:01 -0700 Subject: [PATCH 1140/1172] Centralize JSON extra parsing --- datasette/extras.py | 6 ++++++ datasette/renderer.py | 3 ++- datasette/views/table.py | 7 ++----- tests/test_table_api.py | 11 +++++++++++ 4 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 datasette/extras.py diff --git a/datasette/extras.py b/datasette/extras.py new file mode 100644 index 00000000..01a9fb4b --- /dev/null +++ b/datasette/extras.py @@ -0,0 +1,6 @@ +def extra_names_from_request(request): + extra_bits = request.args.getlist("_extra") + extras = set() + for bit in extra_bits: + extras.update(part for part in bit.split(",") if part) + return extras diff --git a/datasette/renderer.py b/datasette/renderer.py index acf23e59..f40e3dbb 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -1,4 +1,5 @@ import json +from datasette.extras import extra_names_from_request from datasette.utils import ( value_as_boolean, remove_infinites, @@ -108,7 +109,7 @@ def json_renderer(request, args, data, error, truncated=None): # Don't include "columns" in output # https://github.com/simonw/datasette/issues/2136 - if isinstance(data, dict) and "columns" not in request.args.getlist("_extra"): + if isinstance(data, dict) and "columns" not in extra_names_from_request(request): data.pop("columns", None) # Handle _nl option for _shape=array diff --git a/datasette/views/table.py b/datasette/views/table.py index 4df1e1b4..9ba249f4 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -6,6 +6,7 @@ import urllib from asyncinject import Registry import markupsafe +from datasette.extras import extra_names_from_request from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette.events import ( @@ -850,11 +851,7 @@ class TableDropView(BaseView): def _get_extras(request): - extra_bits = request.args.getlist("_extra") - extras = set() - for bit in extra_bits: - extras.update(bit.split(",")) - return extras + return extra_names_from_request(request) async def _columns_to_select(table_columns, pks, request): diff --git a/tests/test_table_api.py b/tests/test_table_api.py index ceeb646d..eeb3dc8b 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1376,6 +1376,17 @@ async def test_table_extras(ds_client, extra, expected_json): assert response.json() == expected_json +@pytest.mark.asyncio +async def test_table_extra_columns_can_be_comma_separated(ds_client): + response = await ds_client.get( + "/fixtures/primary_key_multiple_columns.json?_extra=columns,count" + ) + assert response.status_code == 200 + data = response.json() + assert data["columns"] == ["id", "content", "content2"] + assert data["count"] == 1 + + @pytest.mark.asyncio async def test_extra_render_cell(): """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook""" From 17bbe6855c34630c14b077e08247d453d371cdea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:52:10 -0700 Subject: [PATCH 1141/1172] Refactor table JSON extras into classes --- datasette/extras.py | 94 ++++ datasette/views/table.py | 584 ++----------------------- datasette/views/table_extras.py | 746 ++++++++++++++++++++++++++++++++ 3 files changed, 884 insertions(+), 540 deletions(-) create mode 100644 datasette/views/table_extras.py diff --git a/datasette/extras.py b/datasette/extras.py index 01a9fb4b..786ec4f4 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,6 +1,100 @@ +import re +from enum import Enum +from typing import ClassVar + +from asyncinject import Registry + + def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") extras = set() for bit in extra_bits: extras.update(part for part in bit.split(",") if part) return extras + + +class ExtraScope(Enum): + TABLE = "table" + + +class Provider: + name: ClassVar[str | None] = None + scopes: ClassVar[frozenset[ExtraScope]] = frozenset() + public: ClassVar[bool] = False + + @classmethod + def key(cls): + return cls.name or _camel_to_snake(cls.__name__) + + @classmethod + def available_for(cls, scope): + return scope in cls.scopes + + async def resolve(self, context): + raise NotImplementedError + + +class Extra(Provider): + description: ClassVar[str | None] = None + public: ClassVar[bool] = True + stable: ClassVar[bool] = True + expensive: ClassVar[bool] = False + docs_note: ClassVar[str | None] = None + + @classmethod + def documentation(cls): + return { + "name": cls.key(), + "description": cls.description, + "scopes": [ + scope.value for scope in sorted(cls.scopes, key=lambda s: s.value) + ], + "stable": cls.stable, + "expensive": cls.expensive, + "docs_note": cls.docs_note, + } + + +class ExtraRegistry: + def __init__(self, classes): + self.classes = list(classes) + self.classes_by_name = {cls.key(): cls for cls in self.classes} + + def classes_for_scope(self, scope, include_internal=True): + classes = [ + cls + for cls in self.classes + if cls.available_for(scope) and (include_internal or cls.public) + ] + return classes + + def public_classes_for_scope(self, scope): + return self.classes_for_scope(scope, include_internal=False) + + async def resolve(self, requested, context, scope): + registry = Registry() + + async def context_provider(): + return context + + registry.register(context_provider, name="context") + + for cls in self.classes_for_scope(scope): + registry.register(cls().resolve, name=cls.key()) + + public_names = {cls.key() for cls in self.public_classes_for_scope(scope)} + requested_public_names = [ + name + for name in requested + if name in public_names and name in registry._registry + ] + resolved = await registry.resolve_multi(requested_public_names) + return { + name: resolved[name] for name in requested_public_names if name in resolved + } + + +def _camel_to_snake(name): + name = re.sub(r"(Extra|Provider)$", "", name) + name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() diff --git a/datasette/views/table.py b/datasette/views/table.py index 9ba249f4..c2d520f8 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -3,12 +3,10 @@ import itertools import json import urllib -from asyncinject import Registry import markupsafe from datasette.extras import extra_names_from_request from datasette.plugins import pm -from datasette.database import QueryInterrupted from datasette.events import ( AlterTableEvent, DropTableEvent, @@ -47,6 +45,12 @@ from datasette.filters import Filters import sqlite_utils from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView +from .table_extras import ( + TABLE_EXTRA_BUNDLES, + TableExtraContext, + resolve_table_extras, + table_extra_registry, +) LINK_WITH_LABEL = ( '{label} {id}' @@ -1465,560 +1469,60 @@ async def table_view_data( if extra_extras: extras.update(extra_extras) - async def extra_count_sql(): - return count_sql - - async def extra_count(): - "Total count of rows matching these filters" - # Calculate the total count for this query - count = None - if ( - not db.is_mutable - and datasette.inspect_data - and count_sql == f"select count(*) from {table_name} " - ): - # We can use a previously cached table row count - try: - count = datasette.inspect_data[database_name]["tables"][table_name][ - "count" - ] - except KeyError: - pass - - # Otherwise run a select count(*) ... - if count_sql and count is None and not nocount: - count_sql_limited = ( - f"select count(*) from (select * {from_sql} limit 10001)" - ) - try: - count_rows = list(await db.execute(count_sql_limited, from_sql_params)) - count = count_rows[0][0] - except QueryInterrupted: - pass - return count - - async def facet_instances(extra_count): - facet_instances = [] - facet_classes = list( - itertools.chain.from_iterable(pm.hook.register_facet_classes()) - ) - for facet_class in facet_classes: - facet_instances.append( - facet_class( - datasette, - request, - database_name, - sql=sql_no_order_no_limit, - params=params, - table=table_name, - table_config=table_metadata, - row_count=extra_count, - ) - ) - return facet_instances - - async def extra_facet_results(facet_instances): - "Results of facets calculated against this data" - facet_results = {} - facets_timed_out = [] - - if not nofacet: - # Run them in parallel - facet_awaitables = [facet.facet_results() for facet in facet_instances] - facet_awaitable_results = await run_sequential(*facet_awaitables) - for ( - instance_facet_results, - instance_facets_timed_out, - ) in facet_awaitable_results: - for facet_info in instance_facet_results: - base_key = facet_info["name"] - key = base_key - i = 1 - while key in facet_results: - i += 1 - key = f"{base_key}_{i}" - facet_results[key] = facet_info - facets_timed_out.extend(instance_facets_timed_out) - - return { - "results": facet_results, - "timed_out": facets_timed_out, - } - - async def extra_suggested_facets(facet_instances): - "Suggestions for facets that might return interesting results" - suggested_facets = [] - # Calculate suggested facets - if ( - datasette.setting("suggest_facets") - and datasette.setting("allow_facet") - and not _next - and not nofacet - and not nosuggest - ): - # Run them in parallel - facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] - for suggest_result in await run_sequential(*facet_suggest_awaitables): - suggested_facets.extend(suggest_result) - return suggested_facets - # Faceting if not datasette.setting("allow_facet") and any( arg.startswith("_facet") for arg in request.args ): raise BadRequest("_facet= is not allowed") - # human_description_en combines filters AND search, if provided - async def extra_human_description_en(): - "Human-readable description of the filters" - human_description_en = filters.human_description_en( - extra=extra_human_descriptions - ) - if sort or sort_desc: - human_description_en = " ".join( - [b for b in [human_description_en, sorted_by] if b] - ) - return human_description_en - - if sort or sort_desc: - sorted_by = "sorted by {}{}".format( - (sort or sort_desc), " descending" if sort_desc else "" - ) - - async def extra_next_url(): - "Full URL for the next page of results" - return next_url - - async def extra_columns(): - "Column names returned by this query" - return columns - - async def extra_all_columns(): - "All columns in the table, regardless of _col/_nocol filtering" - return list(table_columns) - - async def extra_primary_keys(): - "Primary keys for this table" - return pks - - async def extra_actions(): - async def actions(): - links = [] - kwargs = { - "datasette": datasette, - "database": database_name, - "actor": request.actor, - "request": request, - } - if is_view: - kwargs["view"] = table_name - method = pm.hook.view_actions - else: - kwargs["table"] = table_name - method = pm.hook.table_actions - for hook in method(**kwargs): - extra_links = await await_me_maybe(hook) - if extra_links: - links.extend(extra_links) - return links - - return actions - - async def extra_is_view(): - return is_view - - async def extra_debug(): - "Extra debug information" - return { - "resolved": repr(resolved), - "url_vars": request.url_vars, - "nofacet": nofacet, - "nosuggest": nosuggest, - } - - async def extra_request(): - "Full information about the request" - return { - "url": request.url, - "path": request.path, - "full_path": request.full_path, - "host": request.host, - "args": request.args._data, - } - - async def run_display_columns_and_rows(): - display_columns, display_rows = await display_columns_and_rows( - datasette, - database_name, - table_name, - results.description, - rows, - link_column=not is_view, - truncate_cells=datasette.setting("truncate_cells_html"), - sortable_columns=sortable_columns, - request=request, - ) - return { - "columns": display_columns, - "rows": display_rows, - } - - async def extra_display_columns(run_display_columns_and_rows): - return run_display_columns_and_rows["columns"] - - async def extra_display_rows(run_display_columns_and_rows): - return run_display_columns_and_rows["rows"] - - async def extra_render_cell(): - "Rendered HTML for each cell using the render_cell plugin hook" - pks_for_display = pks if pks else (["rowid"] if not is_view else []) - col_names = [col[0] for col in results.description] - ct_map = await datasette.get_column_types(database_name, table_name) - rendered_rows = [] - for row in rows: - rendered_row = {} - for value, column in zip(row, col_names): - ct = ct_map.get(column) - plugin_display_value = None - # Try column type render_cell first - if ct: - candidate = await ct.render_cell( - value=value, - column=column, - table=table_name, - database=database_name, - datasette=datasette, - request=request, - ) - if candidate is not None: - plugin_display_value = candidate - if plugin_display_value is None: - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=table_name, - pks=pks_for_display, - database=database_name, - datasette=datasette, - request=request, - column_type=ct, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - rendered_row[column] = str(plugin_display_value) - rendered_rows.append(rendered_row) - return rendered_rows - - async def extra_query(): - "Details of the underlying SQL query" - return { - "sql": sql, - "params": params, - } - - async def extra_column_types(): - "Column type assignments for this table" - ct_map = await datasette.get_column_types(database_name, table_name) - return { - col_name: { - "type": ct.name, - "config": ct.config, - } - for col_name, ct in ct_map.items() - } - - async def extra_set_column_type_ui(): - "Column type UI metadata for this table" - if is_view: - return None - - if not await datasette.allowed( - action="set-column-type", - resource=TableResource(database=database_name, table=table_name), - actor=request.actor, - ): - return None - - column_details = await datasette._get_resource_column_details( - database_name, table_name - ) - ct_map = await datasette.get_column_types(database_name, table_name) - columns = {} - for column_name, column_detail in column_details.items(): - current = ct_map.get(column_name) - columns[column_name] = { - "current": ( - {"type": current.name, "config": current.config} - if current is not None - else None - ), - "options": [ - { - "name": name, - "description": ct_cls.description, - } - for name, ct_cls in sorted(datasette._column_types.items()) - if datasette._column_type_is_applicable(ct_cls, column_detail) - ], - } - return { - "path": "{}/-/set-column-type".format( - datasette.urls.table(database_name, table_name) - ), - "columns": columns, - } - - async def extra_metadata(): - "Metadata about the table and database" - tablemetadata = await datasette.get_resource_metadata(database_name, table_name) - - rows = await datasette.get_internal_database().execute( - """ - SELECT - column_name, - value - FROM metadata_columns - WHERE database_name = ? - AND resource_name = ? - AND key = 'description' - """, - [database_name, table_name], - ) - tablemetadata["columns"] = dict(rows) - return tablemetadata - - async def extra_database(): - return database_name - - async def extra_table(): - return table_name - - async def extra_database_color(): - return db.color - - async def extra_form_hidden_args(): - form_hidden_args = [] - for key in request.args: - if ( - key.startswith("_") - and key not in ("_sort", "_sort_desc", "_search", "_next") - and "__" not in key - ): - for value in request.args.getlist(key): - form_hidden_args.append((key, value)) - return form_hidden_args - - async def extra_filters(): - return filters - - async def extra_custom_table_templates(): - return [ - f"_table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", - f"_table-table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", - "_table.html", - ] - - async def extra_sorted_facet_results(extra_facet_results): - facet_configs = table_metadata.get("facets", []) - if facet_configs: - # Build ordered list of facet names from metadata config - metadata_facet_names = [] - for fc in facet_configs: - if isinstance(fc, str): - metadata_facet_names.append(fc) - elif isinstance(fc, dict): - metadata_facet_names.append(list(fc.values())[0]) - metadata_order = {name: i for i, name in enumerate(metadata_facet_names)} - metadata_facets = [] - request_facets = [] - for f in extra_facet_results["results"].values(): - if f["name"] in metadata_order: - metadata_facets.append(f) - else: - request_facets.append(f) - metadata_facets.sort(key=lambda f: metadata_order[f["name"]]) - request_facets.sort( - key=lambda f: (len(f["results"]), f["name"]), - reverse=True, - ) - return metadata_facets + request_facets - else: - return sorted( - extra_facet_results["results"].values(), - key=lambda f: (len(f["results"]), f["name"]), - reverse=True, - ) - - async def extra_table_definition(): - return await db.get_table_definition(table_name) - - async def extra_view_definition(): - return await db.get_view_definition(table_name) - - async def extra_renderers(extra_expandable_columns, extra_query): - renderers = {} - url_labels_extra = {} - if extra_expandable_columns: - url_labels_extra = {"_labels": "on"} - for key, (_, can_render) in datasette.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=datasette, - columns=columns or [], - rows=rows or [], - sql=extra_query.get("sql", None), - query_name=None, - database=database_name, - table=table_name, - request=request, - view_name="table", - ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = datasette.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format=key, - extra_qs={**url_labels_extra}, - ) - ) - return renderers - - async def extra_private(): - return private - - async def extra_expandable_columns(): - expandables = [] - db = datasette.databases[database_name] - for fk in await db.foreign_keys_for_table(table_name): - label_column = await db.label_column_for_table(fk["other_table"]) - expandables.append((fk, label_column)) - return expandables - - async def extra_extras(): - "Available ?_extra= blocks" - all_extras = [ - (key[len("extra_") :], fn.__doc__) - for key, fn in registry._registry.items() - if key.startswith("extra_") - ] - return [ - { - "name": name, - "description": doc, - "toggle_url": datasette.absolute_url( - request, - datasette.urls.path( - path_with_added_args(request, {"_extra": name}) - if name not in extras - else path_with_removed_args(request, {"_extra": name}) - ), - ), - "selected": name in extras, - } - for name, doc in all_extras - ] - - async def extra_facets_timed_out(extra_facet_results): - return extra_facet_results["timed_out"] - - bundles = { - "html": [ - "suggested_facets", - "facet_results", - "facets_timed_out", - "count", - "count_sql", - "human_description_en", - "next_url", - "metadata", - "query", - "columns", - "display_columns", - "display_rows", - "database", - "table", - "database_color", - "actions", - "filters", - "renderers", - "custom_table_templates", - "sorted_facet_results", - "table_definition", - "view_definition", - "is_view", - "private", - "primary_keys", - "all_columns", - "expandable_columns", - "form_hidden_args", - "set_column_type_ui", - ] - } - - for key, values in bundles.items(): + for key, values in TABLE_EXTRA_BUNDLES.items(): if f"_{key}" in extras: extras.update(values) extras.discard(f"_{key}") - registry = Registry( - extra_count, - extra_count_sql, - extra_facet_results, - extra_facets_timed_out, - extra_suggested_facets, - facet_instances, - extra_human_description_en, - extra_next_url, - extra_columns, - extra_all_columns, - extra_primary_keys, - run_display_columns_and_rows, - extra_display_columns, - extra_display_rows, - extra_render_cell, - extra_debug, - extra_request, - extra_query, - extra_column_types, - extra_set_column_type_ui, - extra_metadata, - extra_extras, - extra_database, - extra_table, - extra_database_color, - extra_actions, - extra_filters, - extra_renderers, - extra_custom_table_templates, - extra_sorted_facet_results, - extra_table_definition, - extra_view_definition, - extra_is_view, - extra_private, - extra_expandable_columns, - extra_form_hidden_args, + table_extra_context = TableExtraContext( + datasette=datasette, + request=request, + resolved=resolved, + db=db, + database_name=database_name, + table_name=table_name, + is_view=is_view, + private=private, + rows=rows, + columns=columns, + results_description=results.description, + table_columns=table_columns, + pks=pks, + count_sql=count_sql, + from_sql=from_sql, + from_sql_params=from_sql_params, + nocount=nocount, + nofacet=nofacet, + nosuggest=nosuggest, + next_arg=request.args.get("_next"), + next_value=next_value, + next_url=next_url, + sql=sql, + sql_no_order_no_limit=sql_no_order_no_limit, + params=params, + table_metadata=table_metadata, + filters=filters, + extra_human_descriptions=extra_human_descriptions, + sort=sort, + sort_desc=sort_desc, + sortable_columns=sortable_columns, + extras=extras, + extra_registry=table_extra_registry, + display_columns_and_rows=display_columns_and_rows, + run_sequential=run_sequential, ) - results = await registry.resolve_multi( - ["extra_{}".format(extra) for extra in extras] - ) data = { "ok": True, "next": next_value and str(next_value) or None, } - data.update( - { - key.replace("extra_", ""): value - for key, value in results.items() - if key.startswith("extra_") and key.replace("extra_", "") in extras - } - ) + data.update(await resolve_table_extras(extras, table_extra_context)) raw_sqlite_rows = rows[:page_size] # Apply transform_value for columns with assigned types ct_map = await datasette.get_column_types(database_name, table_name) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py new file mode 100644 index 00000000..2ec2adf0 --- /dev/null +++ b/datasette/views/table_extras.py @@ -0,0 +1,746 @@ +import itertools +from dataclasses import dataclass + +from datasette.database import QueryInterrupted +from datasette.extras import Extra, ExtraRegistry, ExtraScope, Provider +from datasette.plugins import pm +from datasette.resources import TableResource +from datasette.utils import ( + await_me_maybe, + call_with_supported_arguments, + path_with_added_args, + path_with_format, + path_with_removed_args, + to_css_class, +) + + +@dataclass(frozen=True) +class TableExtraContext: + datasette: object + request: object + resolved: object + db: object + database_name: str + table_name: str + is_view: bool + private: bool + rows: list + columns: list + results_description: list + table_columns: list + pks: list + count_sql: str + from_sql: str + from_sql_params: dict + nocount: object + nofacet: object + nosuggest: object + next_arg: object + next_value: object + next_url: str | None + sql: str + sql_no_order_no_limit: str + params: dict + table_metadata: dict + filters: object + extra_human_descriptions: list + sort: str | None + sort_desc: str | None + sortable_columns: set + extras: set + extra_registry: ExtraRegistry + display_columns_and_rows: object + run_sequential: object + + +class CountSqlExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.count_sql + + +class CountExtra(Extra): + description = "Total count of rows matching these filters" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context): + count = None + if ( + not context.db.is_mutable + and context.datasette.inspect_data + and context.count_sql == f"select count(*) from {context.table_name} " + ): + try: + count = context.datasette.inspect_data[context.database_name]["tables"][ + context.table_name + ]["count"] + except KeyError: + pass + + if context.count_sql and count is None and not context.nocount: + count_sql_limited = ( + f"select count(*) from (select * {context.from_sql} limit 10001)" + ) + try: + count_rows = list( + await context.db.execute(count_sql_limited, context.from_sql_params) + ) + count = count_rows[0][0] + except QueryInterrupted: + pass + return count + + +class FacetInstancesProvider(Provider): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, count): + facet_instances = [] + facet_classes = list( + itertools.chain.from_iterable(pm.hook.register_facet_classes()) + ) + for facet_class in facet_classes: + facet_instances.append( + facet_class( + context.datasette, + context.request, + context.database_name, + sql=context.sql_no_order_no_limit, + params=context.params, + table=context.table_name, + table_config=context.table_metadata, + row_count=count, + ) + ) + return facet_instances + + +class FacetResultsExtra(Extra): + description = "Results of facets calculated against this data" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context, facet_instances): + facet_results = {} + facets_timed_out = [] + + if not context.nofacet: + facet_awaitables = [facet.facet_results() for facet in facet_instances] + facet_awaitable_results = await context.run_sequential(*facet_awaitables) + for ( + instance_facet_results, + instance_facets_timed_out, + ) in facet_awaitable_results: + for facet_info in instance_facet_results: + base_key = facet_info["name"] + key = base_key + i = 1 + while key in facet_results: + i += 1 + key = f"{base_key}_{i}" + facet_results[key] = facet_info + facets_timed_out.extend(instance_facets_timed_out) + + return { + "results": facet_results, + "timed_out": facets_timed_out, + } + + +class FacetsTimedOutExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, facet_results): + return facet_results["timed_out"] + + +class SuggestedFacetsExtra(Extra): + description = "Suggestions for facets that might return interesting results" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context, facet_instances): + suggested_facets = [] + if ( + context.datasette.setting("suggest_facets") + and context.datasette.setting("allow_facet") + and not context.next_arg + and not context.nofacet + and not context.nosuggest + ): + facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] + for suggest_result in await context.run_sequential( + *facet_suggest_awaitables + ): + suggested_facets.extend(suggest_result) + return suggested_facets + + +class HumanDescriptionEnExtra(Extra): + description = "Human-readable description of the filters" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + human_description_en = context.filters.human_description_en( + extra=context.extra_human_descriptions + ) + if context.sort or context.sort_desc: + sorted_by = "sorted by {}{}".format( + (context.sort or context.sort_desc), + " descending" if context.sort_desc else "", + ) + human_description_en = " ".join( + [b for b in [human_description_en, sorted_by] if b] + ) + return human_description_en + + +class NextUrlExtra(Extra): + description = "Full URL for the next page of results" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.next_url + + +class ColumnsExtra(Extra): + description = "Column names returned by this query" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.columns + + +class AllColumnsExtra(Extra): + description = "All columns in the table, regardless of _col/_nocol filtering" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return list(context.table_columns) + + +class PrimaryKeysExtra(Extra): + description = "Primary keys for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.pks + + +class ActionsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + async def actions(): + links = [] + kwargs = { + "datasette": context.datasette, + "database": context.database_name, + "actor": context.request.actor, + "request": context.request, + } + if context.is_view: + kwargs["view"] = context.table_name + method = pm.hook.view_actions + else: + kwargs["table"] = context.table_name + method = pm.hook.table_actions + for hook in method(**kwargs): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + + return actions + + +class IsViewExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.is_view + + +class DebugExtra(Extra): + description = "Extra debug information" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "resolved": repr(context.resolved), + "url_vars": context.request.url_vars, + "nofacet": context.nofacet, + "nosuggest": context.nosuggest, + } + + +class RequestExtra(Extra): + description = "Full information about the request" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "url": context.request.url, + "path": context.request.path, + "full_path": context.request.full_path, + "host": context.request.host, + "args": context.request.args._data, + } + + +class DisplayColumnsAndRowsProvider(Provider): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + display_columns, display_rows = await context.display_columns_and_rows( + context.datasette, + context.database_name, + context.table_name, + context.results_description, + context.rows, + link_column=not context.is_view, + truncate_cells=context.datasette.setting("truncate_cells_html"), + sortable_columns=context.sortable_columns, + request=context.request, + ) + return { + "columns": display_columns, + "rows": display_rows, + } + + +class DisplayColumnsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, display_columns_and_rows): + return display_columns_and_rows["columns"] + + +class DisplayRowsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, display_columns_and_rows): + return display_columns_and_rows["rows"] + + +class RenderCellExtra(Extra): + description = "Rendered HTML for each cell using the render_cell plugin hook" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + pks_for_display = ( + context.pks if context.pks else (["rowid"] if not context.is_view else []) + ) + col_names = [col[0] for col in context.results_description] + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + rendered_rows = [] + for row in context.rows: + rendered_row = {} + for value, column in zip(row, col_names): + ct = ct_map.get(column) + plugin_display_value = None + if ct: + candidate = await ct.render_cell( + value=value, + column=column, + table=context.table_name, + database=context.database_name, + datasette=context.datasette, + request=context.request, + ) + if candidate is not None: + plugin_display_value = candidate + if plugin_display_value is None: + for candidate in pm.hook.render_cell( + row=row, + value=value, + column=column, + table=context.table_name, + pks=pks_for_display, + database=context.database_name, + datasette=context.datasette, + request=context.request, + column_type=ct, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value: + rendered_row[column] = str(plugin_display_value) + rendered_rows.append(rendered_row) + return rendered_rows + + +class QueryExtra(Extra): + description = "Details of the underlying SQL query" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "sql": context.sql, + "params": context.params, + } + + +class ColumnTypesExtra(Extra): + description = "Column type assignments for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + return { + col_name: { + "type": ct.name, + "config": ct.config, + } + for col_name, ct in ct_map.items() + } + + +class SetColumnTypeUiExtra(Extra): + description = "Column type UI metadata for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + if context.is_view: + return None + + if not await context.datasette.allowed( + action="set-column-type", + resource=TableResource( + database=context.database_name, table=context.table_name + ), + actor=context.request.actor, + ): + return None + + column_details = await context.datasette._get_resource_column_details( + context.database_name, context.table_name + ) + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + columns = {} + for column_name, column_detail in column_details.items(): + current = ct_map.get(column_name) + columns[column_name] = { + "current": ( + {"type": current.name, "config": current.config} + if current is not None + else None + ), + "options": [ + { + "name": name, + "description": ct_cls.description, + } + for name, ct_cls in sorted(context.datasette._column_types.items()) + if context.datasette._column_type_is_applicable( + ct_cls, column_detail + ) + ], + } + return { + "path": "{}/-/set-column-type".format( + context.datasette.urls.table(context.database_name, context.table_name) + ), + "columns": columns, + } + + +class MetadataExtra(Extra): + description = "Metadata about the table and database" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + tablemetadata = await context.datasette.get_resource_metadata( + context.database_name, context.table_name + ) + + rows = await context.datasette.get_internal_database().execute( + """ + SELECT + column_name, + value + FROM metadata_columns + WHERE database_name = ? + AND resource_name = ? + AND key = 'description' + """, + [context.database_name, context.table_name], + ) + tablemetadata["columns"] = dict(rows) + return tablemetadata + + +class DatabaseExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.database_name + + +class TableExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.table_name + + +class DatabaseColorExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.db.color + + +class FormHiddenArgsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + form_hidden_args = [] + for key in context.request.args: + if ( + key.startswith("_") + and key not in ("_sort", "_sort_desc", "_search", "_next") + and "__" not in key + ): + for value in context.request.args.getlist(key): + form_hidden_args.append((key, value)) + return form_hidden_args + + +class FiltersExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.filters + + +class CustomTableTemplatesExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return [ + f"_table-{to_css_class(context.database_name)}-{to_css_class(context.table_name)}.html", + f"_table-table-{to_css_class(context.database_name)}-{to_css_class(context.table_name)}.html", + "_table.html", + ] + + +class SortedFacetResultsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, facet_results): + facet_configs = context.table_metadata.get("facets", []) + if facet_configs: + metadata_facet_names = [] + for fc in facet_configs: + if isinstance(fc, str): + metadata_facet_names.append(fc) + elif isinstance(fc, dict): + metadata_facet_names.append(list(fc.values())[0]) + metadata_order = {name: i for i, name in enumerate(metadata_facet_names)} + metadata_facets = [] + request_facets = [] + for f in facet_results["results"].values(): + if f["name"] in metadata_order: + metadata_facets.append(f) + else: + request_facets.append(f) + metadata_facets.sort(key=lambda f: metadata_order[f["name"]]) + request_facets.sort( + key=lambda f: (len(f["results"]), f["name"]), + reverse=True, + ) + return metadata_facets + request_facets + else: + return sorted( + facet_results["results"].values(), + key=lambda f: (len(f["results"]), f["name"]), + reverse=True, + ) + + +class TableDefinitionExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return await context.db.get_table_definition(context.table_name) + + +class ViewDefinitionExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return await context.db.get_view_definition(context.table_name) + + +class RenderersExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, expandable_columns, query): + renderers = {} + url_labels_extra = {} + if expandable_columns: + url_labels_extra = {"_labels": "on"} + for key, (_, can_render) in context.datasette.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=context.datasette, + columns=context.columns or [], + rows=context.rows or [], + sql=query.get("sql", None), + query_name=None, + database=context.database_name, + table=context.table_name, + request=context.request, + view_name="table", + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = context.datasette.urls.path( + path_with_format( + request=context.request, + path=context.request.scope.get("route_path"), + format=key, + extra_qs={**url_labels_extra}, + ) + ) + return renderers + + +class PrivateExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.private + + +class ExpandableColumnsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + expandables = [] + db = context.datasette.databases[context.database_name] + for fk in await db.foreign_keys_for_table(context.table_name): + label_column = await db.label_column_for_table(fk["other_table"]) + expandables.append((fk, label_column)) + return expandables + + +class ExtrasExtra(Extra): + description = "Available ?_extra= blocks" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + all_extras = [ + (cls.key(), cls.description) + for cls in context.extra_registry.public_classes_for_scope(ExtraScope.TABLE) + ] + return [ + { + "name": name, + "description": description, + "toggle_url": context.datasette.absolute_url( + context.request, + context.datasette.urls.path( + path_with_added_args(context.request, {"_extra": name}) + if name not in context.extras + else path_with_removed_args(context.request, {"_extra": name}) + ), + ), + "selected": name in context.extras, + } + for name, description in all_extras + ] + + +TABLE_EXTRA_BUNDLES = { + "html": [ + "suggested_facets", + "facet_results", + "facets_timed_out", + "count", + "count_sql", + "human_description_en", + "next_url", + "metadata", + "query", + "columns", + "display_columns", + "display_rows", + "database", + "table", + "database_color", + "actions", + "filters", + "renderers", + "custom_table_templates", + "sorted_facet_results", + "table_definition", + "view_definition", + "is_view", + "private", + "primary_keys", + "all_columns", + "expandable_columns", + "form_hidden_args", + "set_column_type_ui", + ] +} + + +TABLE_EXTRA_CLASSES = [ + CountExtra, + CountSqlExtra, + FacetResultsExtra, + FacetsTimedOutExtra, + SuggestedFacetsExtra, + FacetInstancesProvider, + HumanDescriptionEnExtra, + NextUrlExtra, + ColumnsExtra, + AllColumnsExtra, + PrimaryKeysExtra, + DisplayColumnsAndRowsProvider, + DisplayColumnsExtra, + DisplayRowsExtra, + RenderCellExtra, + DebugExtra, + RequestExtra, + QueryExtra, + ColumnTypesExtra, + SetColumnTypeUiExtra, + MetadataExtra, + ExtrasExtra, + DatabaseExtra, + TableExtra, + DatabaseColorExtra, + ActionsExtra, + FiltersExtra, + RenderersExtra, + CustomTableTemplatesExtra, + SortedFacetResultsExtra, + TableDefinitionExtra, + ViewDefinitionExtra, + IsViewExtra, + PrivateExtra, + ExpandableColumnsExtra, + FormHiddenArgsExtra, +] + + +table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) + + +async def resolve_table_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) From 111eeaf3702cd5ee417532beb80c746d49d92a11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:56:00 -0700 Subject: [PATCH 1142/1172] Document table JSON extras from metadata --- datasette/views/table_extras.py | 18 +++++++ docs/json_api.rst | 95 +++++++++++++++++++++++++++++++++ docs/json_api_doc.py | 20 +++++++ 3 files changed, 133 insertions(+) create mode 100644 docs/json_api_doc.py diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 2ec2adf0..e71c15d6 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -55,6 +55,7 @@ class TableExtraContext: class CountSqlExtra(Extra): + description = "SQL query used to calculate the total count" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -151,6 +152,7 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): + description = "Facet calculations that timed out" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -231,6 +233,7 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): + description = "Table or view actions made available by plugin hooks" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -258,6 +261,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): + description = "Whether this resource is a view instead of a table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -313,6 +317,7 @@ class DisplayColumnsAndRowsProvider(Provider): class DisplayColumnsExtra(Extra): + description = "Column metadata used by the HTML table display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -320,6 +325,7 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): + description = "Row data formatted for the HTML table display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -482,6 +488,7 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): + description = "Database name" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -489,6 +496,7 @@ class DatabaseExtra(Extra): class TableExtra(Extra): + description = "Table name" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -496,6 +504,7 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): + description = "Color assigned to the database" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -503,6 +512,7 @@ class DatabaseColorExtra(Extra): class FormHiddenArgsExtra(Extra): + description = "Hidden form arguments used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -519,6 +529,7 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): + description = "Filters object used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -526,6 +537,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): + description = "Custom template names considered for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -537,6 +549,7 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): + description = "Facet results sorted for display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -571,6 +584,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): + description = "SQL definition for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -578,6 +592,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): + description = "SQL definition for this view" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -585,6 +600,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): + description = "Alternative output renderers available for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, expandable_columns, query): @@ -619,6 +635,7 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): + description = "Whether this table is private to the current actor" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -626,6 +643,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): + description = "Foreign key columns that can be expanded with labels" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 65031bf4..af60a527 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -235,6 +235,101 @@ query string arguments: Only available if the :ref:`setting_trace_debug` setting is enabled. +.. _json_api_extra: + +Expanding table JSON responses +------------------------------ + +Table JSON responses can be expanded with one or more ``?_extra=`` parameters. +These can be repeated or comma-separated: + +:: + + ?_extra=columns&_extra=count,next_url + +The available table extras are listed below. + +.. [[[cog + from json_api_doc import table_extras + table_extras(cog) +.. ]]] + +.. list-table:: + :header-rows: 1 + + * - Extra + - Description + * - ``count`` + - Total count of rows matching these filters (May execute additional queries.) + * - ``count_sql`` + - SQL query used to calculate the total count + * - ``facet_results`` + - Results of facets calculated against this data (May execute additional queries.) + * - ``facets_timed_out`` + - Facet calculations that timed out + * - ``suggested_facets`` + - Suggestions for facets that might return interesting results (May execute additional queries.) + * - ``human_description_en`` + - Human-readable description of the filters + * - ``next_url`` + - Full URL for the next page of results + * - ``columns`` + - Column names returned by this query + * - ``all_columns`` + - All columns in the table, regardless of _col/_nocol filtering + * - ``primary_keys`` + - Primary keys for this table + * - ``display_columns`` + - Column metadata used by the HTML table display + * - ``display_rows`` + - Row data formatted for the HTML table display + * - ``render_cell`` + - Rendered HTML for each cell using the render_cell plugin hook + * - ``debug`` + - Extra debug information + * - ``request`` + - Full information about the request + * - ``query`` + - Details of the underlying SQL query + * - ``column_types`` + - Column type assignments for this table + * - ``set_column_type_ui`` + - Column type UI metadata for this table + * - ``metadata`` + - Metadata about the table and database + * - ``extras`` + - Available ?_extra= blocks + * - ``database`` + - Database name + * - ``table`` + - Table name + * - ``database_color`` + - Color assigned to the database + * - ``actions`` + - Table or view actions made available by plugin hooks + * - ``filters`` + - Filters object used by the HTML table interface + * - ``renderers`` + - Alternative output renderers available for this table + * - ``custom_table_templates`` + - Custom template names considered for this table + * - ``sorted_facet_results`` + - Facet results sorted for display + * - ``table_definition`` + - SQL definition for this table + * - ``view_definition`` + - SQL definition for this view + * - ``is_view`` + - Whether this resource is a view instead of a table + * - ``private`` + - Whether this table is private to the current actor + * - ``expandable_columns`` + - Foreign key columns that can be expanded with labels + * - ``form_hidden_args`` + - Hidden form arguments used by the HTML table interface + +.. [[[end]]] + .. _table_arguments: Table arguments diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py new file mode 100644 index 00000000..f07c3ba7 --- /dev/null +++ b/docs/json_api_doc.py @@ -0,0 +1,20 @@ +def table_extras(cog): + from datasette.extras import ExtraScope + from datasette.views.table_extras import table_extra_registry + + cog.out("\n.. list-table::\n") + cog.out(" :header-rows: 1\n\n") + cog.out(" * - Extra\n") + cog.out(" - Description\n") + for cls in table_extra_registry.public_classes_for_scope(ExtraScope.TABLE): + description = cls.description or "" + notes = [] + if cls.expensive: + notes.append("May execute additional queries.") + if cls.docs_note: + notes.append(cls.docs_note) + if notes: + description = "{} ({})".format(description, " ".join(notes)).strip() + cog.out(" * - ``{}``\n".format(cls.key())) + cog.out(" - {}\n".format(description)) + cog.out("\n") From 79c8aff31df16e514616a7778fad1386ac9b4b2c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:10:58 -0700 Subject: [PATCH 1143/1172] Add generated examples for table JSON extras --- datasette/extras.py | 11 + datasette/views/table_extras.py | 87 +++++- docs/json_api.rst | 470 +++++++++++++++++++++++++++----- docs/json_api_doc.py | 64 ++++- tests/test_docs.py | 10 + 5 files changed, 561 insertions(+), 81 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 786ec4f4..d867f26c 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,4 +1,5 @@ import re +from dataclasses import dataclass from enum import Enum from typing import ClassVar @@ -17,6 +18,14 @@ class ExtraScope(Enum): TABLE = "table" +@dataclass(frozen=True) +class ExtraExample: + path: str | None = None + key: str | None = None + value: object | None = None + note: str | None = None + + class Provider: name: ClassVar[str | None] = None scopes: ClassVar[frozenset[ExtraScope]] = frozenset() @@ -36,6 +45,7 @@ class Provider: class Extra(Provider): description: ClassVar[str | None] = None + example: ClassVar[ExtraExample | None] = None public: ClassVar[bool] = True stable: ClassVar[bool] = True expensive: ClassVar[bool] = False @@ -52,6 +62,7 @@ class Extra(Provider): "stable": cls.stable, "expensive": cls.expensive, "docs_note": cls.docs_note, + "example": cls.example, } diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index e71c15d6..0eefeaa9 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -2,7 +2,7 @@ import itertools from dataclasses import dataclass from datasette.database import QueryInterrupted -from datasette.extras import Extra, ExtraRegistry, ExtraScope, Provider +from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider from datasette.plugins import pm from datasette.resources import TableResource from datasette.utils import ( @@ -56,6 +56,7 @@ class TableExtraContext: class CountSqlExtra(Extra): description = "SQL query used to calculate the total count" + example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -64,6 +65,7 @@ class CountSqlExtra(Extra): class CountExtra(Extra): description = "Total count of rows matching these filters" + example = ExtraExample("/fixtures/facetable.json?_extra=count") scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -121,6 +123,22 @@ class FacetInstancesProvider(Provider): class FacetResultsExtra(Extra): description = "Results of facets calculated against this data" + example = ExtraExample( + value={ + "results": { + "state": { + "name": "state", + "type": "column", + "results": [ + {"value": "CA", "label": "CA", "count": 10}, + {"value": "MI", "label": "MI", "count": 4}, + ], + } + }, + "timed_out": [], + }, + note="Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.", + ) scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -153,6 +171,9 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): description = "Facet calculations that timed out" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -161,6 +182,15 @@ class FacetsTimedOutExtra(Extra): class SuggestedFacetsExtra(Extra): description = "Suggestions for facets that might return interesting results" + example = ExtraExample( + value=[ + { + "name": "state", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state", + } + ], + note="Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.", + ) scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -183,6 +213,9 @@ class SuggestedFacetsExtra(Extra): class HumanDescriptionEnExtra(Extra): description = "Human-readable description of the filters" + example = ExtraExample( + "/fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -202,6 +235,7 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" + example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -210,6 +244,7 @@ class NextUrlExtra(Extra): class ColumnsExtra(Extra): description = "Column names returned by this query" + example = ExtraExample("/fixtures/facetable.json?_extra=columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -218,6 +253,7 @@ class ColumnsExtra(Extra): class AllColumnsExtra(Extra): description = "All columns in the table, regardless of _col/_nocol filtering" + example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -226,6 +262,7 @@ class AllColumnsExtra(Extra): class PrimaryKeysExtra(Extra): description = "Primary keys for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -262,6 +299,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): description = "Whether this resource is a view instead of a table" + example = ExtraExample("/fixtures/simple_view.json?_extra=is_view") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -318,6 +356,28 @@ class DisplayColumnsAndRowsProvider(Provider): class DisplayColumnsExtra(Extra): description = "Column metadata used by the HTML table display" + example = ExtraExample( + value=[ + { + "name": "pk", + "sortable": True, + "is_pk": True, + "type": "INTEGER", + "notnull": 0, + }, + { + "name": "created", + "sortable": True, + "is_pk": False, + "type": "TEXT", + "notnull": 0, + "description": None, + "column_type": None, + "column_type_config": None, + }, + ], + note="Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.", + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -334,6 +394,13 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" + example = ExtraExample( + value=[ + {}, + {"content": "Custom rendered HTML"}, + ], + note="Only columns whose rendered value differs from the default are included.", + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -385,6 +452,7 @@ class RenderCellExtra(Extra): class QueryExtra(Extra): description = "Details of the underlying SQL query" + example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -396,6 +464,7 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" + example = ExtraExample(value={}) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -464,6 +533,7 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): description = "Metadata about the table and database" + example = ExtraExample("/fixtures/facetable.json?_extra=metadata") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -489,6 +559,7 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): description = "Database name" + example = ExtraExample("/fixtures/facetable.json?_extra=database") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -497,6 +568,7 @@ class DatabaseExtra(Extra): class TableExtra(Extra): description = "Table name" + example = ExtraExample("/fixtures/facetable.json?_extra=table") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -505,6 +577,7 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" + example = ExtraExample("/fixtures/facetable.json?_extra=database_color") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -513,6 +586,9 @@ class DatabaseColorExtra(Extra): class FormHiddenArgsExtra(Extra): description = "Hidden form arguments used by the HTML table interface" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -538,6 +614,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -550,6 +627,9 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): description = "Facet results sorted for display" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -585,6 +665,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): description = "SQL definition for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=table_definition") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -593,6 +674,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): description = "SQL definition for this view" + example = ExtraExample("/fixtures/simple_view.json?_extra=view_definition") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -601,6 +683,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=renderers") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, expandable_columns, query): @@ -636,6 +719,7 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): description = "Whether this table is private to the current actor" + example = ExtraExample("/fixtures/facetable.json?_extra=private") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -644,6 +728,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" + example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index af60a527..d12a388e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -254,79 +254,405 @@ The available table extras are listed below. table_extras(cog) .. ]]] -.. list-table:: - :header-rows: 1 +``count`` + Total count of rows matching these filters (May execute additional queries.) - * - Extra - - Description - * - ``count`` - - Total count of rows matching these filters (May execute additional queries.) - * - ``count_sql`` - - SQL query used to calculate the total count - * - ``facet_results`` - - Results of facets calculated against this data (May execute additional queries.) - * - ``facets_timed_out`` - - Facet calculations that timed out - * - ``suggested_facets`` - - Suggestions for facets that might return interesting results (May execute additional queries.) - * - ``human_description_en`` - - Human-readable description of the filters - * - ``next_url`` - - Full URL for the next page of results - * - ``columns`` - - Column names returned by this query - * - ``all_columns`` - - All columns in the table, regardless of _col/_nocol filtering - * - ``primary_keys`` - - Primary keys for this table - * - ``display_columns`` - - Column metadata used by the HTML table display - * - ``display_rows`` - - Row data formatted for the HTML table display - * - ``render_cell`` - - Rendered HTML for each cell using the render_cell plugin hook - * - ``debug`` - - Extra debug information - * - ``request`` - - Full information about the request - * - ``query`` - - Details of the underlying SQL query - * - ``column_types`` - - Column type assignments for this table - * - ``set_column_type_ui`` - - Column type UI metadata for this table - * - ``metadata`` - - Metadata about the table and database - * - ``extras`` - - Available ?_extra= blocks - * - ``database`` - - Database name - * - ``table`` - - Table name - * - ``database_color`` - - Color assigned to the database - * - ``actions`` - - Table or view actions made available by plugin hooks - * - ``filters`` - - Filters object used by the HTML table interface - * - ``renderers`` - - Alternative output renderers available for this table - * - ``custom_table_templates`` - - Custom template names considered for this table - * - ``sorted_facet_results`` - - Facet results sorted for display - * - ``table_definition`` - - SQL definition for this table - * - ``view_definition`` - - SQL definition for this view - * - ``is_view`` - - Whether this resource is a view instead of a table - * - ``private`` - - Whether this table is private to the current actor - * - ``expandable_columns`` - - Foreign key columns that can be expanded with labels - * - ``form_hidden_args`` - - Hidden form arguments used by the HTML table interface + ``GET /fixtures/facetable.json?_extra=count`` + + .. code-block:: json + + 15 + +``count_sql`` + SQL query used to calculate the total count + + ``GET /fixtures/facetable.json?_size=0&_extra=count_sql`` + + .. code-block:: json + + "select count(*) from facetable " + +``facet_results`` + Results of facets calculated against this data (May execute additional queries.) + + Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results. + + .. code-block:: json + + { + "results": { + "state": { + "name": "state", + "type": "column", + "results": [ + { + "value": "CA", + "label": "CA", + "count": 10 + }, + { + "value": "MI", + "label": "MI", + "count": 4 + } + ] + } + }, + "timed_out": [] + } + +``facets_timed_out`` + Facet calculations that timed out + + ``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out`` + + .. code-block:: json + + [] + +``suggested_facets`` + Suggestions for facets that might return interesting results (May execute additional queries.) + + Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets. + + .. code-block:: json + + [ + { + "name": "state", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state" + } + ] + +``human_description_en`` + Human-readable description of the filters + + ``GET /fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en`` + + .. code-block:: json + + "where state = \"CA\" sorted by pk" + +``next_url`` + Full URL for the next page of results + + ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` + + .. code-block:: json + + "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" + +``columns`` + Column names returned by this query + + ``GET /fixtures/facetable.json?_extra=columns`` + + .. code-block:: json + + [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + "n" + ] + +``all_columns`` + All columns in the table, regardless of _col/_nocol filtering + + ``GET /fixtures/facetable.json?_col=pk&_extra=all_columns`` + + .. code-block:: json + + [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + "n" + ] + +``primary_keys`` + Primary keys for this table + + ``GET /fixtures/facetable.json?_extra=primary_keys`` + + .. code-block:: json + + [ + "pk" + ] + +``display_columns`` + Column metadata used by the HTML table display + + Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns. + + .. code-block:: json + + [ + { + "name": "pk", + "sortable": true, + "is_pk": true, + "type": "INTEGER", + "notnull": 0 + }, + { + "name": "created", + "sortable": true, + "is_pk": false, + "type": "TEXT", + "notnull": 0, + "description": null, + "column_type": null, + "column_type_config": null + } + ] + +``display_rows`` + Row data formatted for the HTML table display + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + [ + {}, + { + "content": "Custom rendered HTML" + } + ] + +``debug`` + Extra debug information + +``request`` + Full information about the request + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/facetable.json?_size=1&_extra=query`` + + .. code-block:: json + + { + "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 2", + "params": {} + } + +``column_types`` + Column type assignments for this table + + .. code-block:: json + + {} + +``set_column_type_ui`` + Column type UI metadata for this table + +``metadata`` + Metadata about the table and database + + ``GET /fixtures/facetable.json?_extra=metadata`` + + .. code-block:: json + + { + "columns": {} + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/facetable.json?_extra=database`` + + .. code-block:: json + + "fixtures" + +``table`` + Table name + + ``GET /fixtures/facetable.json?_extra=table`` + + .. code-block:: json + + "facetable" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/facetable.json?_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``actions`` + Table or view actions made available by plugin hooks + +``filters`` + Filters object used by the HTML table interface + +``renderers`` + Alternative output renderers available for this table + + ``GET /fixtures/facetable.json?_extra=renderers`` + + .. code-block:: json + + { + "json": "/fixtures/facetable.json?_extra=renderers&_format=json&_labels=on" + } + +``custom_table_templates`` + Custom template names considered for this table + + ``GET /fixtures/facetable.json?_extra=custom_table_templates`` + + .. code-block:: json + + [ + "_table-fixtures-facetable.html", + "_table-table-fixtures-facetable.html", + "_table.html" + ] + +``sorted_facet_results`` + Facet results sorted for display + + ``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results`` + + .. code-block:: json + + [ + { + "name": "state", + "type": "column", + "hideable": true, + "toggle_url": "/fixtures/facetable.json?_extra=sorted_facet_results", + "results": [ + { + "value": "CA", + "label": "CA", + "count": 10, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=CA", + "selected": false + }, + { + "value": "MI", + "label": "MI", + "count": 4, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MI", + "selected": false + }, + { + "value": "MC", + "label": "MC", + "count": 1, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MC", + "selected": false + } + ], + "truncated": false + } + ] + +``table_definition`` + SQL definition for this table + + ``GET /fixtures/facetable.json?_extra=table_definition`` + + .. code-block:: json + + "CREATE TABLE facetable (\n pk integer primary key,\n created text,\n planet_int integer,\n on_earth integer,\n state text,\n _city_id integer,\n _neighborhood text,\n tags text,\n complex_array text,\n distinct_some_null,\n n text,\n FOREIGN KEY (\"_city_id\") REFERENCES [facet_cities](id)\n);" + +``view_definition`` + SQL definition for this view + + ``GET /fixtures/simple_view.json?_extra=view_definition`` + + .. code-block:: json + + "CREATE VIEW simple_view AS\n SELECT content, upper(content) AS upper_content FROM simple_primary_key;" + +``is_view`` + Whether this resource is a view instead of a table + + ``GET /fixtures/simple_view.json?_extra=is_view`` + + .. code-block:: json + + true + +``private`` + Whether this table is private to the current actor + + ``GET /fixtures/facetable.json?_extra=private`` + + .. code-block:: json + + false + +``expandable_columns`` + Foreign key columns that can be expanded with labels + + ``GET /fixtures/facetable.json?_extra=expandable_columns`` + + .. code-block:: json + + [ + [ + { + "column": "_city_id", + "other_table": "facet_cities", + "other_column": "id" + }, + "name" + ] + ] + +``form_hidden_args`` + Hidden form arguments used by the HTML table interface + + ``GET /fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args`` + + .. code-block:: json + + [ + [ + "_facet", + "state" + ], + [ + "_size", + "1" + ], + [ + "_extra", + "form_hidden_args" + ] + ] .. [[[end]]] diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index f07c3ba7..69ec6e5e 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -1,12 +1,20 @@ +import asyncio +import json +import pathlib +import tempfile +import textwrap + + def table_extras(cog): from datasette.extras import ExtraScope from datasette.views.table_extras import table_extra_registry - cog.out("\n.. list-table::\n") - cog.out(" :header-rows: 1\n\n") - cog.out(" * - Extra\n") - cog.out(" - Description\n") - for cls in table_extra_registry.public_classes_for_scope(ExtraScope.TABLE): + classes = table_extra_registry.public_classes_for_scope(ExtraScope.TABLE) + + live_examples = asyncio.run(_fetch_live_examples(classes)) + cog.out("\n") + for cls in classes: + example = cls.example description = cls.description or "" notes = [] if cls.expensive: @@ -15,6 +23,46 @@ def table_extras(cog): notes.append(cls.docs_note) if notes: description = "{} ({})".format(description, " ".join(notes)).strip() - cog.out(" * - ``{}``\n".format(cls.key())) - cog.out(" - {}\n".format(description)) - cog.out("\n") + + cog.out("``{}``\n".format(cls.key())) + cog.out(" {}\n\n".format(description)) + if example is None: + continue + + if example.path: + value = live_examples[(example.path, example.key or cls.key())] + cog.out(" ``GET {}``\n\n".format(example.path)) + else: + value = example.value + if example.note: + cog.out(" {}\n\n".format(example.note)) + cog.out(" .. code-block:: json\n\n") + cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) + cog.out("\n\n") + + +async def _fetch_live_examples(classes): + from datasette.app import Datasette + from datasette.fixtures import write_fixture_database + + examples = {} + with tempfile.TemporaryDirectory() as tmpdir: + db_path = pathlib.Path(tmpdir) / "fixtures.db" + write_fixture_database(db_path) + datasette = Datasette([str(db_path)], settings={"num_sql_threads": 1}) + try: + for cls in classes: + example = cls.example + if example is None or not example.path: + continue + key = example.key or cls.key() + response = await datasette.client.get(example.path) + assert response.status_code == 200, example.path + data = response.json() + assert key in data, "{} missing from {}".format(key, example.path) + examples[(example.path, key)] = data[key] + finally: + for db in datasette.databases.values(): + if not db.is_memory: + db.close() + return examples diff --git a/tests/test_docs.py b/tests/test_docs.py index 51caf595..784755e9 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -112,6 +112,16 @@ def test_table_filters_are_documented(documented_table_filters, subtests): assert f.key in documented_table_filters +def test_table_extra_examples_are_documented(): + from datasette.views.table_extras import CountExtra + + assert CountExtra.example.path == "/fixtures/facetable.json?_extra=count" + content = (docs_path / "json_api.rst").read_text() + section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + assert "GET /fixtures/facetable.json?_extra=count" in section + assert ".. code-block:: json" in section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 22f80b819625b9f6b5aa0661f58d97c89882a932 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:13:53 -0700 Subject: [PATCH 1144/1172] Clarify render_cell JSON extra example --- datasette/views/table_extras.py | 20 +++++++++++++++----- docs/json_api.rst | 26 +++++++++++++++++++------- tests/test_docs.py | 8 ++++++++ 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 0eefeaa9..b6e653c4 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -395,11 +395,21 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" example = ExtraExample( - value=[ - {}, - {"content": "Custom rendered HTML"}, - ], - note="Only columns whose rendered value differs from the default are included.", + value={ + "rows": [ + {"id": 1, "content": "hello"}, + {"id": 4, "content": "RENDER_CELL_DEMO"}, + ], + "render_cell": [ + {}, + {"content": "Custom rendered HTML"}, + ], + }, + note=( + "The ``render_cell`` array has one item per row, in the same order as " + "the ``rows`` array. Each object is keyed by column name. Only columns " + "whose rendered value differs from the default are included." + ), ) scopes = frozenset({ExtraScope.TABLE}) diff --git a/docs/json_api.rst b/docs/json_api.rst index d12a388e..24d59577 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -428,16 +428,28 @@ The available table extras are listed below. ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook - Only columns whose rendered value differs from the default are included. + The ``render_cell`` array has one item per row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. .. code-block:: json - [ - {}, - { - "content": "Custom rendered HTML" - } - ] + { + "rows": [ + { + "id": 1, + "content": "hello" + }, + { + "id": 4, + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + {}, + { + "content": "Custom rendered HTML" + } + ] + } ``debug`` Extra debug information diff --git a/tests/test_docs.py b/tests/test_docs.py index 784755e9..c4e0a849 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -122,6 +122,14 @@ def test_table_extra_examples_are_documented(): assert ".. code-block:: json" in section +def test_render_cell_extra_example_explains_row_and_column_mapping(): + content = (docs_path / "json_api.rst").read_text() + section = content.split("``render_cell``")[-1].split("``query``")[0] + assert "same order as the ``rows`` array" in section + assert '"rows": [' in section + assert '"render_cell": [' in section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 0fa872d43842d87af9e7b8c193f90addfcf164ba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:20:06 -0700 Subject: [PATCH 1145/1172] Add debug and request JSON extra examples --- datasette/views/table_extras.py | 2 ++ docs/json_api.rst | 31 +++++++++++++++++++++++++++++++ tests/test_docs.py | 13 +++++++++++++ 3 files changed, 46 insertions(+) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index b6e653c4..e888ee9f 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -308,6 +308,7 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" + example = ExtraExample("/fixtures/facetable.json?_extra=debug") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -321,6 +322,7 @@ class DebugExtra(Extra): class RequestExtra(Extra): description = "Full information about the request" + example = ExtraExample("/fixtures/facetable.json?_extra=request") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 24d59577..d418d16c 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -454,9 +454,40 @@ The available table extras are listed below. ``debug`` Extra debug information + ``GET /fixtures/facetable.json?_extra=debug`` + + .. code-block:: json + + { + "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", + "url_vars": { + "database": "fixtures", + "table": "facetable", + "format": "json" + }, + "nofacet": null, + "nosuggest": null + } + ``request`` Full information about the request + ``GET /fixtures/facetable.json?_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/facetable.json?_extra=request", + "path": "/fixtures/facetable.json", + "full_path": "/fixtures/facetable.json?_extra=request", + "host": "localhost", + "args": { + "_extra": [ + "request" + ] + } + } + ``query`` Details of the underlying SQL query diff --git a/tests/test_docs.py b/tests/test_docs.py index c4e0a849..3aa67730 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -130,6 +130,19 @@ def test_render_cell_extra_example_explains_row_and_column_mapping(): assert '"render_cell": [' in section +def test_debug_and_request_extra_examples_are_documented(): + content = (docs_path / "json_api.rst").read_text() + section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + + debug_section = section.split("``debug``")[-1].split("``request``")[0] + assert "GET /fixtures/facetable.json?_extra=debug" in debug_section + assert '"url_vars": {' in debug_section + + request_section = section.split("``request``")[-1].split("``query``")[0] + assert "GET /fixtures/facetable.json?_extra=request" in request_section + assert '"full_path":' in request_section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 4d6daa175a67c4c6e895fe8b32ae051b1e9136a7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 02:56:27 -0700 Subject: [PATCH 1146/1172] Add row and query JSON extras --- datasette/extras.py | 7 + datasette/views/database.py | 46 +++- datasette/views/row.py | 67 ++---- datasette/views/table_extras.py | 258 ++++++++++++++++++--- docs/json_api.rst | 386 +++++++++++++++++++++++++++++++- docs/json_api_doc.py | 141 ++++++++---- tests/test_api.py | 22 ++ tests/test_docs.py | 16 +- tests/test_table_api.py | 49 ++++ 9 files changed, 862 insertions(+), 130 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index d867f26c..f655e517 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -16,6 +16,8 @@ def extra_names_from_request(request): class ExtraScope(Enum): TABLE = "table" + ROW = "row" + QUERY = "query" @dataclass(frozen=True) @@ -46,11 +48,16 @@ class Provider: class Extra(Provider): description: ClassVar[str | None] = None example: ClassVar[ExtraExample | None] = None + examples: ClassVar[dict[ExtraScope, ExtraExample | list[ExtraExample]]] = {} public: ClassVar[bool] = True stable: ClassVar[bool] = True expensive: ClassVar[bool] = False docs_note: ClassVar[str | None] = None + @classmethod + def example_for_scope(cls, scope): + return cls.examples.get(scope, cls.example) + @classmethod def documentation(cls): return { diff --git a/datasette/views/database.py b/datasette/views/database.py index a1647ca9..96a58758 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -11,6 +11,7 @@ import sqlite_utils import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent +from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import stored_query_to_dict @@ -38,6 +39,11 @@ from datasette.plugins import pm from .base import BaseView, DatasetteError, View, _error, stream_csv from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns +from .table_extras import ( + QueryExtraContext, + resolve_query_extras, + table_extra_registry, +) from . import Context @@ -692,6 +698,34 @@ class QueryView(View): except DatasetteError: raise + extras = extra_names_from_request(request) + metadata = None + data = {"ok": True, "rows": rows, "columns": columns} + if extras: + metadata = await datasette.get_database_metadata(database) + if stored_query: + metadata = stored_query_to_dict(stored_query) + metadata.pop("source", None) + query_extra_context = QueryExtraContext( + datasette=datasette, + request=request, + db=db, + database_name=database, + private=private, + rows=rows, + columns=columns, + sql=sql, + params=params_for_query, + query_name=stored_query.name if stored_query else None, + stored_query=stored_query, + stored_query_write=stored_query_write, + error=query_error, + metadata=metadata, + extras=extras, + extra_registry=table_extra_registry, + ) + data.update(await resolve_query_extras(extras, query_extra_context)) + # Handle formats from plugins if format_ == "csv": if not sql: @@ -721,7 +755,7 @@ class QueryView(View): error=query_error, # These will be deprecated in Datasette 1.0: args=request.args, - data={"ok": True, "rows": rows, "columns": columns}, + data=data, ) if asyncio.iscoroutine(result): result = await result @@ -770,11 +804,11 @@ class QueryView(View): ) } ) - metadata = await datasette.get_database_metadata(database) - if stored_query: - metadata = stored_query_to_dict(stored_query) - metadata.pop("source", None) - + if metadata is None: + metadata = await datasette.get_database_metadata(database) + if stored_query: + metadata = stored_query_to_dict(stored_query) + metadata.pop("source", None) renderers = {} for key, (_, can_render) in datasette.renderers.items(): it_can_render = call_with_supported_arguments( diff --git a/datasette/views/row.py b/datasette/views/row.py index 4eacfe49..3fe213d7 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -15,6 +15,7 @@ import json import markupsafe import sqlite_utils from .table import display_columns_and_rows, _get_extras +from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry class RowView(DataView): @@ -172,52 +173,26 @@ class RowView(DataView): extras.add("foreign_key_tables") # Process extras - if "foreign_key_tables" in extras: - data["foreign_key_tables"] = await self.foreign_key_tables( - database, table, pk_values - ) - - if "render_cell" in extras: - # Call render_cell plugin hook for each cell - ct_map = await self.ds.get_column_types(database, table) - rendered_rows = [] - for row in rows: - rendered_row = {} - for value, column in zip(row, columns): - ct = ct_map.get(column) - plugin_display_value = None - # Try column type render_cell first - if ct: - candidate = await ct.render_cell( - value=value, - column=column, - table=table, - database=database, - datasette=self.ds, - request=request, - ) - if candidate is not None: - plugin_display_value = candidate - if plugin_display_value is None: - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=table, - pks=resolved.pks, - database=database, - datasette=self.ds, - request=request, - column_type=ct, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - rendered_row[column] = str(plugin_display_value) - rendered_rows.append(rendered_row) - data["render_cell"] = rendered_rows + row_extra_context = RowExtraContext( + datasette=self.ds, + request=request, + resolved=resolved, + db=db, + database_name=database, + table_name=table, + private=private, + rows=rows, + columns=columns, + results_description=results.description, + pks=pks, + pk_values=pk_values, + sql=resolved.sql, + params=resolved.params, + extras=extras, + extra_registry=table_extra_registry, + foreign_key_tables=self.foreign_key_tables, + ) + data.update(await resolve_row_extras(extras, row_extra_context)) return ( data, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index e888ee9f..ec104be3 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -8,6 +8,7 @@ from datasette.resources import TableResource from datasette.utils import ( await_me_maybe, call_with_supported_arguments, + named_parameters as derive_named_parameters, path_with_added_args, path_with_format, path_with_removed_args, @@ -52,6 +53,50 @@ class TableExtraContext: extra_registry: ExtraRegistry display_columns_and_rows: object run_sequential: object + scope: ExtraScope = ExtraScope.TABLE + + +@dataclass(frozen=True) +class RowExtraContext: + datasette: object + request: object + resolved: object + db: object + database_name: str + table_name: str + private: bool + rows: list + columns: list + results_description: list + pks: list + pk_values: list + sql: str + params: dict + extras: set + extra_registry: ExtraRegistry + foreign_key_tables: object + scope: ExtraScope = ExtraScope.ROW + + +@dataclass(frozen=True) +class QueryExtraContext: + datasette: object + request: object + db: object + database_name: str + private: bool + rows: list + columns: list + sql: str | None + params: dict + query_name: str | None + stored_query: object + stored_query_write: bool + error: str | None + metadata: dict + extras: set + extra_registry: ExtraRegistry + scope: ExtraScope = ExtraScope.QUERY class CountSqlExtra(Extra): @@ -245,7 +290,15 @@ class NextUrlExtra(Extra): class ColumnsExtra(Extra): description = "Column names returned by this query" example = ExtraExample("/fixtures/facetable.json?_extra=columns") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=columns" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=columns" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.columns @@ -263,7 +316,12 @@ class AllColumnsExtra(Extra): class PrimaryKeysExtra(Extra): description = "Primary keys for this table" example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=primary_keys" + ) + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): return context.pks @@ -309,21 +367,49 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" example = ExtraExample("/fixtures/facetable.json?_extra=debug") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=debug" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=debug" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - return { - "resolved": repr(context.resolved), + debug = { "url_vars": context.request.url_vars, - "nofacet": context.nofacet, - "nosuggest": context.nosuggest, } + if context.scope == ExtraScope.TABLE: + debug["resolved"] = repr(context.resolved) + elif context.scope == ExtraScope.ROW: + debug["resolved"] = { + "table": context.table_name, + "sql": context.sql, + "params": context.params, + "pks": context.pks, + "pk_values": context.pk_values, + } + if hasattr(context, "nofacet"): + debug["nofacet"] = context.nofacet + if hasattr(context, "nosuggest"): + debug["nosuggest"] = context.nosuggest + return debug class RequestExtra(Extra): description = "Full information about the request" example = ExtraExample("/fixtures/facetable.json?_extra=request") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=request" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=request" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return { @@ -413,15 +499,48 @@ class RenderCellExtra(Extra): "whose rendered value differs from the default are included." ), ) - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + value={ + "rows": [{"id": 4, "content": "RENDER_CELL_DEMO"}], + "render_cell": [{"content": "Custom rendered HTML"}], + }, + note=( + "The ``render_cell`` array has one item for the requested row. " + "The object is keyed by column name. Only columns whose rendered " + "value differs from the default are included." + ), + ), + ExtraScope.QUERY: ExtraExample( + value={ + "rows": [{"content": "RENDER_CELL_DEMO"}], + "render_cell": [{"content": "Custom rendered HTML"}], + }, + note=( + "The ``render_cell`` array has one item per query result row, in " + "the same order as the ``rows`` array. Each object is keyed by " + "column name. Only columns whose rendered value differs from the " + "default are included." + ), + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + table_name = getattr(context, "table_name", None) + is_view = getattr(context, "is_view", False) + pks = getattr(context, "pks", []) pks_for_display = ( - context.pks if context.pks else (["rowid"] if not context.is_view else []) + pks if pks else (["rowid"] if table_name and not is_view else []) ) - col_names = [col[0] for col in context.results_description] - ct_map = await context.datasette.get_column_types( - context.database_name, context.table_name + if hasattr(context, "results_description"): + col_names = [col[0] for col in context.results_description] + else: + col_names = context.columns + ct_map = ( + await context.datasette.get_column_types(context.database_name, table_name) + if table_name + else {} ) rendered_rows = [] for row in context.rows: @@ -433,7 +552,7 @@ class RenderCellExtra(Extra): candidate = await ct.render_cell( value=value, column=column, - table=context.table_name, + table=table_name, database=context.database_name, datasette=context.datasette, request=context.request, @@ -445,7 +564,7 @@ class RenderCellExtra(Extra): row=row, value=value, column=column, - table=context.table_name, + table=table_name, pks=pks_for_display, database=context.database_name, datasette=context.datasette, @@ -465,19 +584,36 @@ class RenderCellExtra(Extra): class QueryExtra(Extra): description = "Details of the underlying SQL query" example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=query" + ), + ExtraScope.QUERY: [ + ExtraExample("/fixtures/-/query.json?sql=select+1+as+one&_extra=query"), + ExtraExample("/fixtures/neighborhood_search.json?text=town&_extra=query"), + ], + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + params = context.params + if context.scope == ExtraScope.QUERY and context.sql: + parameter_names = set(derive_named_parameters(context.sql)) + params = { + key: value + for key, value in dict(context.params).items() + if key in parameter_names + } return { "sql": context.sql, - "params": context.params, + "params": params, } class ColumnTypesExtra(Extra): description = "Column type assignments for this table" example = ExtraExample(value={}) - scopes = frozenset({ExtraScope.TABLE}) + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): ct_map = await context.datasette.get_column_types( @@ -544,11 +680,22 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): - description = "Metadata about the table and database" + description = "Metadata about the table, database or stored query" example = ExtraExample("/fixtures/facetable.json?_extra=metadata") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=metadata" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/neighborhood_search.json?text=town&_extra=metadata" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + if context.scope == ExtraScope.QUERY: + return context.metadata + tablemetadata = await context.datasette.get_resource_metadata( context.database_name, context.table_name ) @@ -572,7 +719,15 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): description = "Database name" example = ExtraExample("/fixtures/facetable.json?_extra=database") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=database" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=database" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.database_name @@ -581,7 +736,10 @@ class DatabaseExtra(Extra): class TableExtra(Extra): description = "Table name" example = ExtraExample("/fixtures/facetable.json?_extra=table") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample("/fixtures/simple_primary_key/1.json?_extra=table") + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): return context.table_name @@ -590,7 +748,15 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" example = ExtraExample("/fixtures/facetable.json?_extra=database_color") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=database_color" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=database_color" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.db.color @@ -703,6 +869,8 @@ class RenderersExtra(Extra): url_labels_extra = {} if expandable_columns: url_labels_extra = {"_labels": "on"} + table_name = getattr(context, "table_name", None) + view_name = "table" if context.scope == ExtraScope.TABLE else "database" for key, (_, can_render) in context.datasette.renderers.items(): it_can_render = call_with_supported_arguments( can_render, @@ -710,11 +878,11 @@ class RenderersExtra(Extra): columns=context.columns or [], rows=context.rows or [], sql=query.get("sql", None), - query_name=None, + query_name=getattr(context, "query_name", None), database=context.database_name, - table=context.table_name, + table=table_name, request=context.request, - view_name="table", + view_name=view_name, ) it_can_render = await await_me_maybe(it_can_render) if it_can_render: @@ -730,9 +898,17 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): - description = "Whether this table is private to the current actor" + description = "Whether this resource is private to the current actor" example = ExtraExample("/fixtures/facetable.json?_extra=private") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=private" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.private @@ -752,14 +928,27 @@ class ExpandableColumnsExtra(Extra): return expandables +class ForeignKeyTablesExtra(Extra): + description = "Tables that link to this row using foreign keys" + example = ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" + ) + scopes = frozenset({ExtraScope.ROW}) + + async def resolve(self, context): + return await context.foreign_key_tables( + context.database_name, context.table_name, context.pk_values + ) + + class ExtrasExtra(Extra): description = "Available ?_extra= blocks" - scopes = frozenset({ExtraScope.TABLE}) + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): all_extras = [ (cls.key(), cls.description) - for cls in context.extra_registry.public_classes_for_scope(ExtraScope.TABLE) + for cls in context.extra_registry.public_classes_for_scope(context.scope) ] return [ { @@ -850,6 +1039,7 @@ TABLE_EXTRA_CLASSES = [ IsViewExtra, PrivateExtra, ExpandableColumnsExtra, + ForeignKeyTablesExtra, FormHiddenArgsExtra, ] @@ -859,3 +1049,11 @@ table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) async def resolve_table_extras(extras, context): return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) + + +async def resolve_row_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.ROW) + + +async def resolve_query_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.QUERY) diff --git a/docs/json_api.rst b/docs/json_api.rst index d418d16c..379d26a0 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -237,23 +237,26 @@ query string arguments: .. _json_api_extra: -Expanding table JSON responses ------------------------------- +Expanding JSON responses +------------------------ -Table JSON responses can be expanded with one or more ``?_extra=`` parameters. +Table, row and query JSON responses can be expanded with one or more ``?_extra=`` parameters. These can be repeated or comma-separated: :: ?_extra=columns&_extra=count,next_url -The available table extras are listed below. - .. [[[cog from json_api_doc import table_extras table_extras(cog) .. ]]] +Table JSON responses +~~~~~~~~~~~~~~~~~~~~ + +The available table extras are listed below. + ``count`` Total count of rows matching these filters (May execute additional queries.) @@ -459,12 +462,12 @@ The available table extras are listed below. .. code-block:: json { - "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", "url_vars": { "database": "fixtures", "table": "facetable", "format": "json" }, + "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", "nofacet": null, "nosuggest": null } @@ -511,7 +514,7 @@ The available table extras are listed below. Column type UI metadata for this table ``metadata`` - Metadata about the table and database + Metadata about the table, database or stored query ``GET /fixtures/facetable.json?_extra=metadata`` @@ -649,7 +652,7 @@ The available table extras are listed below. true ``private`` - Whether this table is private to the current actor + Whether this resource is private to the current actor ``GET /fixtures/facetable.json?_extra=private`` @@ -697,6 +700,373 @@ The available table extras are listed below. ] ] +Row JSON responses +~~~~~~~~~~~~~~~~~~ + +The following extras are available for row JSON responses. + +``columns`` + Column names returned by this query + + ``GET /fixtures/simple_primary_key/1.json?_extra=columns`` + + .. code-block:: json + + [ + "id", + "content" + ] + +``primary_keys`` + Primary keys for this table + + ``GET /fixtures/simple_primary_key/1.json?_extra=primary_keys`` + + .. code-block:: json + + [ + "id" + ] + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + The ``render_cell`` array has one item for the requested row. The object is keyed by column name. Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + { + "rows": [ + { + "id": 4, + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + { + "content": "Custom rendered HTML" + } + ] + } + +``debug`` + Extra debug information + + ``GET /fixtures/simple_primary_key/1.json?_extra=debug`` + + .. code-block:: json + + { + "url_vars": { + "database": "fixtures", + "table": "simple_primary_key", + "pks": "1", + "format": "json" + }, + "resolved": { + "table": "simple_primary_key", + "sql": "select * from simple_primary_key where \"id\"=:p0", + "params": { + "p0": "1" + }, + "pks": [ + "id" + ], + "pk_values": [ + "1" + ] + } + } + +``request`` + Full information about the request + + ``GET /fixtures/simple_primary_key/1.json?_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/simple_primary_key/1.json?_extra=request", + "path": "/fixtures/simple_primary_key/1.json", + "full_path": "/fixtures/simple_primary_key/1.json?_extra=request", + "host": "localhost", + "args": { + "_extra": [ + "request" + ] + } + } + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/simple_primary_key/1.json?_extra=query`` + + .. code-block:: json + + { + "sql": "select * from simple_primary_key where \"id\"=:p0", + "params": { + "p0": "1" + } + } + +``column_types`` + Column type assignments for this table + + .. code-block:: json + + {} + +``metadata`` + Metadata about the table, database or stored query + + ``GET /fixtures/simple_primary_key/1.json?_extra=metadata`` + + .. code-block:: json + + { + "columns": {} + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/simple_primary_key/1.json?_extra=database`` + + .. code-block:: json + + "fixtures" + +``table`` + Table name + + ``GET /fixtures/simple_primary_key/1.json?_extra=table`` + + .. code-block:: json + + "simple_primary_key" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/simple_primary_key/1.json?_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``private`` + Whether this resource is private to the current actor + + ``GET /fixtures/simple_primary_key/1.json?_extra=private`` + + .. code-block:: json + + false + +``foreign_key_tables`` + Tables that link to this row using foreign keys + + ``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables`` + + .. code-block:: json + + [ + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f1", + "count": 1, + "link": "/fixtures/complex_foreign_keys?f1=1" + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f2", + "count": 0, + "link": "/fixtures/complex_foreign_keys?f2=1" + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f3", + "count": 1, + "link": "/fixtures/complex_foreign_keys?f3=1" + }, + { + "other_table": "foreign_key_references", + "column": "id", + "other_column": "foreign_key_with_blank_label", + "count": 0, + "link": "/fixtures/foreign_key_references?foreign_key_with_blank_label=1" + }, + { + "other_table": "foreign_key_references", + "column": "id", + "other_column": "foreign_key_with_label", + "count": 1, + "link": "/fixtures/foreign_key_references?foreign_key_with_label=1" + } + ] + +Query JSON responses +~~~~~~~~~~~~~~~~~~~~ + +The following extras are available for arbitrary SQL query responses and stored, named query responses. + +``columns`` + Column names returned by this query + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=columns`` + + .. code-block:: json + + [ + "one" + ] + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + The ``render_cell`` array has one item per query result row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + { + "rows": [ + { + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + { + "content": "Custom rendered HTML" + } + ] + } + +``debug`` + Extra debug information + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=debug`` + + .. code-block:: json + + { + "url_vars": { + "database": "fixtures", + "format": "json" + } + } + +``request`` + Full information about the request + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/-/query.json?sql=select+1+as+one&_extra=request", + "path": "/fixtures/-/query.json", + "full_path": "/fixtures/-/query.json?sql=select+1+as+one&_extra=request", + "host": "localhost", + "args": { + "sql": [ + "select 1 as one" + ], + "_extra": [ + "request" + ] + } + } + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query`` + + .. code-block:: json + + { + "sql": "select 1 as one", + "params": {} + } + + ``GET /fixtures/neighborhood_search.json?text=town&_extra=query`` + + .. code-block:: json + + { + "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities\n on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n", + "params": { + "text": "town" + } + } + +``metadata`` + Metadata about the table, database or stored query + + ``GET /fixtures/neighborhood_search.json?text=town&_extra=metadata`` + + .. code-block:: json + + { + "database": "fixtures", + "name": "neighborhood_search", + "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities\n on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n", + "title": "Search neighborhoods", + "description": null, + "description_html": null, + "hide_sql": false, + "fragment": null, + "params": [], + "parameters": [], + "is_write": false, + "is_private": false, + "is_trusted": true, + "owner_id": null, + "on_success_message": null, + "on_success_message_sql": null, + "on_success_redirect": null, + "on_error_message": null, + "on_error_redirect": null + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database`` + + .. code-block:: json + + "fixtures" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``private`` + Whether this resource is private to the current actor + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=private`` + + .. code-block:: json + + false + .. [[[end]]] .. _table_arguments: diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index 69ec6e5e..44ef4a42 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -9,39 +9,80 @@ def table_extras(cog): from datasette.extras import ExtraScope from datasette.views.table_extras import table_extra_registry - classes = table_extra_registry.public_classes_for_scope(ExtraScope.TABLE) + scopes = [ + ( + ExtraScope.TABLE, + "Table JSON responses", + "The available table extras are listed below.", + ), + ( + ExtraScope.ROW, + "Row JSON responses", + "The following extras are available for row JSON responses.", + ), + ( + ExtraScope.QUERY, + "Query JSON responses", + ( + "The following extras are available for arbitrary SQL query " + "responses and stored, named query responses." + ), + ), + ] + classes_by_scope = [ + (scope, heading, intro, table_extra_registry.public_classes_for_scope(scope)) + for scope, heading, intro in scopes + ] - live_examples = asyncio.run(_fetch_live_examples(classes)) + live_examples = asyncio.run( + _fetch_live_examples( + [ + (scope, cls) + for scope, _, _, classes in classes_by_scope + for cls in classes + ] + ) + ) cog.out("\n") - for cls in classes: - example = cls.example - description = cls.description or "" - notes = [] - if cls.expensive: - notes.append("May execute additional queries.") - if cls.docs_note: - notes.append(cls.docs_note) - if notes: - description = "{} ({})".format(description, " ".join(notes)).strip() + for scope, heading, intro, classes in classes_by_scope: + cog.out("{}\n{}\n\n".format(heading, "~" * len(heading))) + cog.out("{}\n\n".format(intro)) + for cls in classes: + examples = _examples_for_scope(cls, scope) + description = cls.description or "" + notes = [] + if cls.expensive: + notes.append("May execute additional queries.") + if cls.docs_note: + notes.append(cls.docs_note) + if notes: + description = "{} ({})".format(description, " ".join(notes)).strip() - cog.out("``{}``\n".format(cls.key())) - cog.out(" {}\n\n".format(description)) - if example is None: - continue - - if example.path: - value = live_examples[(example.path, example.key or cls.key())] - cog.out(" ``GET {}``\n\n".format(example.path)) - else: - value = example.value - if example.note: - cog.out(" {}\n\n".format(example.note)) - cog.out(" .. code-block:: json\n\n") - cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) - cog.out("\n\n") + cog.out("``{}``\n".format(cls.key())) + cog.out(" {}\n\n".format(description)) + for example in examples: + if example.path: + value = live_examples[(example.path, example.key or cls.key())] + cog.out(" ``GET {}``\n\n".format(example.path)) + else: + value = example.value + if example.note: + cog.out(" {}\n\n".format(example.note)) + cog.out(" .. code-block:: json\n\n") + cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) + cog.out("\n\n") -async def _fetch_live_examples(classes): +def _examples_for_scope(cls, scope): + examples = cls.example_for_scope(scope) + if examples is None: + return [] + if isinstance(examples, list): + return examples + return [examples] + + +async def _fetch_live_examples(scoped_classes): from datasette.app import Datasette from datasette.fixtures import write_fixture_database @@ -49,18 +90,40 @@ async def _fetch_live_examples(classes): with tempfile.TemporaryDirectory() as tmpdir: db_path = pathlib.Path(tmpdir) / "fixtures.db" write_fixture_database(db_path) - datasette = Datasette([str(db_path)], settings={"num_sql_threads": 1}) + datasette = Datasette( + [str(db_path)], + settings={"num_sql_threads": 1}, + config={ + "databases": { + "fixtures": { + "queries": { + "neighborhood_search": { + "sql": textwrap.dedent(""" + select _neighborhood, facet_cities.name, state + from facetable + join facet_cities + on facetable._city_id = facet_cities.id + where _neighborhood like '%' || :text || '%' + order by _neighborhood; + """), + "title": "Search neighborhoods", + } + } + } + } + }, + ) try: - for cls in classes: - example = cls.example - if example is None or not example.path: - continue - key = example.key or cls.key() - response = await datasette.client.get(example.path) - assert response.status_code == 200, example.path - data = response.json() - assert key in data, "{} missing from {}".format(key, example.path) - examples[(example.path, key)] = data[key] + for scope, cls in scoped_classes: + for example in _examples_for_scope(cls, scope): + if not example.path: + continue + key = example.key or cls.key() + response = await datasette.client.get(example.path) + assert response.status_code == 200, example.path + data = response.json() + assert key in data, "{} missing from {}".format(key, example.path) + examples[(example.path, key)] = data[key] finally: for db in datasette.databases.values(): if not db.is_memory: diff --git a/tests/test_api.py b/tests/test_api.py index f6187529..e1385b6f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -426,6 +426,28 @@ async def test_row_foreign_key_tables(ds_client): ] +@pytest.mark.asyncio +async def test_row_extras(ds_client): + response = await ds_client.get( + "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables" + ) + assert response.status_code == 200 + data = response.json() + assert data["database"] == "fixtures" + assert data["table"] == "simple_primary_key" + assert data["primary_keys"] == ["id"] + assert data["query"]["sql"] == 'select * from simple_primary_key where "id"=:p0' + assert data["query"]["params"] == {"p0": "1"} + assert data["request"]["path"] == "/fixtures/simple_primary_key/1.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "table": "simple_primary_key", + "pks": "1", + "format": "json", + } + assert len(data["foreign_key_tables"]) == 5 + + @pytest.mark.asyncio async def test_row_extra_render_cell(): """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook on row pages""" diff --git a/tests/test_docs.py b/tests/test_docs.py index 3aa67730..13b3a549 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -132,7 +132,7 @@ def test_render_cell_extra_example_explains_row_and_column_mapping(): def test_debug_and_request_extra_examples_are_documented(): content = (docs_path / "json_api.rst").read_text() - section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + section = content.split("Table JSON responses")[-1].split("Row JSON responses")[0] debug_section = section.split("``debug``")[-1].split("``request``")[0] assert "GET /fixtures/facetable.json?_extra=debug" in debug_section @@ -143,6 +143,20 @@ def test_debug_and_request_extra_examples_are_documented(): assert '"full_path":' in request_section +def test_row_and_query_extra_sections_are_documented(): + content = (docs_path / "json_api.rst").read_text() + assert "Row JSON responses" in content + assert ( + "``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables``" + in content + ) + assert "Query JSON responses" in content + assert "``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query``" in content + assert ( + "``GET /fixtures/neighborhood_search.json?text=town&_extra=query``" in content + ) + + @pytest.fixture(scope="session") def documented_labels(): labels = set() diff --git a/tests/test_table_api.py b/tests/test_table_api.py index eeb3dc8b..388e3979 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -68,6 +68,55 @@ async def test_table_shape_arrayfirst(ds_client): ] +@pytest.mark.asyncio +async def test_query_extras_for_arbitrary_sql(ds_client): + response = await ds_client.get( + "/fixtures/-/query.json?" + + urllib.parse.urlencode( + { + "sql": "select 1 as one", + "_extra": "columns,database,query,request,debug", + } + ) + ) + assert response.status_code == 200 + data = response.json() + assert data["rows"] == [{"one": 1}] + assert data["columns"] == ["one"] + assert data["database"] == "fixtures" + assert data["query"]["sql"] == "select 1 as one" + assert data["request"]["path"] == "/fixtures/-/query.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "format": "json", + } + + +@pytest.mark.asyncio +async def test_query_extras_for_stored_query(ds_client): + response = await ds_client.get( + "/fixtures/neighborhood_search.json?" + + urllib.parse.urlencode( + { + "text": "town", + "_extra": "columns,database,query,request,debug", + } + ) + ) + assert response.status_code == 200 + data = response.json() + assert data["columns"] == ["_neighborhood", "name", "state"] + assert data["database"] == "fixtures" + assert data["query"]["sql"].strip().startswith("select _neighborhood") + assert data["query"]["params"]["text"] == "town" + assert data["request"]["path"] == "/fixtures/neighborhood_search.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "table": "neighborhood_search", + "format": "json", + } + + @pytest.mark.asyncio async def test_table_shape_objects(ds_client): response = await ds_client.get("/fixtures/simple_primary_key.json?_shape=objects") From d8605ef4c2c054610d2f4fbf1c00d182afa617e4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 19:58:00 -0700 Subject: [PATCH 1147/1172] Fix execute_isolated_fn() against immutable databases execute_isolated_fn() always opened its temporary connection with write=True, which is not allowed for immutable databases - so APIs that rely on it, like SQL analysis when storing a query, failed. An immutable database can never receive writes, so there is no write queue to block: in that case the function now opens a read-only connection and runs it on the executor, bypassing the write thread entirely. Mutable databases keep the existing write-thread behavior. Also fixed a latent bug in the write thread where a connect() failure for an isolated task would crash the thread instead of delivering the exception back to the caller. Closes #2768 Co-Authored-By: Claude Fable 5 --- datasette/database.py | 46 +++++++++++++++++++------------- tests/test_internals_database.py | 33 +++++++++++++++++++++++ tests/test_queries.py | 34 ++++++++++++++++++++++- 3 files changed, 94 insertions(+), 19 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 0a32442c..6cd5d11e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -298,13 +298,14 @@ class Database: async def execute_isolated_fn(self, fn): self._check_not_closed() - # Open a new connection just for the duration of this function + # Open a new connection just for the duration of this function, # blocking the write queue to avoid any writes occurring during it - if self.ds.executor is None: - # non-threaded mode - isolated_connection = self.connect(write=True) + write = self.is_mutable + + def _run(): + isolated_connection = self.connect(write=write) try: - result = fn(isolated_connection) + return fn(isolated_connection) finally: isolated_connection.close() try: @@ -312,10 +313,18 @@ class Database: except ValueError: # Was probably a memory connection pass - return result - else: - # Threaded mode - send to write thread - return await self._send_to_write_thread(fn, isolated_connection=True) + + if self.ds.executor is None: + # non-threaded mode + return _run() + if not write: + # Immutable database - no writes can ever occur, so there is no + # write queue to block; run against a fresh read-only connection + return await asyncio.get_running_loop().run_in_executor( + self.ds.executor, _run + ) + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) async def analyze_sql(self, sql, params=None) -> SQLAnalysis: self._check_not_closed() @@ -449,20 +458,21 @@ class Database: if conn_exception is not None: exception = conn_exception elif task.isolated_connection: - isolated_connection = self.connect(write=True) try: - result = task.fn(isolated_connection) + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass except Exception as e: sys.stderr.write("{}\n".format(e)) sys.stderr.flush() exception = e - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass else: try: if task.transaction: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index bb209649..bad4e8ca 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -863,6 +863,39 @@ async def test_execute_isolated(db, disable_threads): assert not await db.execute_isolated_fn(table_exists_checker("created_by_isolated")) +@pytest.mark.asyncio +async def test_execute_isolated_connect_failure_does_not_kill_write_thread(): + # A connect() failure for an isolated task should be returned to the + # caller as an exception, not crash the write thread + class ConnectError(Exception): + pass + + ds = Datasette(memory=True) + db = ds.add_memory_database("test_isolated_connect_failure") + # Start the write thread with a healthy dedicated write connection + await db.execute_write("create table dogs (id integer primary key)") + + original_connect = db.connect + + def broken_connect(write=False): + raise ConnectError("Could not connect") + + db.connect = broken_connect + try: + with pytest.raises(ConnectError): + await asyncio.wait_for(db.execute_isolated_fn(lambda conn: None), timeout=2) + finally: + db.connect = original_connect + + # Write thread should still be alive and processing tasks + assert db._write_thread.is_alive() + await db.execute_write("insert into dogs (id) values (1)") + count = await db.execute_isolated_fn( + lambda conn: conn.execute("select count(*) from dogs").fetchone()[0] + ) + assert count == 1 + + @pytest.mark.asyncio async def test_analyze_sql(): ds = Datasette(memory=True) diff --git a/tests/test_queries.py b/tests/test_queries.py index 6e9bcbdb..0354f73a 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -9,7 +9,7 @@ from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, StoredQueryPage from datasette.utils.asgi import Forbidden -from datasette.utils.sqlite import supports_returning +from datasette.utils.sqlite import sqlite3, supports_returning requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" @@ -593,6 +593,38 @@ async def test_query_store_api_creates_read_only_query(): assert data["query"]["owner_id"] == "root" +@pytest.mark.asyncio +async def test_query_store_api_creates_query_for_immutable_database(tmp_path): + db_path = tmp_path / "immutable.db" + conn = sqlite3.connect(str(db_path)) + conn.execute("create table dogs (id integer primary key, name text)") + conn.commit() + conn.close() + + ds = Datasette([], immutables=[str(db_path)], default_deny=True) + ds.root_enabled = True + await ds.invoke_startup() + + response = await ds.client.post( + "/immutable/-/queries/store", + actor={"id": "root"}, + json={ + "query": { + "name": "by_name", + "sql": "select * from dogs where name = :name", + } + }, + ) + + ds.close() + assert response.status_code == 201 + data = response.json() + assert data["ok"] is True + assert data["query"]["name"] == "by_name" + assert data["query"]["parameters"] == ["name"] + assert data["query"]["is_write"] is False + + @pytest.mark.asyncio async def test_query_list_and_definition_api(): ds = Datasette(memory=True) From 3c1012dcc2995d184ea24fe70e8ccd6580592aff Mon Sep 17 00:00:00 2001 From: Viraat Das Date: Wed, 10 Jun 2026 20:15:03 -0700 Subject: [PATCH 1148/1172] Fix write query failing when a named parameter is called :sql (#2765) Closes #2761 --- .../templates/_sql_parameter_scripts.html | 34 ++++++++++++----- datasette/templates/_sql_parameters.html | 5 ++- datasette/views/execute_write.py | 2 + datasette/views/query_helpers.py | 14 ++++--- tests/test_api_write.py | 38 +++++++++++++++++++ tests/test_html.py | 4 +- tests/test_stored_queries.py | 2 +- 7 files changed, 79 insertions(+), 20 deletions(-) diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html index 159a141c..9b83889e 100644 --- a/datasette/templates/_sql_parameter_scripts.html +++ b/datasette/templates/_sql_parameter_scripts.html @@ -27,16 +27,20 @@ window.datasetteSqlParameters = (() => { manager.section .querySelectorAll("[data-parameter-control]") .forEach((control) => { - manager.parameterState.set(control.name, controlState(control)); + manager.parameterState.set( + control.dataset.parameterName, + controlState(control) + ); }); } - function createControl(parameter, id, state) { + function createControl(parameter, id, state, namePrefix) { const control = document.createElement(state.expanded ? "textarea" : "input"); control.id = id; - control.name = parameter; + control.name = `${namePrefix || ""}${parameter}`; control.value = state.value; control.setAttribute("data-parameter-control", ""); + control.dataset.parameterName = parameter; if (state.expanded) { control.rows = 5; } else { @@ -53,10 +57,16 @@ window.datasetteSqlParameters = (() => { value, selectionStart ) { - const replacement = createControl(control.name, control.id, { - value: value === undefined ? control.value : value, - expanded: expand, - }); + const parameter = control.dataset.parameterName; + const replacement = createControl( + parameter, + control.id, + { + value: value === undefined ? control.value : value, + expanded: expand, + }, + manager.namePrefix + ); button.textContent = expand ? "Collapse" : "Expand"; button.setAttribute("aria-expanded", expand ? "true" : "false"); control.replaceWith(replacement); @@ -64,7 +74,7 @@ window.datasetteSqlParameters = (() => { if (selectionStart !== undefined && replacement.setSelectionRange) { replacement.setSelectionRange(selectionStart, selectionStart); } - manager.parameterState.set(replacement.name, controlState(replacement)); + manager.parameterState.set(parameter, controlState(replacement)); } function renderParameters(manager, parameters) { @@ -99,7 +109,7 @@ window.datasetteSqlParameters = (() => { label.htmlFor = id; label.textContent = parameter; - const control = createControl(parameter, id, state); + const control = createControl(parameter, id, state, manager.namePrefix); row.append(label, control); if (manager.allowExpand) { @@ -124,7 +134,10 @@ window.datasetteSqlParameters = (() => { if (!control.matches || !control.matches("[data-parameter-control]")) { return; } - manager.parameterState.set(control.name, controlState(control)); + manager.parameterState.set( + control.dataset.parameterName, + controlState(control) + ); }); if (!manager.allowExpand) { @@ -230,6 +243,7 @@ window.datasetteSqlParameters = (() => { ? section.dataset.allowExpand === "1" : false : options.allowExpand, + namePrefix: section ? section.dataset.parameterNamePrefix || "" : "", parameterState: new Map(), }; if (section) { diff --git a/datasette/templates/_sql_parameters.html b/datasette/templates/_sql_parameters.html index 58801d40..b5c1bde8 100644 --- a/datasette/templates/_sql_parameters.html +++ b/datasette/templates/_sql_parameters.html @@ -1,9 +1,10 @@ -
+{% set sql_parameter_name_prefix = sql_parameter_name_prefix|default("") %} +
{% if parameter_names %}

Parameters

{% for parameter in parameter_names %} {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %} -

{% if sql_parameters_allow_expand|default(false) %} {% endif %}

+

{% if sql_parameters_allow_expand|default(false) %} {% endif %}

{% endfor %} {% endif %}
diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index c5d55b80..2817f56e 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -9,6 +9,7 @@ from .base import BaseView, _error from .database import display_rows as display_query_rows from .query_helpers import ( QueryValidationError, + SQL_PARAMETER_FORM_PREFIX, _analysis_is_write, _analysis_rows, _analysis_rows_with_permissions, @@ -295,6 +296,7 @@ class ExecuteWriteView(BaseView): "execute_write_columns": execute_write_columns, "execute_write_display_rows": execute_write_display_rows, "execute_write_truncated": execute_write_truncated, + "sql_parameter_name_prefix": SQL_PARAMETER_FORM_PREFIX, "execute_disabled": bool(execute_disabled_reason), "execute_disabled_reason": execute_disabled_reason, "table_columns": table_columns, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 9efe3f81..026a999f 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -49,6 +49,8 @@ _query_write_fields = { "on_error_redirect", } +SQL_PARAMETER_FORM_PREFIX = "_sql_param_" + class QueryValidationError(Exception): def __init__(self, message, status=400, *, flash=False): @@ -289,11 +291,13 @@ def _coerce_execute_write_payload(data, is_json): ) params = data.get("params") or {} else: - params = { - key: value - for key, value in data.items() - if key not in {"sql", "csrftoken", "_json"} - } + params = {} + for key, value in data.items(): + if key in {"sql", "csrftoken", "_json"}: + continue + if key.startswith(SQL_PARAMETER_FORM_PREFIX): + key = key[len(SQL_PARAMETER_FORM_PREFIX) :] + params[key] = value if not isinstance(params, dict): raise QueryValidationError("params must be a dictionary") return data.get("sql"), params diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 64f91701..b7ceb6b2 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -794,6 +794,44 @@ async def test_update_row_alter(ds_write): assert response.json() == {"ok": True} +@pytest.mark.asyncio +async def test_execute_write_form_parameter_called_sql(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_parameter_sql", name="data") + await db.execute_write("create table docs (id integer primary key, title text)") + await db.execute_write("insert into docs (id, title) values (1, 'Initial')") + await ds.invoke_startup() + + form_response = await ds.client.get( + "/data/-/execute-write", + actor={"id": "root"}, + params={"sql": "update docs set title = :sql where id = :id"}, + ) + assert form_response.status_code == 200 + assert 'data-parameter-name-prefix="_sql_param_"' in form_response.text + assert '' in form_response.text + assert 'name="_sql_param_sql"' in form_response.text + assert 'data-parameter-name="sql"' in form_response.text + assert 'name="_sql_param_id"' in form_response.text + + response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "update docs set title = :sql where id = :id", + "_sql_param_sql": "Updated", + "_sql_param_id": "1", + }, + ) + + assert response.status_code == 200 + assert "Query executed, 1 row affected" in response.text + assert (await db.execute("select title from docs where id = 1")).first()[ + 0 + ] == "Updated" + + @pytest.mark.asyncio @pytest.mark.parametrize( "input,expected_errors", diff --git a/tests/test_html.py b/tests/test_html.py index bb7f612e..20ab22bc 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -342,7 +342,7 @@ async def test_query_parameter_form_fields(ds_client): response = await ds_client.get("/fixtures/-/query?sql=select+:name") assert response.status_code == 200 assert ( - ' ' + ' ' in response.text ) assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text @@ -351,7 +351,7 @@ async def test_query_parameter_form_fields(ds_client): response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") assert response2.status_code == 200 assert ( - ' ' + ' ' in response2.text ) diff --git a/tests/test_stored_queries.py b/tests/test_stored_queries.py index 2c648d5f..46420749 100644 --- a/tests/test_stored_queries.py +++ b/tests/test_stored_queries.py @@ -201,7 +201,7 @@ def test_error_in_on_success_message_sql(stored_write_client): def test_custom_params(stored_write_client): response = stored_write_client.get("/data/update_name?extra=foo") assert ( - '' + '' in response.text ) From f4b450603559b6a6412ed67e9eb170255dd1ab6b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:49:23 -0700 Subject: [PATCH 1149/1172] Remove legacy ?_extras= row parameter The pre-1.0 ?_extras= (plural) parameter was kept for backwards compatibility with the old row JSON API. ?_extra= is the documented mechanism now that row pages share the extras registry. Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 5 ----- tests/test_api.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 3fe213d7..ce15a822 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -165,13 +165,8 @@ class RowView(DataView): "primary_key_values": pk_values, } - # Handle _extra parameter (new style) extras = _get_extras(request) - # Also support legacy _extras parameter for backward compatibility - if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): - extras.add("foreign_key_tables") - # Process extras row_extra_context = RowExtraContext( datasette=self.ds, diff --git a/tests/test_api.py b/tests/test_api.py index e1385b6f..f57d0206 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -383,7 +383,7 @@ async def test_row_strange_table_name(ds_client): @pytest.mark.asyncio async def test_row_foreign_key_tables(ds_client): response = await ds_client.get( - "/fixtures/simple_primary_key/1.json?_extras=foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" ) assert response.status_code == 200 # Foreign keys are sorted by (other_table, column, other_column) From d825d8c4f38d980356abc50c739a440585253062 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:53:41 -0700 Subject: [PATCH 1150/1172] Remove _get_extras() shim in favor of extra_names_from_request() Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 5 +++-- datasette/views/table.py | 6 +----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index ce15a822..c6721ca0 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -14,7 +14,8 @@ from datasette.plugins import pm import json import markupsafe import sqlite_utils -from .table import display_columns_and_rows, _get_extras +from datasette.extras import extra_names_from_request +from .table import display_columns_and_rows from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry @@ -165,7 +166,7 @@ class RowView(DataView): "primary_key_values": pk_values, } - extras = _get_extras(request) + extras = extra_names_from_request(request) # Process extras row_extra_context = RowExtraContext( diff --git a/datasette/views/table.py b/datasette/views/table.py index c2d520f8..1b298c50 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -854,10 +854,6 @@ class TableDropView(BaseView): return Response.json({"ok": True}, status=200) -def _get_extras(request): - return extra_names_from_request(request) - - async def _columns_to_select(table_columns, pks, request): columns = list(table_columns) if "_col" in request.args: @@ -1461,7 +1457,7 @@ async def table_view_data( rows = rows[:page_size] # Resolve extras - extras = _get_extras(request) + extras = extra_names_from_request(request) if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")): extras.add("facet_results") if request.args.get("_shape") == "object": From df8a61450b478e66c458b3f05c286daeb2c2a6b0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:57:02 -0700 Subject: [PATCH 1151/1172] Remove hasattr/getattr probing from multi-scope extras TableExtraContext, RowExtraContext and QueryExtraContext now share normalized table_name, is_view, pks and query_name fields (defaulting to None/False where inapplicable) so DebugExtra, RenderCellExtra and RenderersExtra can read them directly. RenderCellExtra uses context.columns in every scope - the table and row views both derive columns from results.description so output is unchanged. Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 1 - datasette/views/table_extras.py | 30 +++++++++++++----------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index c6721ca0..e15dfce9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -179,7 +179,6 @@ class RowView(DataView): private=private, rows=rows, columns=columns, - results_description=results.description, pks=pks, pk_values=pk_values, sql=resolved.sql, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index ec104be3..63c87a6f 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -53,6 +53,7 @@ class TableExtraContext: extra_registry: ExtraRegistry display_columns_and_rows: object run_sequential: object + query_name: str | None = None scope: ExtraScope = ExtraScope.TABLE @@ -67,7 +68,6 @@ class RowExtraContext: private: bool rows: list columns: list - results_description: list pks: list pk_values: list sql: str @@ -75,6 +75,7 @@ class RowExtraContext: extras: set extra_registry: ExtraRegistry foreign_key_tables: object + is_view: bool = False scope: ExtraScope = ExtraScope.ROW @@ -96,6 +97,9 @@ class QueryExtraContext: metadata: dict extras: set extra_registry: ExtraRegistry + table_name: str | None = None + is_view: bool = False + pks: list | None = None scope: ExtraScope = ExtraScope.QUERY @@ -383,6 +387,8 @@ class DebugExtra(Extra): } if context.scope == ExtraScope.TABLE: debug["resolved"] = repr(context.resolved) + debug["nofacet"] = context.nofacet + debug["nosuggest"] = context.nosuggest elif context.scope == ExtraScope.ROW: debug["resolved"] = { "table": context.table_name, @@ -391,10 +397,6 @@ class DebugExtra(Extra): "pks": context.pks, "pk_values": context.pk_values, } - if hasattr(context, "nofacet"): - debug["nofacet"] = context.nofacet - if hasattr(context, "nosuggest"): - debug["nosuggest"] = context.nosuggest return debug @@ -527,16 +529,10 @@ class RenderCellExtra(Extra): scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - table_name = getattr(context, "table_name", None) - is_view = getattr(context, "is_view", False) - pks = getattr(context, "pks", []) - pks_for_display = ( - pks if pks else (["rowid"] if table_name and not is_view else []) + table_name = context.table_name + pks_for_display = context.pks or ( + ["rowid"] if table_name and not context.is_view else [] ) - if hasattr(context, "results_description"): - col_names = [col[0] for col in context.results_description] - else: - col_names = context.columns ct_map = ( await context.datasette.get_column_types(context.database_name, table_name) if table_name @@ -545,7 +541,7 @@ class RenderCellExtra(Extra): rendered_rows = [] for row in context.rows: rendered_row = {} - for value, column in zip(row, col_names): + for value, column in zip(row, context.columns): ct = ct_map.get(column) plugin_display_value = None if ct: @@ -869,7 +865,7 @@ class RenderersExtra(Extra): url_labels_extra = {} if expandable_columns: url_labels_extra = {"_labels": "on"} - table_name = getattr(context, "table_name", None) + table_name = context.table_name view_name = "table" if context.scope == ExtraScope.TABLE else "database" for key, (_, can_render) in context.datasette.renderers.items(): it_can_render = call_with_supported_arguments( @@ -878,7 +874,7 @@ class RenderersExtra(Extra): columns=context.columns or [], rows=context.rows or [], sql=query.get("sql", None), - query_name=getattr(context, "query_name", None), + query_name=context.query_name, database=context.database_name, table=table_name, request=context.request, From ab62ec96d187fa05f7d672d48a0b3f962fb8c228 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:45:13 -0700 Subject: [PATCH 1152/1172] Fix _extra=private for arbitrary SQL query pages QueryView hardcoded private=False unless the request was for a stored query, so /db/-/query.json?_extra=private reported false even when execute-sql was restricted to the authenticated actor. Use check_visibility() like the table and row views do. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 6 ++++-- tests/test_table_api.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 96a58758..e6efddea 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -612,11 +612,13 @@ class QueryView(View): ) else: - await datasette.ensure_permission( + visible, private = await datasette.check_visibility( + request.actor, action="execute-sql", resource=DatabaseResource(database=database), - actor=request.actor, ) + if not visible: + raise Forbidden("execute-sql") # Flattened because of ?sql=&name1=value1&name2=value2 feature params = {key: request.args.get(key) for key in request.args} diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 388e3979..4ab2f596 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -117,6 +117,29 @@ async def test_query_extras_for_stored_query(ds_client): } +def test_query_extra_private_for_arbitrary_sql(): + with make_app_client(config={"allow_sql": {"id": "root"}}) as client: + cookies = {"ds_actor": client.actor_cookie({"id": "root"})} + response = client.get( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private", + cookies=cookies, + ) + assert response.status == 200 + assert response.json["private"] is True + # Anonymous users cannot execute SQL at all here + anon = client.get("/fixtures/-/query.json?sql=select+1+as+one") + assert anon.status == 403 + + +def test_query_extra_private_false_when_sql_is_public(): + with make_app_client() as client: + response = client.get( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" + ) + assert response.status == 200 + assert response.json["private"] is False + + @pytest.mark.asyncio async def test_table_shape_objects(ds_client): response = await ds_client.get("/fixtures/simple_primary_key.json?_shape=objects") From 8f888515b618bc0eb18e23c861b938b6bbbbf5d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:47:26 -0700 Subject: [PATCH 1153/1172] Fix _extra=query to report the params that were actually bound QueryExtra re-derived named parameters from the SQL with a regex, which missed parameters declared in a stored query's params list, reported magic _-prefixed parameters with raw querystring values that were never bound, and echoed the entire querystring when no SQL was present. QueryView now passes its named_parameter_values dict - the parameters it actually bound - through QueryExtraContext. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 2 +- datasette/views/table_extras.py | 11 +--------- tests/test_table_api.py | 38 +++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index e6efddea..a719fa4f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -717,7 +717,7 @@ class QueryView(View): rows=rows, columns=columns, sql=sql, - params=params_for_query, + params=named_parameter_values, query_name=stored_query.name if stored_query else None, stored_query=stored_query, stored_query_write=stored_query_write, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 63c87a6f..21a908a0 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -8,7 +8,6 @@ from datasette.resources import TableResource from datasette.utils import ( await_me_maybe, call_with_supported_arguments, - named_parameters as derive_named_parameters, path_with_added_args, path_with_format, path_with_removed_args, @@ -592,17 +591,9 @@ class QueryExtra(Extra): scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - params = context.params - if context.scope == ExtraScope.QUERY and context.sql: - parameter_names = set(derive_named_parameters(context.sql)) - params = { - key: value - for key, value in dict(context.params).items() - if key in parameter_names - } return { "sql": context.sql, - "params": params, + "params": context.params, } diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 4ab2f596..cfa3b512 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -131,6 +131,44 @@ def test_query_extra_private_for_arbitrary_sql(): assert anon.status == 403 +def test_query_extra_query_reports_bound_params(): + config = { + "databases": { + "fixtures": { + "queries": { + "declared_params": { + "sql": "select 1 as one", + "params": ["foo"], + }, + "magic_host": { + "sql": "select :_header_host as h", + }, + } + } + } + } + with make_app_client(config=config) as client: + # Declared parameters are reported even when the regex cannot find them + response = client.get("/fixtures/declared_params.json?foo=bar&_extra=query") + assert response.status == 200 + assert response.json["query"]["params"] == {"foo": "bar"} + # Magic parameters are bound internally and should not be reported, + # especially not as a value taken from the querystring + response = client.get( + "/fixtures/magic_host.json?_extra=query&_header_host=spoofed" + ) + assert response.status == 200 + assert response.json["rows"] == [{"h": "localhost"}] + assert response.json["query"]["params"] == {} + + +def test_query_extra_query_does_not_echo_querystring_without_sql(): + with make_app_client() as client: + response = client.get("/fixtures/-/query.json?_extra=query&foo=bar") + assert response.status == 200 + assert response.json["query"]["params"] == {} + + def test_query_extra_private_false_when_sql_is_public(): with make_app_client() as client: response = client.get( From b635dc53f42e06908c7510d743e85100a6488f22 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:50:44 -0700 Subject: [PATCH 1154/1172] Make filters, actions and display_rows extras internal These three extras return values that exist for the HTML templates - a Filters instance, an async function and markupsafe/sqlite3.Row data - so requesting them on a .json page returned a 500 serialization error, while the generated documentation and ?_extra=extras both advertised them as API surface. They are now public=False: ignored like any unknown name on JSON requests, omitted from the docs and the extras list, and still resolved for the HTML view via the new include_internal flag on ExtraRegistry.resolve(). Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 17 ++++++++--------- datasette/views/table.py | 9 ++++++++- datasette/views/table_extras.py | 12 ++++++++++-- docs/json_api.rst | 9 --------- tests/test_table_api.py | 18 ++++++++++++++++++ 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index f655e517..d5847937 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -89,7 +89,7 @@ class ExtraRegistry: def public_classes_for_scope(self, scope): return self.classes_for_scope(scope, include_internal=False) - async def resolve(self, requested, context, scope): + async def resolve(self, requested, context, scope, include_internal=False): registry = Registry() async def context_provider(): @@ -100,15 +100,14 @@ class ExtraRegistry: for cls in self.classes_for_scope(scope): registry.register(cls().resolve, name=cls.key()) - public_names = {cls.key() for cls in self.public_classes_for_scope(scope)} - requested_public_names = [ - name - for name in requested - if name in public_names and name in registry._registry - ] - resolved = await registry.resolve_multi(requested_public_names) + allowed_names = { + cls.key() + for cls in self.classes_for_scope(scope, include_internal=include_internal) + } + requested_names = [name for name in requested if name in allowed_names] + resolved = await registry.resolve_multi(requested_names) return { - name: resolved[name] for name in requested_public_names if name in resolved + name: resolved[name] for name in requested_names if name in resolved } diff --git a/datasette/views/table.py b/datasette/views/table.py index 1b298c50..3cf8e6c6 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1518,7 +1518,14 @@ async def table_view_data( "ok": True, "next": next_value and str(next_value) or None, } - data.update(await resolve_table_extras(extras, table_extra_context)) + data.update( + await resolve_table_extras( + extras, + table_extra_context, + # The HTML view needs extras that are not JSON serializable + include_internal=bool(extra_extras), + ) + ) raw_sqlite_rows = rows[:page_size] # Apply transform_value for columns with assigned types ct_map = await datasette.get_column_types(database_name, table_name) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 21a908a0..c98ae22c 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -333,6 +333,8 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): description = "Table or view actions made available by plugin hooks" scopes = frozenset({ExtraScope.TABLE}) + # Returns an async function for the HTML templates - not JSON serializable + public = False async def resolve(self, context): async def actions(): @@ -476,6 +478,8 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): description = "Row data formatted for the HTML table display" scopes = frozenset({ExtraScope.TABLE}) + # Contains markupsafe/sqlite3.Row values - not JSON serializable + public = False async def resolve(self, context, display_columns_and_rows): return display_columns_and_rows["rows"] @@ -772,6 +776,8 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): description = "Filters object used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) + # Returns a Filters instance for the HTML templates - not JSON serializable + public = False async def resolve(self, context): return context.filters @@ -1034,8 +1040,10 @@ TABLE_EXTRA_CLASSES = [ table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) -async def resolve_table_extras(extras, context): - return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) +async def resolve_table_extras(extras, context, include_internal=False): + return await table_extra_registry.resolve( + extras, context, ExtraScope.TABLE, include_internal=include_internal + ) async def resolve_row_extras(extras, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 379d26a0..6b595577 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -425,9 +425,6 @@ The available table extras are listed below. } ] -``display_rows`` - Row data formatted for the HTML table display - ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook @@ -554,12 +551,6 @@ The available table extras are listed below. "9403e5" -``actions`` - Table or view actions made available by plugin hooks - -``filters`` - Filters object used by the HTML table interface - ``renderers`` Alternative output renderers available for this table diff --git a/tests/test_table_api.py b/tests/test_table_api.py index cfa3b512..0cb67164 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -117,6 +117,24 @@ async def test_query_extras_for_stored_query(ds_client): } +@pytest.mark.parametrize("extra", ["filters", "actions", "display_rows"]) +@pytest.mark.asyncio +async def test_html_only_extras_are_not_available_via_json(ds_client, extra): + # These extras exist for the HTML view; their values are not JSON + # serializable so they are internal, not part of the JSON API + response = await ds_client.get(f"/fixtures/facetable.json?_extra={extra}") + assert response.status_code == 200 + assert extra not in response.json() + + +@pytest.mark.asyncio +async def test_html_only_extras_are_not_advertised(ds_client): + response = await ds_client.get("/fixtures/facetable.json?_extra=extras") + assert response.status_code == 200 + names = {e["name"] for e in response.json()["extras"]} + assert {"filters", "actions", "display_rows"}.isdisjoint(names) + + def test_query_extra_private_for_arbitrary_sql(): with make_app_client(config={"allow_sql": {"id": "root"}}) as client: cookies = {"ds_actor": client.actor_cookie({"id": "root"})} From bbf0424c4519441715f73ee6468e0c53cc959861 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:51:25 -0700 Subject: [PATCH 1155/1172] Changelog for row/query extras and related fixes Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 75e4f3e8..19089dd1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,13 @@ Changelog ------------------- - Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) +- Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. See :ref:`json_api_extra` for the full list. +- New generated reference documentation for every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra`. +- ``?_extra=`` values can be separated by commas as well as repeated, e.g. ``?_extra=count,next_url``. Previously a comma-separated value that included ``columns`` failed to include the ``columns`` key in the response. +- The ``?_extra=private`` extra on arbitrary SQL query pages now correctly reflects whether the SQL execution permission is private to the current actor - it previously always returned ``false``. +- The ``?_extra=query`` extra on query pages now reports the named parameters that were actually bound when the query executed, including parameters declared in a stored query's ``params`` list. Magic ``_``-prefixed parameters are no longer echoed back with unbound values taken from the querystring. +- Extras that exist to serve the HTML interface (``filters``, ``actions``, ``display_rows``) are no longer advertised or reachable through the JSON API, where requesting them previously returned a 500 serialization error. +- The pre-1.0 ``?_extras=`` (plural) parameter on row pages has been removed - use ``?_extra=foreign_key_tables`` instead. .. _v1_0_a32: From 6babd23cec9c41edd3d0ba2fab1c319905446b21 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:53:00 -0700 Subject: [PATCH 1156/1172] QueryView: only resolve extras for renderer formats, single metadata path Extras were resolved before the format dispatch, so a .csv request carrying ?_extra= parameters paid for extras (including per-cell render_cell plugin calls) whose results were then discarded, and the HTML path duplicated the stored-query metadata derivation. Extras now resolve inside the renderer-dispatch branch only, and both consumers share a query_metadata() helper that no longer fetches database metadata just to throw it away for stored queries. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 55 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index a719fa4f..ad3fb843 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -700,33 +700,12 @@ class QueryView(View): except DatasetteError: raise - extras = extra_names_from_request(request) - metadata = None - data = {"ok": True, "rows": rows, "columns": columns} - if extras: - metadata = await datasette.get_database_metadata(database) + async def query_metadata(): if stored_query: metadata = stored_query_to_dict(stored_query) metadata.pop("source", None) - query_extra_context = QueryExtraContext( - datasette=datasette, - request=request, - db=db, - database_name=database, - private=private, - rows=rows, - columns=columns, - sql=sql, - params=named_parameter_values, - query_name=stored_query.name if stored_query else None, - stored_query=stored_query, - stored_query_write=stored_query_write, - error=query_error, - metadata=metadata, - extras=extras, - extra_registry=table_extra_registry, - ) - data.update(await resolve_query_extras(extras, query_extra_context)) + return metadata + return await datasette.get_database_metadata(database) # Handle formats from plugins if format_ == "csv": @@ -740,6 +719,28 @@ class QueryView(View): return await stream_csv(datasette, fetch_data_for_csv, request, db.name) elif format_ in datasette.renderers.keys(): + data = {"ok": True, "rows": rows, "columns": columns} + extras = extra_names_from_request(request) + if extras: + query_extra_context = QueryExtraContext( + datasette=datasette, + request=request, + db=db, + database_name=database, + private=private, + rows=rows, + columns=columns, + sql=sql, + params=named_parameter_values, + query_name=stored_query.name if stored_query else None, + stored_query=stored_query, + stored_query_write=stored_query_write, + error=query_error, + metadata=await query_metadata(), + extras=extras, + extra_registry=table_extra_registry, + ) + data.update(await resolve_query_extras(extras, query_extra_context)) # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) result = call_with_supported_arguments( @@ -806,11 +807,7 @@ class QueryView(View): ) } ) - if metadata is None: - metadata = await datasette.get_database_metadata(database) - if stored_query: - metadata = stored_query_to_dict(stored_query) - metadata.pop("source", None) + metadata = await query_metadata() renderers = {} for key, (_, can_render) in datasette.renderers.items(): it_can_render = call_with_supported_arguments( From a1b6a6976d0ddafba6b927ce9fb83e62cb9091c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:55:28 -0700 Subject: [PATCH 1157/1172] Remove dead weight from the extras machinery - TableExtraContext.next_value, RowExtraContext.resolved and QueryExtraContext.stored_query/stored_query_write/error had no readers - drop the fields and the arguments that populated them - Extra.documentation() and the stable classvar were unused parallel descriptions of what the docs generator reads directly - ExtraRegistry.resolve no longer carries an always-true membership guard (resolve_multi returns every requested registered name) Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 19 +------------------ datasette/views/database.py | 3 --- datasette/views/row.py | 1 - datasette/views/table.py | 1 - datasette/views/table_extras.py | 5 ----- 5 files changed, 1 insertion(+), 28 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index d5847937..4aa93057 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -50,7 +50,6 @@ class Extra(Provider): example: ClassVar[ExtraExample | None] = None examples: ClassVar[dict[ExtraScope, ExtraExample | list[ExtraExample]]] = {} public: ClassVar[bool] = True - stable: ClassVar[bool] = True expensive: ClassVar[bool] = False docs_note: ClassVar[str | None] = None @@ -58,20 +57,6 @@ class Extra(Provider): def example_for_scope(cls, scope): return cls.examples.get(scope, cls.example) - @classmethod - def documentation(cls): - return { - "name": cls.key(), - "description": cls.description, - "scopes": [ - scope.value for scope in sorted(cls.scopes, key=lambda s: s.value) - ], - "stable": cls.stable, - "expensive": cls.expensive, - "docs_note": cls.docs_note, - "example": cls.example, - } - class ExtraRegistry: def __init__(self, classes): @@ -106,9 +91,7 @@ class ExtraRegistry: } requested_names = [name for name in requested if name in allowed_names] resolved = await registry.resolve_multi(requested_names) - return { - name: resolved[name] for name in requested_names if name in resolved - } + return {name: resolved[name] for name in requested_names} def _camel_to_snake(name): diff --git a/datasette/views/database.py b/datasette/views/database.py index ad3fb843..46e26496 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -733,9 +733,6 @@ class QueryView(View): sql=sql, params=named_parameter_values, query_name=stored_query.name if stored_query else None, - stored_query=stored_query, - stored_query_write=stored_query_write, - error=query_error, metadata=await query_metadata(), extras=extras, extra_registry=table_extra_registry, diff --git a/datasette/views/row.py b/datasette/views/row.py index e15dfce9..c300758b 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -172,7 +172,6 @@ class RowView(DataView): row_extra_context = RowExtraContext( datasette=self.ds, request=request, - resolved=resolved, db=db, database_name=database, table_name=table, diff --git a/datasette/views/table.py b/datasette/views/table.py index 3cf8e6c6..65388c9c 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1497,7 +1497,6 @@ async def table_view_data( nofacet=nofacet, nosuggest=nosuggest, next_arg=request.args.get("_next"), - next_value=next_value, next_url=next_url, sql=sql, sql_no_order_no_limit=sql_no_order_no_limit, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index c98ae22c..493135f3 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -37,7 +37,6 @@ class TableExtraContext: nofacet: object nosuggest: object next_arg: object - next_value: object next_url: str | None sql: str sql_no_order_no_limit: str @@ -60,7 +59,6 @@ class TableExtraContext: class RowExtraContext: datasette: object request: object - resolved: object db: object database_name: str table_name: str @@ -90,9 +88,6 @@ class QueryExtraContext: sql: str | None params: dict query_name: str | None - stored_query: object - stored_query_write: bool - error: str | None metadata: dict extras: set extra_registry: ExtraRegistry From cfafa5b37f5350303600e912955c43e210a113b7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:56:40 -0700 Subject: [PATCH 1158/1172] Use plain set literals for Extra scopes frozenset({...}) was immutability ceremony for class attributes that nothing mutates. scopes = {ExtraScope.TABLE} reads cleaner. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 2 +- datasette/views/table_extras.py | 74 ++++++++++++++++----------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 4aa93057..fee92939 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -30,7 +30,7 @@ class ExtraExample: class Provider: name: ClassVar[str | None] = None - scopes: ClassVar[frozenset[ExtraScope]] = frozenset() + scopes: ClassVar[set[ExtraScope]] = set() public: ClassVar[bool] = False @classmethod diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 493135f3..ce1d7bdf 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -100,7 +100,7 @@ class QueryExtraContext: class CountSqlExtra(Extra): description = "SQL query used to calculate the total count" example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.count_sql @@ -109,7 +109,7 @@ class CountSqlExtra(Extra): class CountExtra(Extra): description = "Total count of rows matching these filters" example = ExtraExample("/fixtures/facetable.json?_extra=count") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context): @@ -141,7 +141,7 @@ class CountExtra(Extra): class FacetInstancesProvider(Provider): - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, count): facet_instances = [] @@ -182,7 +182,7 @@ class FacetResultsExtra(Extra): }, note="Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context, facet_instances): @@ -217,7 +217,7 @@ class FacetsTimedOutExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, facet_results): return facet_results["timed_out"] @@ -234,7 +234,7 @@ class SuggestedFacetsExtra(Extra): ], note="Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context, facet_instances): @@ -259,7 +259,7 @@ class HumanDescriptionEnExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): human_description_en = context.filters.human_description_en( @@ -279,7 +279,7 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.next_url @@ -296,7 +296,7 @@ class ColumnsExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=columns" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.columns @@ -305,7 +305,7 @@ class ColumnsExtra(Extra): class AllColumnsExtra(Extra): description = "All columns in the table, regardless of _col/_nocol filtering" example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return list(context.table_columns) @@ -319,7 +319,7 @@ class PrimaryKeysExtra(Extra): "/fixtures/simple_primary_key/1.json?_extra=primary_keys" ) } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): return context.pks @@ -327,7 +327,7 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): description = "Table or view actions made available by plugin hooks" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Returns an async function for the HTML templates - not JSON serializable public = False @@ -358,7 +358,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): description = "Whether this resource is a view instead of a table" example = ExtraExample("/fixtures/simple_view.json?_extra=is_view") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.is_view @@ -375,7 +375,7 @@ class DebugExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=debug" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): debug = { @@ -407,7 +407,7 @@ class RequestExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=request" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return { @@ -420,7 +420,7 @@ class RequestExtra(Extra): class DisplayColumnsAndRowsProvider(Provider): - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): display_columns, display_rows = await context.display_columns_and_rows( @@ -464,7 +464,7 @@ class DisplayColumnsExtra(Extra): ], note="Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, display_columns_and_rows): return display_columns_and_rows["columns"] @@ -472,7 +472,7 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): description = "Row data formatted for the HTML table display" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Contains markupsafe/sqlite3.Row values - not JSON serializable public = False @@ -524,7 +524,7 @@ class RenderCellExtra(Extra): ), ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): table_name = context.table_name @@ -587,7 +587,7 @@ class QueryExtra(Extra): ExtraExample("/fixtures/neighborhood_search.json?text=town&_extra=query"), ], } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return { @@ -599,7 +599,7 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" example = ExtraExample(value={}) - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): ct_map = await context.datasette.get_column_types( @@ -616,7 +616,7 @@ class ColumnTypesExtra(Extra): class SetColumnTypeUiExtra(Extra): description = "Column type UI metadata for this table" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): if context.is_view: @@ -676,7 +676,7 @@ class MetadataExtra(Extra): "/fixtures/neighborhood_search.json?text=town&_extra=metadata" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): if context.scope == ExtraScope.QUERY: @@ -713,7 +713,7 @@ class DatabaseExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=database" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.database_name @@ -725,7 +725,7 @@ class TableExtra(Extra): examples = { ExtraScope.ROW: ExtraExample("/fixtures/simple_primary_key/1.json?_extra=table") } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): return context.table_name @@ -742,7 +742,7 @@ class DatabaseColorExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=database_color" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.db.color @@ -753,7 +753,7 @@ class FormHiddenArgsExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): form_hidden_args = [] @@ -770,7 +770,7 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): description = "Filters object used by the HTML table interface" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Returns a Filters instance for the HTML templates - not JSON serializable public = False @@ -781,7 +781,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return [ @@ -796,7 +796,7 @@ class SortedFacetResultsExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, facet_results): facet_configs = context.table_metadata.get("facets", []) @@ -832,7 +832,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): description = "SQL definition for this table" example = ExtraExample("/fixtures/facetable.json?_extra=table_definition") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return await context.db.get_table_definition(context.table_name) @@ -841,7 +841,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): description = "SQL definition for this view" example = ExtraExample("/fixtures/simple_view.json?_extra=view_definition") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return await context.db.get_view_definition(context.table_name) @@ -850,7 +850,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" example = ExtraExample("/fixtures/facetable.json?_extra=renderers") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, expandable_columns, query): renderers = {} @@ -896,7 +896,7 @@ class PrivateExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.private @@ -905,7 +905,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): expandables = [] @@ -921,7 +921,7 @@ class ForeignKeyTablesExtra(Extra): example = ExtraExample( "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" ) - scopes = frozenset({ExtraScope.ROW}) + scopes = {ExtraScope.ROW} async def resolve(self, context): return await context.foreign_key_tables( @@ -931,7 +931,7 @@ class ForeignKeyTablesExtra(Extra): class ExtrasExtra(Extra): description = "Available ?_extra= blocks" - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): all_extras = [ From 4edea3ad2637f4bf275f3a322e4c7747b964d907 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:04:12 -0700 Subject: [PATCH 1159/1172] Build extras registries once per scope instead of per request ExtraRegistry.resolve() previously constructed a fresh asyncinject Registry on every table, row and query request - instantiating all ~37 Extra classes and re-running inspect.signature reflection over each resolve method every time. The Extra classes are stateless, so the asyncinject Registry for each scope is now built lazily once and shared, along with the allowed-name sets. The per-request context reaches the shared registry through a contextvars.ContextVar provider rather than resolve_multi(results=...) seeding: asyncinject's parallel executor never schedules anything when the only initially-ready node is an unregistered pre-seeded value, so seeding would have stalled every resolution. asyncio tasks copy the caller's context, which keeps concurrent resolves isolated - covered by a new test. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 63 ++++++++++++++++++++++++++++++++---------- tests/test_extras.py | 65 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 tests/test_extras.py diff --git a/datasette/extras.py b/datasette/extras.py index fee92939..2c3450b2 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,3 +1,4 @@ +import contextvars import re from dataclasses import dataclass from enum import Enum @@ -5,6 +6,11 @@ from typing import ClassVar from asyncinject import Registry +# Per-request context for Extra.resolve(), so the asyncinject registries can +# be shared across requests. asyncio tasks copy the caller's context, so +# concurrent resolve() calls each see their own value. +_resolve_context = contextvars.ContextVar("datasette_extras_context") + def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") @@ -62,6 +68,13 @@ class ExtraRegistry: def __init__(self, classes): self.classes = list(classes) self.classes_by_name = {cls.key(): cls for cls in self.classes} + # Lazily-built shared state, keyed by scope. Safe to share across + # requests because Extra instances are stateless and asyncinject's + # Registry keeps per-call state local to each resolve_multi() call. + # If extras classes ever become registerable at runtime (e.g. via a + # plugin hook) these caches will need invalidating. + self._scope_registries = {} + self._allowed_names = {} def classes_for_scope(self, scope, include_internal=True): classes = [ @@ -74,23 +87,43 @@ class ExtraRegistry: def public_classes_for_scope(self, scope): return self.classes_for_scope(scope, include_internal=False) + def _registry_for_scope(self, scope): + registry = self._scope_registries.get(scope) + if registry is None: + registry = Registry() + + async def context_provider(): + return _resolve_context.get() + + registry.register(context_provider, name="context") + for cls in self.classes_for_scope(scope): + registry.register(cls().resolve, name=cls.key()) + self._scope_registries[scope] = registry + return registry + + def _allowed_names_for_scope(self, scope, include_internal): + key = (scope, include_internal) + names = self._allowed_names.get(key) + if names is None: + names = { + cls.key() + for cls in self.classes_for_scope( + scope, include_internal=include_internal + ) + } + self._allowed_names[key] = names + return names + async def resolve(self, requested, context, scope, include_internal=False): - registry = Registry() - - async def context_provider(): - return context - - registry.register(context_provider, name="context") - - for cls in self.classes_for_scope(scope): - registry.register(cls().resolve, name=cls.key()) - - allowed_names = { - cls.key() - for cls in self.classes_for_scope(scope, include_internal=include_internal) - } + allowed_names = self._allowed_names_for_scope(scope, include_internal) requested_names = [name for name in requested if name in allowed_names] - resolved = await registry.resolve_multi(requested_names) + token = _resolve_context.set(context) + try: + resolved = await self._registry_for_scope(scope).resolve_multi( + requested_names + ) + finally: + _resolve_context.reset(token) return {name: resolved[name] for name in requested_names} diff --git a/tests/test_extras.py b/tests/test_extras.py new file mode 100644 index 00000000..ad8a9f00 --- /dev/null +++ b/tests/test_extras.py @@ -0,0 +1,65 @@ +import asyncio + +import pytest + +from datasette.extras import Extra, ExtraRegistry, ExtraScope + + +class SlowValueExtra(Extra): + description = "Returns context['value'], optionally slowly" + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + if context["slow"]: + await asyncio.sleep(0.05) + return context["value"] + + +class DependentExtra(Extra): + description = "Depends on slow_value" + scopes = {ExtraScope.TABLE} + + async def resolve(self, context, slow_value): + return slow_value + 1 + + +def test_registry_is_built_once_per_scope(): + registry = ExtraRegistry([SlowValueExtra, DependentExtra]) + first = registry._registry_for_scope(ExtraScope.TABLE) + second = registry._registry_for_scope(ExtraScope.TABLE) + assert first is second + + +@pytest.mark.asyncio +async def test_concurrent_resolves_do_not_share_state(): + # The asyncinject registry is shared across requests - resolved values + # must not leak between concurrent resolve() calls with different contexts + registry = ExtraRegistry([SlowValueExtra, DependentExtra]) + slow, fast = await asyncio.gather( + registry.resolve( + {"slow_value", "dependent"}, + {"value": 100, "slow": True}, + ExtraScope.TABLE, + ), + registry.resolve( + {"slow_value", "dependent"}, + {"value": 200, "slow": False}, + ExtraScope.TABLE, + ), + ) + assert slow == {"slow_value": 100, "dependent": 101} + assert fast == {"slow_value": 200, "dependent": 201} + + +@pytest.mark.asyncio +async def test_table_row_and_query_scopes_use_separate_registries(): + from datasette.views.table_extras import table_extra_registry + + registries = { + scope: table_extra_registry._registry_for_scope(scope) for scope in ExtraScope + } + assert len(set(map(id, registries.values()))) == 3 + # Scope-specific extras only registered where they belong + assert "count" in registries[ExtraScope.TABLE]._registry + assert "count" not in registries[ExtraScope.QUERY]._registry + assert "foreign_key_tables" in registries[ExtraScope.ROW]._registry From 96226621325c5aa19e6a700efcd1b441006958fa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:15:18 -0700 Subject: [PATCH 1160/1172] Fix SQL injection via bracket escape bypass in escape_sqlite() (#2677) escape_sqlite() wrapped identifiers in [brackets] without escaping any ] characters inside the string. Since SQLite does not support escaping ] within bracket quoting, an identifier containing ] could break out and inject arbitrary SQL. Fall back to double-quote quoting (doubling any embedded ") when the identifier contains ]. Co-Authored-By: Claude Fable 5 --- datasette/utils/__init__.py | 4 ++++ tests/test_utils.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 2dff9667..55e539b9 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -410,6 +410,10 @@ def escape_css_string(s): def escape_sqlite(s): if _boring_keyword_re.match(s) and (s.lower() not in reserved_words): return s + elif "]" in s: + # SQLite does not support escaping ] inside [bracket] quoting, so fall + # back to double-quote quoting (doubling any embedded ") - #2677 + return '"{}"'.format(s.replace('"', '""')) else: return f"[{s}]" diff --git a/tests/test_utils.py b/tests/test_utils.py index 64607244..74f1963f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -216,6 +216,38 @@ def test_detect_fts(open_quote, close_quote): conn.close() +@pytest.mark.parametrize( + "identifier,expected", + ( + ("plain", "plain"), + ("select", "[select]"), + ("has space", "[has space]"), + ("has'quote", "[has'quote]"), + # Identifiers containing ] must fall back to double-quote quoting + # (SQLite does not support escaping ] inside [brackets]) - #2677 + ("has]bracket", '"has]bracket"'), + ('has"dquote]', '"has""dquote]"'), + ), +) +def test_escape_sqlite(identifier, expected): + assert utils.escape_sqlite(identifier) == expected + + +def test_escape_sqlite_prevents_injection(): + # https://github.com/simonw/datasette/issues/2677 + conn = utils.sqlite3.connect(":memory:") + conn.execute("CREATE TABLE users (id INTEGER, password TEXT)") + conn.execute("INSERT INTO users VALUES (1, 'super_secret_password')") + malicious = "users] UNION SELECT password FROM users--" + conn.execute('CREATE TABLE "{}" (id INTEGER)'.format(malicious)) + sql = "select count(*) from {}".format(utils.escape_sqlite(malicious)) + results = conn.execute(sql).fetchall() + conn.close() + # The injected UNION must not execute - only the empty malicious table + # is queried, so we get a single count row and no leaked password + assert results == [(0,)] + + @pytest.mark.parametrize("table", ("regular", "has'single quote")) def test_detect_fts_different_table_names(table): sql = """ From 1c514d69f6cc09c820c119e7bbf4dc75235e90cc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:17:16 -0700 Subject: [PATCH 1161/1172] Prevent open redirect via backslash in path (#2680) asgi_send_redirect() only collapsed leading forward slashes, so a path like /\example.com/ produced a Location of /\example.com. Browsers normalise backslashes to forward slashes, turning that into the protocol-relative //example.com and redirecting off-site. Collapse any run of leading slashes and backslashes to a single slash. Co-Authored-By: Claude Fable 5 --- datasette/utils/asgi.py | 8 +++++--- tests/test_custom_pages.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 35f243b6..55eba1bb 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -330,9 +330,11 @@ async def asgi_send_html(send, html, status=200, headers=None): async def asgi_send_redirect(send, location, status=302): - # Prevent open redirect vulnerability: strip multiple leading slashes - # //example.com would be interpreted as a protocol-relative URL (e.g., https://example.com/) - location = re.sub(r"^/+", "/", location) + # Prevent open redirect vulnerability: collapse leading slashes and + # backslashes down to a single slash. //example.com is a protocol-relative + # URL, and browsers normalise backslashes to slashes so /\example.com would + # be treated as //example.com - https://github.com/simonw/datasette/issues/2680 + location = re.sub(r"^[/\\]+", "/", location) await asgi_send( send, "", diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 39a4c06b..86cdcc6b 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -104,3 +104,24 @@ def test_custom_route_pattern_with_slash_slash_302(custom_pages_client): response = custom_pages_client.get("//example.com/") assert response.status == 302 assert response.headers["location"] == "/example.com" + + +@pytest.mark.parametrize( + "path", + ( + "/\\example.com/", + "/\\\\example.com/", + "/\\/example.com/", + ), +) +def test_redirect_does_not_allow_backslash_open_redirect(custom_pages_client, path): + # https://github.com/simonw/datasette/issues/2680 + # Browsers normalise backslashes to forward slashes, so a Location of + # /\example.com would be treated as the protocol-relative //example.com + response = custom_pages_client.get(path) + assert response.status == 302 + location = response.headers["location"] + assert location == "/example.com" + # Must not start with anything a browser reads as protocol-relative + assert not location.startswith("//") + assert not location.startswith("/\\") From c31bb55011567d13f39d8096da4aef5b5a8a720a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:24:21 -0700 Subject: [PATCH 1162/1172] Add regression test for --default-deny index 500 (#2644) datasette --default-deny --root with no config file previously 500'd on the instance and database index pages: rendering them computes is_private (include_is_private=True), which references the anon_rules CTE, but that CTE was only defined when anonymous permission rules existed. This was fixed by the empty-anon_rules fallback added in 4b5fac9c; this commit adds a regression test that fails without that fallback (SQLite "no such table: anon_rules" -> 500). Co-Authored-By: Claude Fable 5 --- tests/test_default_deny.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_default_deny.py b/tests/test_default_deny.py index 81e95b84..f1e43064 100644 --- a/tests/test_default_deny.py +++ b/tests/test_default_deny.py @@ -127,3 +127,23 @@ async def test_default_deny_basic_permissions(): # Authenticated user without explicit permission should also be denied assert await ds.allowed(action="view-instance", actor={"id": "user"}) is False + + +@pytest.mark.asyncio +async def test_default_deny_root_no_config_index_does_not_500(): + # https://github.com/simonw/datasette/issues/2644 + # --default-deny --root with no config file must not 500 on the index + # pages. Rendering those pages computes is_private (include_is_private), + # which references the anon_rules CTE - that CTE must still be defined + # even when there are no anonymous permission rules at all. + ds = Datasette(default_deny=True) + ds.root_enabled = True + await ds.invoke_startup() + db = ds.add_memory_database("test_db_2644") + await db.execute_write("create table test_table (id integer primary key)") + await ds._refresh_schemas() + + cookie = ds.sign({"a": {"id": "root"}}, "actor") + for path in ("/", "/test_db_2644", "/test_db_2644/test_table"): + response = await ds.client.get(path, cookies={"ds_actor": cookie}) + assert response.status_code == 200, f"{path} returned {response.status_code}" From d5141a5778ac5ce6a6f4cfda990b2ab556b7f9f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:27:13 -0700 Subject: [PATCH 1163/1172] Fix /-/check 500 for query actions (#2756) _check_permission_for_actor() constructed child resources with resource_class(database=parent, table=child), but QueryResource takes a "query" argument, not "table", so /-/check?action=delete-query (and view-query / update-query) raised TypeError. Construct the resource positionally so it works for any child resource class. Co-Authored-By: Claude Fable 5 --- datasette/views/special.py | 8 +++++--- tests/test_permissions.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/datasette/views/special.py b/datasette/views/special.py index 75c54c3c..aa063ad6 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -497,11 +497,13 @@ async def _check_permission_for_actor(ds, action, parent, child, actor): if action_obj.resource_class is None: resource_obj = None elif action_obj.takes_parent and action_obj.takes_child: - # Child-level resource (e.g., TableResource, QueryResource) - resource_obj = action_obj.resource_class(database=parent, table=child) + # Child-level resource (e.g., TableResource, QueryResource). The child + # argument is named differently per resource class (table, query, ...), + # so pass positionally - https://github.com/simonw/datasette/issues/2756 + resource_obj = action_obj.resource_class(parent, child) elif action_obj.takes_parent: # Parent-level resource (e.g., DatabaseResource) - resource_obj = action_obj.resource_class(database=parent) + resource_obj = action_obj.resource_class(parent) else: # This shouldn't happen given validation in Action.__post_init__ return {"error": f"Invalid action configuration: {action}"}, 500 diff --git a/tests/test_permissions.py b/tests/test_permissions.py index e5e75432..8323fe92 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1733,6 +1733,29 @@ async def test_permission_check_view_requires_debug_permission(): assert data["allowed"] is True +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ("view-query", "update-query", "delete-query")) +async def test_permission_check_view_query_actions(action): + # https://github.com/simonw/datasette/issues/2756 + # QueryResource takes a "query" argument, not "table", so /-/check must + # not assume every child resource class accepts table= + ds = Datasette() + ds.root_enabled = True + root_token = await ds.create_token("root", handler="signed") + response = await ds.client.get( + f"/-/check.json?action={action}&parent=mydb&child=myquery", + headers={"Authorization": f"Bearer {root_token}"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["action"] == action + assert data["resource"] == { + "parent": "mydb", + "child": "myquery", + "path": "/mydb/myquery", + } + + @pytest.mark.asyncio async def test_root_allow_block_with_table_restricted_actor(): """ From 154ea483eaba7a636289a6972baeced7163acd60 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:30:12 -0700 Subject: [PATCH 1164/1172] Pass columns and rows to can_render for canned queries (#2711) The HTML branch of QueryView built an empty data dict before looping over register_output_renderer can_render callbacks, so renderers that depend on the result columns or rows (e.g. datasette-atom, datasette-ics) never appeared as export options for canned queries. Populate data with the executed query's rows, columns, SQL and query name. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 10 +++++++++- tests/test_plugins.py | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index a1647ca9..66887f9b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -762,7 +762,15 @@ class QueryView(View): ) ), ) - data = {} + data = { + "ok": query_error is None, + "rows": rows, + "columns": columns, + "query": {"sql": sql, "params": params}, + "query_name": stored_query.name if stored_query else None, + "database": database, + "table": None, + } headers.update( { "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 32276437..cf753c9e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -626,6 +626,31 @@ async def test_hook_register_output_renderer_can_render(ds_client): }.items() <= ds_client.ds._can_render_saw.items() +@pytest.mark.asyncio +async def test_hook_register_output_renderer_can_render_canned_query(ds_client): + # https://github.com/simonw/datasette/issues/2711 + # can_render for a canned query must be passed the query's columns, rows + # and SQL - previously it received an empty data dict, so renderers that + # depend on the columns (datasette-atom, datasette-ics) never showed up. + response = await ds_client.get("/fixtures/pragma_cache_size") + assert response.status_code == 200 + saw = ds_client.ds._can_render_saw + assert saw["columns"] == ["cache_size"] + assert len(saw["rows"]) == 1 + assert saw["sql"] == "PRAGMA cache_size;" + assert saw["query_name"] == "pragma_cache_size" + # The renderer's export link should therefore be offered + links = ( + Soup(response.text, "html.parser") + .find("p", {"class": "export-links"}) + .find_all("a") + ) + actual = [link["href"] for link in links] + assert any( + href.startswith("/fixtures/pragma_cache_size.testall") for href in actual + ) + + @pytest.mark.asyncio async def test_hook_prepare_jinja2_environment(ds_client): ds_client.ds._HELLO = "HI" From 92848c06b8cf5b8bb6b93088bb547810f9fce8b6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:43:32 -0700 Subject: [PATCH 1165/1172] Stop facet counts from wrapping (#2754) ul.tight-bullets li uses word-break: break-all so long facet labels can wrap, but that also let the count number break across lines. Wrap each count in a span.facet-count with white-space: nowrap so the label can still wrap while the count stays on one line. Co-Authored-By: Claude Fable 5 --- datasette/static/app.css | 5 +++++ datasette/templates/_facet_results.html | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 815f6db8..6d675d9f 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -706,6 +706,11 @@ button.core[type=button] { color: #666; padding-right: 0.25em; } +/* The label may wrap (word-break: break-all on the li) but the count should + stay on one line - https://github.com/simonw/datasette/issues/2754 */ +.facet-count { + white-space: nowrap; +} .facet-info li, .facet-info ul { margin: 0; diff --git a/datasette/templates/_facet_results.html b/datasette/templates/_facet_results.html index 034e9678..570bb37e 100644 --- a/datasette/templates/_facet_results.html +++ b/datasette/templates/_facet_results.html @@ -12,9 +12,9 @@
    {% for facet_value in facet_info.results %} {% if not facet_value.selected %} -
  • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
    • +
  • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
    • {% else %} -
  • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
    • +
  • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
    • {% endif %} {% endfor %} {% if facet_info.truncated %} From 9adb5416743a7312758e986c254baa1758228ad0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 06:42:08 -0700 Subject: [PATCH 1166/1172] Use asyncinject 0.7 results= seeding for per-request extras context asyncinject 0.7 fixed the parallel executor stalling when every initially-ready node is a seeded value, and made seeded values take precedence over registered functions. That lets the shared per-scope registries receive the per-request context directly via resolve_multi(results={'context': ...}) instead of the contextvars.ContextVar workaround. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 21 +++------------------ pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 2c3450b2..5cab52a4 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,4 +1,3 @@ -import contextvars import re from dataclasses import dataclass from enum import Enum @@ -6,11 +5,6 @@ from typing import ClassVar from asyncinject import Registry -# Per-request context for Extra.resolve(), so the asyncinject registries can -# be shared across requests. asyncio tasks copy the caller's context, so -# concurrent resolve() calls each see their own value. -_resolve_context = contextvars.ContextVar("datasette_extras_context") - def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") @@ -91,11 +85,6 @@ class ExtraRegistry: registry = self._scope_registries.get(scope) if registry is None: registry = Registry() - - async def context_provider(): - return _resolve_context.get() - - registry.register(context_provider, name="context") for cls in self.classes_for_scope(scope): registry.register(cls().resolve, name=cls.key()) self._scope_registries[scope] = registry @@ -117,13 +106,9 @@ class ExtraRegistry: async def resolve(self, requested, context, scope, include_internal=False): allowed_names = self._allowed_names_for_scope(scope, include_internal) requested_names = [name for name in requested if name in allowed_names] - token = _resolve_context.set(context) - try: - resolved = await self._registry_for_scope(scope).resolve_multi( - requested_names - ) - finally: - _resolve_context.reset(token) + resolved = await self._registry_for_scope(scope).resolve_multi( + requested_names, results={"context": context} + ) return {name: resolved[name] for name in requested_names} diff --git a/pyproject.toml b/pyproject.toml index 38085476..0d136d60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "mergedeep>=1.1.1", "itsdangerous>=1.1", "sqlite-utils>=3.30", - "asyncinject>=0.6.1", + "asyncinject>=0.7", "setuptools", "pip", ] From 648a34ce8196ecf02504c0daed594bd1cd540210 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:13:07 -0700 Subject: [PATCH 1167/1172] Fix for test I broke in 92848c06 refs #2754 --- tests/test_table_html.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 2e671d55..63e233fa 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -349,7 +349,11 @@ async def test_facet_display(ds_client): { "name": a.text, "qs": a["href"].split("?")[-1], - "count": int(str(a.parent).split("")[1].split("<")[0]), + "count": int( + a.parent.find( + "span", {"class": "facet-count"} + ).text.replace(",", "") + ), } for a in div.find("ul").find_all("a") ], @@ -695,7 +699,7 @@ async def test_table_html_foreign_key_facets(ds_client): assert response.status_code == 200 assert ( '
  • - 1
    • ' + ' data-facet-value="3">- 1' ) in response.text From 26f3b20e58bd2ad582d5fae326acf22b42627eb1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:29:27 -0700 Subject: [PATCH 1168/1172] Fix to our pytest plugin to better support pytest-cov Refs https://github.com/simonw/datasette/pulls#issuecomment-4681621052 --- datasette/_pytest_plugin.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 5fb6b473..103c616d 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -19,23 +19,38 @@ import weakref import pytest -from datasette.app import Datasette - _active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( "datasette_active_instances", default=None ) -_original_init = Datasette.__init__ +_original_init = None -def _tracking_init(self, *args, **kwargs): - _original_init(self, *args, **kwargs) - instances = _active_instances.get() - if instances is not None: - instances.append(weakref.ref(self)) +def _install_tracking(): + # datasette.app is imported lazily here rather than at module level: + # as a pytest11 entry point this module is imported during pytest + # startup, before pytest-cov starts measuring, so a module-level + # import would drag in all of datasette and make every import-time + # line in the package invisible to coverage + global _original_init + if _original_init is not None: + return + from datasette.app import Datasette + + _original_init = Datasette.__init__ + + def _tracking_init(self, *args, **kwargs): + _original_init(self, *args, **kwargs) + instances = _active_instances.get() + if instances is not None: + instances.append(weakref.ref(self)) + + Datasette.__init__ = _tracking_init -Datasette.__init__ = _tracking_init +def pytest_configure(config): + if _enabled(config): + _install_tracking() def pytest_addoption(parser): From 993169ae496aa0fa30271b6cb4dfc50202f6e7c1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 08:24:37 -0700 Subject: [PATCH 1169/1172] Release 1.0a33 Refs #2735, #2677, #2680, #2711, #2756, #2761, #2768, #2754 --- datasette/version.py | 2 +- docs/changelog.rst | 40 ++++++++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index 1e8c61d5..9536d459 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a32" +__version__ = "1.0a33" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19089dd1..48bef0bf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,15 +9,43 @@ Changelog 1.0a33 (unreleased) ------------------- -- Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) -- Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. See :ref:`json_api_extra` for the full list. -- New generated reference documentation for every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra`. -- ``?_extra=`` values can be separated by commas as well as repeated, e.g. ``?_extra=count,next_url``. Previously a comma-separated value that included ``columns`` failed to include the ``columns`` key in the response. -- The ``?_extra=private`` extra on arbitrary SQL query pages now correctly reflects whether the SQL execution permission is private to the current actor - it previously always returned ``false``. -- The ``?_extra=query`` extra on query pages now reports the named parameters that were actually bound when the query executed, including parameters declared in a stored query's ``params`` list. Magic ``_``-prefixed parameters are no longer echoed back with unbound values taken from the querystring. +Stored queries can now be edited and deleted through the web interface, and the JSON API ``?_extra=`` mechanism has been extended to cover row and query pages in addition to tables. This release also fixes two security issues: an identifier-quoting bug involving table and column names that contain ``]``, and an open redirect. + +Editing and deleting stored queries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) + +``?_extra=`` support for row and query pages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. + +New generated reference documentation describes every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra` for the full list. + +You can explore the new extras using this `Datasette extras API explorer tool `__. + +Other improvements and fixes to the extras mechanism: + - Extras that exist to serve the HTML interface (``filters``, ``actions``, ``display_rows``) are no longer advertised or reachable through the JSON API, where requesting them previously returned a 500 serialization error. - The pre-1.0 ``?_extras=`` (plural) parameter on row pages has been removed - use ``?_extra=foreign_key_tables`` instead. +Security fixes +~~~~~~~~~~~~~~ + +- Fixed an identifier-quoting bug in ``datasette.utils.escape_sqlite()``. Datasette uses this helper when constructing SQL around table and column names; identifiers containing ``]`` could break out of SQLite bracket quoting and alter the generated SQL, for example by adding a ``UNION SELECT``. Identifiers containing ``]`` are now quoted using double quotes instead. (:issue:`2677`) +- Fixed an open redirect vulnerability. Requesting a path such as ``/\example.com/`` produced a redirect with a ``Location: /\example.com`` header - browsers normalize backslashes to forward slashes, turning that into the protocol-relative URL ``//example.com`` and redirecting the user off-site. Any run of leading slashes and backslashes in a redirect path is now collapsed to a single slash. (:issue:`2680`) + +Bug fixes +~~~~~~~~~ + +- ``can_render()`` callbacks registered by the :ref:`register_output_renderer() ` plugin hook now receive the result ``rows`` and ``columns`` for stored queries. Previously renderers that inspect the available columns - such as `datasette-atom `__ and `datasette-ics `__ - never appeared as export options on stored query pages. (:issue:`2711`) +- Fixed a 500 error from the :ref:`/-/check ` permission debugging endpoint when checking query actions such as ``view-query``, ``update-query`` and ``delete-query``. (:issue:`2756`) +- Write queries that use a named parameter called ``:sql`` no longer fail with an error. (:issue:`2761`) +- :ref:`db.execute_isolated_fn() ` now works against immutable databases, using a read-only connection that bypasses the write thread. It previously always attempted to open a writable connection, which would fail - breaking features built on top of it, such as the SQL analysis step used when storing a query. An exception raised while opening the connection for an isolated function no longer crashes the write thread. (:issue:`2768`) +- Facet counts are now displayed on the same line as the facet value instead of wrapping onto a second line. (:issue:`2754`) +- Datasette's pytest plugin no longer imports the rest of Datasette at pytest startup time. This means plugin test suites using ``pytest-cov`` now correctly record coverage of code that runs when ``datasette`` modules are first imported. + .. _v1_0_a32: 1.0a32 (2026-05-31) From 1d4212122e5597f2e13625193fb7d45b25928447 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 10:36:16 -0700 Subject: [PATCH 1170/1172] Add release date for 1.0a33 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 48bef0bf..c0bd7e6b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changelog .. _v1_0_a33: -1.0a33 (unreleased) +1.0a33 (2026-06-11) ------------------- Stored queries can now be edited and deleted through the web interface, and the JSON API ``?_extra=`` mechanism has been extended to cover row and query pages in addition to tables. This release also fixes two security issues: an identifier-quoting bug involving table and column names that contain ``]``, and an open redirect. From fa86ac7b11c44ef80146db6eed25d88c954ee37a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 19:41:24 -0700 Subject: [PATCH 1171/1172] Clearer examples and descriptions for JSON API extras (#2773) Review of the generated ?_extra= documentation found several extras with no example output or with examples that needed explanation: - extras: now shows an abbreviated example of the toggle list and has a clearer description (which also improves the live API output) - set_column_type_ui: example of the shape seen with set-column-type permission, plus a note that it is null otherwise - column_types: live example generated from a table with an assigned column type instead of an empty {} - metadata: live table example now demonstrates a table description and column descriptions; row and query examples gained explanatory notes - expandable_columns, foreign_key_tables, facets_timed_out, next_url, renderers: notes explaining the shape of their output Also added docs_note cross-references to the relevant documentation: facets, pagination, render_cell and register_output_renderer plugin hooks, column type configuration and API, metadata, custom templates, permissions and foreign key label expansion. foreign_key_tables is now flagged as potentially executing additional queries. https://claude.ai/code/session_01EfjBe6E817m9XNFW7EX3Vm Co-authored-by: Claude --- datasette/views/table_extras.py | 182 +++++++++++++++++++++++++-- docs/json_api.rst | 215 ++++++++++++++++++++++++++------ docs/json_api_doc.py | 19 ++- 3 files changed, 367 insertions(+), 49 deletions(-) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index ce1d7bdf..948f3daa 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -184,6 +184,7 @@ class FacetResultsExtra(Extra): ) scopes = {ExtraScope.TABLE} expensive = True + docs_note = "See :ref:`facets` for details of how facets work." async def resolve(self, context, facet_instances): facet_results = {} @@ -215,7 +216,12 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): description = "Facet calculations that timed out" example = ExtraExample( - "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" + "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out", + note=( + "A list of the names of any facets that exceeded the " + ":ref:`setting_facet_time_limit_ms` time limit - an empty list " + "if every facet calculation completed." + ), ) scopes = {ExtraScope.TABLE} @@ -236,6 +242,9 @@ class SuggestedFacetsExtra(Extra): ) scopes = {ExtraScope.TABLE} expensive = True + docs_note = ( + "Suggestions are controlled by the :ref:`setting_suggest_facets` setting." + ) async def resolve(self, context, facet_instances): suggested_facets = [] @@ -278,7 +287,13 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" - example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") + example = ExtraExample( + "/fixtures/facetable.json?_size=1&_extra=next_url", + note=( + "``null`` if there are no more pages of results. " + "See :ref:`json_api_pagination`." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -366,6 +381,10 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" + docs_note = ( + "The contents of this block are not a stable part of the Datasette " + "API and may change without warning." + ) example = ExtraExample("/fixtures/facetable.json?_extra=debug") examples = { ExtraScope.ROW: ExtraExample( @@ -482,6 +501,10 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" + docs_note = ( + "See the :ref:`render_cell() plugin hook ` " + "documentation." + ) example = ExtraExample( value={ "rows": [ @@ -598,7 +621,28 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" - example = ExtraExample(value={}) + docs_note = ( + "An empty object if no column types have been assigned. Column types " + "can be assigned in :ref:`configuration " + "` or using the :ref:`set column " + "type API `." + ) + example = ExtraExample( + "/fixtures/facetable.json?_size=0&_extra=column_types", + note=( + "This example is from an instance where the ``tags`` column has " + "been assigned the ``json`` column type." + ), + ) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/facetable/1.json?_extra=column_types", + note=( + "This example is from an instance where the ``tags`` column " + "has been assigned the ``json`` column type." + ), + ) + } scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): @@ -615,7 +659,40 @@ class ColumnTypesExtra(Extra): class SetColumnTypeUiExtra(Extra): - description = "Column type UI metadata for this table" + description = "Information needed to build an interface for assigning column types" + docs_note = ( + "``null`` unless the current actor is allowed to use the :ref:`set " + "column type API ` for this table." + ) + example = ExtraExample( + value={ + "path": "/fixtures/facetable/-/set-column-type", + "columns": { + "created": { + "current": None, + "options": [ + {"name": "email", "description": "Email address"}, + {"name": "json", "description": "JSON data"}, + {"name": "url", "description": "URL"}, + ], + }, + "tags": { + "current": {"type": "json", "config": None}, + "options": [ + {"name": "email", "description": "Email address"}, + {"name": "json", "description": "JSON data"}, + {"name": "url", "description": "URL"}, + ], + }, + }, + }, + note=( + "Shape abbreviated to two columns, as seen by an actor with " + "``set-column-type`` permission. ``current`` is the column type " + "currently assigned to each column and ``options`` lists the " + "types that could be assigned to it." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -667,13 +744,33 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): description = "Metadata about the table, database or stored query" - example = ExtraExample("/fixtures/facetable.json?_extra=metadata") + docs_note = "See :ref:`metadata` for how to attach metadata to tables." + example = ExtraExample( + "/fixtures/facetable.json?_extra=metadata", + note=( + "This example is from an instance where the ``facetable`` table " + "has a metadata ``description`` and a :ref:`column description " + "` for its ``state`` column. The " + "``columns`` object is empty for tables with no column " + "descriptions." + ), + ) examples = { ExtraScope.ROW: ExtraExample( - "/fixtures/simple_primary_key/1.json?_extra=metadata" + "/fixtures/simple_primary_key/1.json?_extra=metadata", + note=( + "This table has no metadata, so only an empty ``columns`` " + "object is returned." + ), ), ExtraScope.QUERY: ExtraExample( - "/fixtures/neighborhood_search.json?text=town&_extra=metadata" + "/fixtures/neighborhood_search.json?text=town&_extra=metadata", + note=( + "For stored queries this returns the full configuration of " + "the query, including the :ref:`stored query options " + "`. For ``?sql=`` queries it returns an " + "empty object." + ), ), } scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} @@ -733,6 +830,10 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" + docs_note = ( + "A six character hex color, without the leading ``#``, derived from " + "a hash of the database name and used in the Datasette interface." + ) example = ExtraExample("/fixtures/facetable.json?_extra=database_color") examples = { ExtraScope.ROW: ExtraExample( @@ -780,6 +881,11 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" + docs_note = ( + "The first template in this list that exists will be used to render " + "the table on the HTML version of this page. See " + ":ref:`customization_custom_templates`." + ) example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") scopes = {ExtraScope.TABLE} @@ -793,6 +899,12 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): description = "Facet results sorted for display" + docs_note = ( + "The same data as ``facet_results``, as a list in the order used by " + "the HTML interface: facets from :ref:`facet configuration " + "` first, then other facets ordered by their number " + "of results." + ) example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" ) @@ -849,7 +961,15 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" - example = ExtraExample("/fixtures/facetable.json?_extra=renderers") + example = ExtraExample( + "/fixtures/facetable.json?_extra=renderers", + note=( + "Each key is the name of an output format, each value the URL " + "for this data in that format. Plugins can add additional " + "formats using the :ref:`register_output_renderer() plugin hook " + "`." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context, expandable_columns, query): @@ -887,6 +1007,10 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): description = "Whether this resource is private to the current actor" + docs_note = ( + "``true`` if the current actor can see this resource but an " + "anonymous user could not. See :ref:`authentication_permissions`." + ) example = ExtraExample("/fixtures/facetable.json?_extra=private") examples = { ExtraScope.ROW: ExtraExample( @@ -904,7 +1028,15 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" - example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") + docs_note = "See :ref:`expand_foreign_keys` for how to expand these labels." + example = ExtraExample( + "/fixtures/facetable.json?_extra=expandable_columns", + note=( + "Each item is a ``[foreign_key, label_column]`` pair: the " + "foreign key relationship, then the column in the other table " + "that would be used as the label for each expanded value." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -919,9 +1051,14 @@ class ExpandableColumnsExtra(Extra): class ForeignKeyTablesExtra(Extra): description = "Tables that link to this row using foreign keys" example = ExtraExample( - "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables", + note=( + "``count`` is the number of rows in the other table that " + "reference this row, and ``link`` is a URL to browse those rows." + ), ) scopes = {ExtraScope.ROW} + expensive = True async def resolve(self, context): return await context.foreign_key_tables( @@ -930,7 +1067,30 @@ class ForeignKeyTablesExtra(Extra): class ExtrasExtra(Extra): - description = "Available ?_extra= blocks" + description = "List of ?_extra= blocks that can be used on this page" + example = ExtraExample( + value=[ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": False, + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": True, + }, + ], + note=( + "Shape abbreviated from /fixtures/facetable.json?_extra=extras - " + "the full response lists every extra described on this page. " + "``toggle_url`` is the current URL with that extra added or " + "removed, and ``selected`` is ``true`` for extras included in " + "the current request." + ), + ) scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 6b595577..fbc3cf60 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -276,7 +276,7 @@ The available table extras are listed below. "select count(*) from facetable " ``facet_results`` - Results of facets calculated against this data (May execute additional queries.) + Results of facets calculated against this data (May execute additional queries. See :ref:`facets` for details of how facets work.) Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results. @@ -309,12 +309,14 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out`` + A list of the names of any facets that exceeded the :ref:`setting_facet_time_limit_ms` time limit - an empty list if every facet calculation completed. + .. code-block:: json [] ``suggested_facets`` - Suggestions for facets that might return interesting results (May execute additional queries.) + Suggestions for facets that might return interesting results (May execute additional queries. Suggestions are controlled by the :ref:`setting_suggest_facets` setting.) Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets. @@ -341,6 +343,8 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` + ``null`` if there are no more pages of results. See :ref:`json_api_pagination`. + .. code-block:: json "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" @@ -426,7 +430,7 @@ The available table extras are listed below. ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item per row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -452,7 +456,7 @@ The available table extras are listed below. } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/facetable.json?_extra=debug`` @@ -501,28 +505,108 @@ The available table extras are listed below. } ``column_types`` - Column type assignments for this table + Column type assignments for this table (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) - .. code-block:: json + ``GET /fixtures/facetable.json?_size=0&_extra=column_types`` - {} - -``set_column_type_ui`` - Column type UI metadata for this table - -``metadata`` - Metadata about the table, database or stored query - - ``GET /fixtures/facetable.json?_extra=metadata`` + This example is from an instance where the ``tags`` column has been assigned the ``json`` column type. .. code-block:: json { - "columns": {} + "tags": { + "type": "json", + "config": null + } + } + +``set_column_type_ui`` + Information needed to build an interface for assigning column types (``null`` unless the current actor is allowed to use the :ref:`set column type API ` for this table.) + + Shape abbreviated to two columns, as seen by an actor with ``set-column-type`` permission. ``current`` is the column type currently assigned to each column and ``options`` lists the types that could be assigned to it. + + .. code-block:: json + + { + "path": "/fixtures/facetable/-/set-column-type", + "columns": { + "created": { + "current": null, + "options": [ + { + "name": "email", + "description": "Email address" + }, + { + "name": "json", + "description": "JSON data" + }, + { + "name": "url", + "description": "URL" + } + ] + }, + "tags": { + "current": { + "type": "json", + "config": null + }, + "options": [ + { + "name": "email", + "description": "Email address" + }, + { + "name": "json", + "description": "JSON data" + }, + { + "name": "url", + "description": "URL" + } + ] + } + } + } + +``metadata`` + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) + + ``GET /fixtures/facetable.json?_extra=metadata`` + + This example is from an instance where the ``facetable`` table has a metadata ``description`` and a :ref:`column description ` for its ``state`` column. The ``columns`` object is empty for tables with no column descriptions. + + .. code-block:: json + + { + "description": "A demo table of places, used to demonstrate facets", + "columns": { + "state": "Two letter US state code" + } } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -543,7 +627,7 @@ The available table extras are listed below. "facetable" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/facetable.json?_extra=database_color`` @@ -556,6 +640,8 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_extra=renderers`` + Each key is the name of an output format, each value the URL for this data in that format. Plugins can add additional formats using the :ref:`register_output_renderer() plugin hook `. + .. code-block:: json { @@ -563,7 +649,7 @@ The available table extras are listed below. } ``custom_table_templates`` - Custom template names considered for this table + Custom template names considered for this table (The first template in this list that exists will be used to render the table on the HTML version of this page. See :ref:`customization_custom_templates`.) ``GET /fixtures/facetable.json?_extra=custom_table_templates`` @@ -576,7 +662,7 @@ The available table extras are listed below. ] ``sorted_facet_results`` - Facet results sorted for display + Facet results sorted for display (The same data as ``facet_results``, as a list in the order used by the HTML interface: facets from :ref:`facet configuration ` first, then other facets ordered by their number of results.) ``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results`` @@ -643,7 +729,7 @@ The available table extras are listed below. true ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/facetable.json?_extra=private`` @@ -652,10 +738,12 @@ The available table extras are listed below. false ``expandable_columns`` - Foreign key columns that can be expanded with labels + Foreign key columns that can be expanded with labels (See :ref:`expand_foreign_keys` for how to expand these labels.) ``GET /fixtures/facetable.json?_extra=expandable_columns`` + Each item is a ``[foreign_key, label_column]`` pair: the foreign key relationship, then the column in the other table that would be used as the label for each expanded value. + .. code-block:: json [ @@ -720,7 +808,7 @@ The following extras are available for row JSON responses. ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item for the requested row. The object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -741,7 +829,7 @@ The following extras are available for row JSON responses. } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/simple_primary_key/1.json?_extra=debug`` @@ -803,17 +891,28 @@ The following extras are available for row JSON responses. } ``column_types`` - Column type assignments for this table + Column type assignments for this table (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) + + ``GET /fixtures/facetable/1.json?_extra=column_types`` + + This example is from an instance where the ``tags`` column has been assigned the ``json`` column type. .. code-block:: json - {} + { + "tags": { + "type": "json", + "config": null + } + } ``metadata`` - Metadata about the table, database or stored query + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) ``GET /fixtures/simple_primary_key/1.json?_extra=metadata`` + This table has no metadata, so only an empty ``columns`` object is returned. + .. code-block:: json { @@ -821,7 +920,26 @@ The following extras are available for row JSON responses. } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -842,7 +960,7 @@ The following extras are available for row JSON responses. "simple_primary_key" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/simple_primary_key/1.json?_extra=database_color`` @@ -851,7 +969,7 @@ The following extras are available for row JSON responses. "9403e5" ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/simple_primary_key/1.json?_extra=private`` @@ -860,10 +978,12 @@ The following extras are available for row JSON responses. false ``foreign_key_tables`` - Tables that link to this row using foreign keys + Tables that link to this row using foreign keys (May execute additional queries.) ``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables`` + ``count`` is the number of rows in the other table that reference this row, and ``link`` is a URL to browse those rows. + .. code-block:: json [ @@ -921,7 +1041,7 @@ The following extras are available for arbitrary SQL query responses and stored, ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item per query result row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -941,7 +1061,7 @@ The following extras are available for arbitrary SQL query responses and stored, } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=debug`` @@ -1000,10 +1120,12 @@ The following extras are available for arbitrary SQL query responses and stored, } ``metadata`` - Metadata about the table, database or stored query + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) ``GET /fixtures/neighborhood_search.json?text=town&_extra=metadata`` + For stored queries this returns the full configuration of the query, including the :ref:`stored query options `. For ``?sql=`` queries it returns an empty object. + .. code-block:: json { @@ -1029,7 +1151,26 @@ The following extras are available for arbitrary SQL query responses and stored, } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -1041,7 +1182,7 @@ The following extras are available for arbitrary SQL query responses and stored, "fixtures" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database_color`` @@ -1050,7 +1191,7 @@ The following extras are available for arbitrary SQL query responses and stored, "9403e5" ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=private`` diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index 44ef4a42..422e67f4 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -93,9 +93,26 @@ async def _fetch_live_examples(scoped_classes): datasette = Datasette( [str(db_path)], settings={"num_sql_threads": 1}, + metadata={ + "databases": { + "fixtures": { + "tables": { + "facetable": { + "description": "A demo table of places, used to demonstrate facets", + "columns": {"state": "Two letter US state code"}, + } + } + } + } + }, config={ "databases": { "fixtures": { + "tables": { + "facetable": { + "column_types": {"tags": "json"}, + } + }, "queries": { "neighborhood_search": { "sql": textwrap.dedent(""" @@ -108,7 +125,7 @@ async def _fetch_live_examples(scoped_classes): """), "title": "Search neighborhoods", } - } + }, } } }, From d4cb8b464bf1cbe69a8921fc8c9315e04a5f49cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jun 2026 13:21:58 -0700 Subject: [PATCH 1172/1172] Fix for trace_child_tasks exception handling I had Claude Fable 5 review our use of contextvar and it spotted this place where exceptions were not correctly handled. --- datasette/tracer.py | 6 ++++-- tests/test_tracer.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/datasette/tracer.py b/datasette/tracer.py index 9e66613b..28f3cc09 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -27,8 +27,10 @@ def get_task_id(): @contextmanager def trace_child_tasks(): token = trace_task_id.set(get_task_id()) - yield - trace_task_id.reset(token) + try: + yield + finally: + trace_task_id.reset(token) @contextmanager diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 6cc80fc4..9db211d3 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -70,6 +70,19 @@ def test_trace_query_errors(): assert trace_info["traces"][-1]["error"] == "no such table: non_existent_table" +@pytest.mark.asyncio +async def test_trace_child_tasks_resets_contextvar_on_exception(): + from datasette import tracer + + before = tracer.trace_task_id.get() + with pytest.raises(ValueError): + with tracer.trace_child_tasks(): + assert tracer.trace_task_id.get() is not None + raise ValueError("simulated error") + # The contextvar must be reset even though the block raised + assert tracer.trace_task_id.get() == before + + def test_trace_parallel_queries(): with make_app_client(settings={"trace_debug": True}) as client: response = client.get("/parallel-queries?_trace=1")