From 7b41521b338304bd4065a9ec0db9b4c651ec6fed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Apr 2023 16:25:29 -0700 Subject: [PATCH 001/767] WIP new JSON for queries, refs #2049 --- datasette/app.py | 13 +- datasette/views/database.py | 329 ++++++++++++++++++++++++++++++++++++ 2 files changed, 340 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index d7dace67..966a6faf 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -33,7 +33,12 @@ from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound from .views.base import ureg -from .views.database import DatabaseDownload, DatabaseView, TableCreateView +from .views.database import ( + DatabaseDownload, + DatabaseView, + TableCreateView, + database_view, +) from .views.index import IndexView from .views.special import ( JsonDataView, @@ -1365,8 +1370,12 @@ class Datasette: r"/-/patterns$", ) add_route(DatabaseDownload.as_view(self), r"/(?P[^\/\.]+)\.db$") + # add_route( + # DatabaseView.as_view(self), r"/(?P[^\/\.]+)(\.(?P\w+))?$" + # ) add_route( - DatabaseView.as_view(self), r"/(?P[^\/\.]+)(\.(?P\w+))?$" + wrap_view(database_view, self), + r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") add_route( diff --git a/datasette/views/database.py b/datasette/views/database.py index dda82510..7b90a2db 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -757,3 +757,332 @@ async def _table_columns(datasette, database_name): for view_name in await db.view_names(): table_columns[view_name] = [] return table_columns + + +async def database_view(request, datasette): + return await database_view_impl(request, datasette) + + +async def database_view_impl( + request, + datasette, + canned_query=None, + _size=None, + named_parameters=None, + write=False, +): + db = await datasette.resolve_database(request) + database = db.name + params = {key: request.args.get(key) for key in request.args} + sql = "" + if "sql" in params: + sql = params.pop("sql") + _shape = None + if "_shape" in params: + _shape = params.pop("_shape") + + private = False + if canned_query: + # Respect canned query permissions + visible, private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-query", (database, canned_query)), + ("view-database", database), + "view-instance", + ], + ) + if not visible: + raise Forbidden("You do not have permission to view this query") + + else: + await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + + # Extract any :named parameters + named_parameters = named_parameters or await derive_named_parameters(db, sql) + named_parameter_values = { + named_parameter: params.get(named_parameter) or "" + for named_parameter in named_parameters + if not named_parameter.startswith("_") + } + + # Set to blank string if missing from params + for named_parameter in named_parameters: + if named_parameter not in params and not named_parameter.startswith("_"): + params[named_parameter] = "" + + extra_args = {} + if params.get("_timelimit"): + extra_args["custom_time_limit"] = int(params["_timelimit"]) + if _size: + extra_args["page_size"] = _size + + templates = [f"query-{to_css_class(database)}.html", "query.html"] + if canned_query: + templates.insert( + 0, + f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", + ) + + query_error = None + + # Execute query - as write or as read + if write: + raise NotImplementedError("Write queries not yet implemented") + # if request.method == "POST": + # # If database is immutable, return an error + # if not db.is_mutable: + # raise Forbidden("Database is immutable") + # body = await request.post_body() + # body = body.decode("utf-8").strip() + # if body.startswith("{") and body.endswith("}"): + # params = json.loads(body) + # # But we want key=value strings + # for key, value in params.items(): + # params[key] = str(value) + # else: + # params = dict(parse_qsl(body, keep_blank_values=True)) + # # Should we return JSON? + # should_return_json = ( + # request.headers.get("accept") == "application/json" + # or request.args.get("_json") + # or params.get("_json") + # ) + # if canned_query: + # params_for_query = MagicParameters(params, request, self.ds) + # else: + # params_for_query = params + # ok = None + # try: + # cursor = await self.ds.databases[database].execute_write( + # sql, params_for_query + # ) + # message = metadata.get( + # "on_success_message" + # ) or "Query executed, {} row{} affected".format( + # cursor.rowcount, "" if cursor.rowcount == 1 else "s" + # ) + # message_type = self.ds.INFO + # redirect_url = metadata.get("on_success_redirect") + # ok = True + # except Exception as e: + # message = metadata.get("on_error_message") or str(e) + # message_type = self.ds.ERROR + # redirect_url = metadata.get("on_error_redirect") + # ok = False + # if should_return_json: + # return Response.json( + # { + # "ok": ok, + # "message": message, + # "redirect": redirect_url, + # } + # ) + # else: + # self.ds.add_message(request, message, message_type) + # return self.redirect(request, redirect_url or request.path) + # else: + + # async def extra_template(): + # return { + # "request": request, + # "db_is_immutable": not db.is_mutable, + # "path_with_added_args": path_with_added_args, + # "path_with_removed_args": path_with_removed_args, + # "named_parameter_values": named_parameter_values, + # "canned_query": canned_query, + # "success_message": request.args.get("_success") or "", + # "canned_write": True, + # } + + # return ( + # { + # "database": database, + # "rows": [], + # "truncated": False, + # "columns": [], + # "query": {"sql": sql, "params": params}, + # "private": private, + # }, + # extra_template, + # templates, + # ) + + # Not a write + if canned_query: + params_for_query = MagicParameters(params, request, datasette) + else: + params_for_query = params + try: + results = await datasette.execute( + database, sql, params_for_query, truncate=True, **extra_args + ) + columns = [r[0] for r in results.description] + except sqlite3.DatabaseError as e: + query_error = e + results = None + columns = [] + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + + return Response.json( + { + "ok": True, + "rows": [dict(r) for r in results], + # "columns": columns, + # "database": database, + # "params": params, + # "sql": sql, + # "_shape": _shape, + # "named_parameters": named_parameters, + # "named_parameter_values": named_parameter_values, + # "extra_args": extra_args, + # "templates": templates, + } + ) + + async def extra_template(): + display_rows = [] + truncate_cells = datasette.setting("truncate_cells_html") + for row in results.rows if results else []: + display_row = [] + for column, value in zip(results.columns, row): + display_value = value + # Let the plugins have a go + # pylint: disable=no-member + plugin_display_value = None + for candidate in pm.hook.render_cell( + row=row, + value=value, + column=column, + table=None, + database=database, + datasette=self.ds, + request=request, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value is not None: + display_value = plugin_display_value + else: + if value in ("", None): + display_value = Markup(" ") + elif is_url(str(display_value).strip()): + display_value = markupsafe.Markup( + '{truncated_url}'.format( + url=markupsafe.escape(value.strip()), + truncated_url=markupsafe.escape( + truncate_url(value.strip(), truncate_cells) + ), + ) + ) + elif isinstance(display_value, bytes): + blob_url = path_with_format( + request=request, + format="blob", + extra_qs={ + "_blob_column": column, + "_blob_hash": hashlib.sha256(display_value).hexdigest(), + }, + ) + formatted = format_bytes(len(value)) + display_value = markupsafe.Markup( + '<Binary: {:,} byte{}>'.format( + blob_url, + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "", + len(value), + "" if len(value) == 1 else "s", + ) + ) + else: + display_value = str(value) + if truncate_cells and len(display_value) > truncate_cells: + display_value = display_value[:truncate_cells] + "\u2026" + display_row.append(display_value) + display_rows.append(display_row) + + # Show 'Edit SQL' button only if: + # - User is allowed to execute SQL + # - SQL is an approved SELECT statement + # - No magic parameters, so no :_ in the SQL string + edit_sql_url = None + is_validated_sql = False + try: + validate_sql_select(sql) + is_validated_sql = True + except InvalidSql: + pass + if allow_execute_sql and is_validated_sql and ":_" not in sql: + edit_sql_url = ( + self.ds.urls.database(database) + + "?" + + urlencode( + { + **{ + "sql": sql, + }, + **named_parameter_values, + } + ) + ) + + show_hide_hidden = "" + if metadata.get("hide_sql"): + if bool(params.get("_show_sql")): + show_hide_link = path_with_removed_args(request, {"_show_sql"}) + show_hide_text = "hide" + show_hide_hidden = '' + else: + show_hide_link = path_with_added_args(request, {"_show_sql": 1}) + show_hide_text = "show" + else: + if bool(params.get("_hide_sql")): + show_hide_link = path_with_removed_args(request, {"_hide_sql"}) + show_hide_text = "show" + show_hide_hidden = '' + else: + show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) + show_hide_text = "hide" + hide_sql = show_hide_text == "show" + return { + "display_rows": display_rows, + "custom_sql": True, + "named_parameter_values": named_parameter_values, + "editable": editable, + "canned_query": canned_query, + "edit_sql_url": edit_sql_url, + "metadata": metadata, + "settings": self.ds.settings_dict(), + "request": request, + "show_hide_link": self.ds.urls.path(show_hide_link), + "show_hide_text": show_hide_text, + "show_hide_hidden": markupsafe.Markup(show_hide_hidden), + "hide_sql": hide_sql, + "table_columns": await _table_columns(self.ds, database) + if allow_execute_sql + else {}, + } + + return ( + { + "ok": not query_error, + "database": database, + "query_name": canned_query, + "rows": results.rows if results else [], + "truncated": results.truncated if results else False, + "columns": columns, + "query": {"sql": sql, "params": params}, + "error": str(query_error) if query_error else None, + "private": private, + "allow_execute_sql": allow_execute_sql, + }, + extra_template, + templates, + 400 if query_error else 200, + ) From 40dc5f5c501c4c32120148d94ec7c9d130141571 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Apr 2023 17:04:26 -0700 Subject: [PATCH 002/767] WIP --- datasette/views/database.py | 227 +++++++++++++++++++++++++++++++++--- datasette/views/table.py | 1 - 2 files changed, 212 insertions(+), 16 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 7b90a2db..d097c933 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -12,6 +12,7 @@ import markupsafe from datasette.utils import ( add_cors_headers, await_me_maybe, + call_with_supported_arguments, derive_named_parameters, format_bytes, tilde_decode, @@ -763,6 +764,119 @@ async def database_view(request, datasette): return await database_view_impl(request, datasette) +async def database_index_view(request, datasette, db): + database = db.name + visible, private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-database", database), + "view-instance", + ], + ) + if not visible: + raise Forbidden("You do not have permission to view this database") + + metadata = (datasette.metadata("databases") or {}).get(database, {}) + datasette.update_with_inherited_metadata(metadata) + + table_counts = await db.table_counts(5) + hidden_table_names = set(await db.hidden_table_names()) + all_foreign_keys = await db.get_all_foreign_keys() + + views = [] + for view_name in await db.view_names(): + view_visible, view_private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-table", (database, view_name)), + ("view-database", database), + "view-instance", + ], + ) + if view_visible: + views.append( + { + "name": view_name, + "private": view_private, + } + ) + + tables = [] + for table in table_counts: + table_visible, table_private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-table", (database, table)), + ("view-database", database), + "view-instance", + ], + ) + if not table_visible: + continue + table_columns = await db.table_columns(table) + tables.append( + { + "name": table, + "columns": table_columns, + "primary_keys": await db.primary_keys(table), + "count": table_counts[table], + "hidden": table in hidden_table_names, + "fts_table": await db.fts_table(table), + "foreign_keys": all_foreign_keys[table], + "private": table_private, + } + ) + + tables.sort(key=lambda t: (t["hidden"], t["name"])) + canned_queries = [] + for query in (await datasette.get_canned_queries(database, request.actor)).values(): + query_visible, query_private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-query", (database, query["name"])), + ("view-database", database), + "view-instance", + ], + ) + if query_visible: + canned_queries.append(dict(query, private=query_private)) + + async def database_actions(): + links = [] + for hook in pm.hook.database_actions( + datasette=datasette, + database=database, + actor=request.actor, + request=request, + ): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + + attached_databases = [d.name for d in await db.attached_databases()] + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + return Response.json( + { + "database": db.name, + "private": private, + "path": datasette.urls.database(database), + "size": db.size, + "tables": tables, + "hidden_count": len([t for t in tables if t["hidden"]]), + "views": views, + "queries": canned_queries, + "allow_execute_sql": allow_execute_sql, + "table_columns": await _table_columns(datasette, database) + if allow_execute_sql + else {}, + } + ) + + async def database_view_impl( request, datasette, @@ -798,6 +912,12 @@ async def database_view_impl( else: await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + # If there's no sql, show the database index page + if not sql: + return await database_index_view(request, datasette, db) + + validate_sql_select(sql) + # Extract any :named parameters named_parameters = named_parameters or await derive_named_parameters(db, sql) named_parameter_values = { @@ -909,6 +1029,7 @@ async def database_view_impl( # ) # Not a write + rows = [] if canned_query: params_for_query = MagicParameters(params, request, datasette) else: @@ -918,6 +1039,7 @@ async def database_view_impl( database, sql, params_for_query, truncate=True, **extra_args ) columns = [r[0] for r in results.description] + rows = list(results.rows) except sqlite3.DatabaseError as e: query_error = e results = None @@ -927,21 +1049,96 @@ async def database_view_impl( request.actor, "execute-sql", database ) - return Response.json( - { - "ok": True, - "rows": [dict(r) for r in results], - # "columns": columns, - # "database": database, - # "params": params, - # "sql": sql, - # "_shape": _shape, - # "named_parameters": named_parameters, - # "named_parameter_values": named_parameter_values, - # "extra_args": extra_args, - # "templates": templates, - } - ) + format_ = request.url_vars.get("format") or "html" + + if format_ == "csv": + raise NotImplementedError("CSV format not yet implemented") + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=None, + database=db.name, + table=None, + request=request, + view_name="table", # TODO: should this be "query"? + # These will be deprecated in Datasette 1.0: + args=request.args, + data={ + "rows": rows, + }, # TODO what should this be? + ) + result = await await_me_maybe(result) + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(database)}.html", "query.html"] + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + r = Response.html( + await datasette.render_template( + template, + dict( + data, + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + # TODO: review up all of these hacks: + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=any(c["sortable"] for c in data["display_columns"]), + allow_execute_sql=await datasette.permission_allowed( + request.actor, "execute-sql", resolved.db.name + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="table", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + # if next_url: + # r.headers["link"] = f'<{next_url}>; rel="next"' + return r async def extra_template(): display_rows = [] diff --git a/datasette/views/table.py b/datasette/views/table.py index c102c103..e367a075 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -9,7 +9,6 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette import tracer -from datasette.renderer import json_renderer from datasette.utils import ( add_cors_headers, await_me_maybe, From 026429fadd7a1f4f85ebfda1bbfe882938f455f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 20:47:03 -0700 Subject: [PATCH 003/767] Work in progress on query view, refs #2049 --- datasette/views/database.py | 420 ++++++++++++++++++++++++++++++++++++ setup.py | 2 +- tests/test_cli_serve_get.py | 9 +- 3 files changed, 426 insertions(+), 5 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index d097c933..33b6702b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,3 +1,4 @@ +from asyncinject import Registry import os import hashlib import itertools @@ -877,6 +878,416 @@ async def database_index_view(request, datasette, db): ) +async def query_view( + request, + datasette, + canned_query=None, + _size=None, + named_parameters=None, + write=False, +): + print("query_view") + db = await datasette.resolve_database(request) + database = db.name + # TODO: Why do I do this? Is it to eliminate multi-args? + # It's going to break ?_extra=...&_extra=... + params = {key: request.args.get(key) for key in request.args} + sql = "" + if "sql" in params: + sql = params.pop("sql") + + # TODO: Behave differently for canned query here: + await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + + _shape = None + if "_shape" in params: + _shape = params.pop("_shape") + + # ?_shape=arrays - "rows" is the default option, shown above + # ?_shape=objects - "rows" is a list of JSON key/value objects + # ?_shape=array - an JSON array of objects + # ?_shape=array&_nl=on - a newline-separated list of JSON objects + # ?_shape=arrayfirst - a flat JSON array containing just the first value from each row + # ?_shape=object - a JSON object keyed using the primary keys of the rows + async def _results(_sql, _params): + # Returns (results, error (can be None)) + try: + return await db.execute(_sql, _params, truncate=True), None + except Exception as e: + return None, e + + async def shape_arrays(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return { + "ok": True, + "rows": [list(r) for r in results.rows], + "truncated": results.truncated, + } + + async def shape_objects(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return { + "ok": True, + "rows": [dict(r) for r in results.rows], + "truncated": results.truncated, + } + + async def shape_array(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return [dict(r) for r in results.rows] + + shape_fn = { + "arrays": shape_arrays, + "objects": shape_objects, + "array": shape_array, + # "arrayfirst": shape_arrayfirst, + # "object": shape_object, + }[_shape or "objects"] + + registry = Registry.from_dict( + { + "_results": _results, + "_shape": shape_fn, + }, + parallel=False, + ) + + results = await registry.resolve_multi( + ["_shape"], + results={ + "_sql": sql, + "_params": params, + }, + ) + + # If "shape" does not include "rows" we return that as the response + if "rows" not in results["_shape"]: + return Response.json(results["_shape"]) + + output = results["_shape"] + output.update(dict((k, v) for k, v in results.items() if not k.startswith("_"))) + + response = Response.json(output) + + assert False + + import pdb + + pdb.set_trace() + + if isinstance(output, dict) and output.get("ok") is False: + # TODO: Other error codes? + + response.status_code = 400 + + if datasette.cors: + add_cors_headers(response.headers) + + return response + + # registry = Registry( + # extra_count, + # extra_facet_results, + # extra_facets_timed_out, + # extra_suggested_facets, + # facet_instances, + # extra_human_description_en, + # extra_next_url, + # extra_columns, + # extra_primary_keys, + # run_display_columns_and_rows, + # extra_display_columns, + # extra_display_rows, + # extra_debug, + # extra_request, + # extra_query, + # extra_metadata, + # extra_extras, + # extra_database, + # extra_table, + # extra_database_color, + # extra_table_actions, + # extra_filters, + # extra_renderers, + # extra_custom_table_templates, + # extra_sorted_facet_results, + # extra_table_definition, + # extra_view_definition, + # extra_is_view, + # extra_private, + # extra_expandable_columns, + # extra_form_hidden_args, + # ) + + results = await registry.resolve_multi( + ["extra_{}".format(extra) for extra in extras] + ) + data = { + "ok": True, + "next": next_value and str(next_value) or None, + } + data.update( + { + key.replace("extra_", ""): value + for key, value in results.items() + if key.startswith("extra_") and key.replace("extra_", "") in extras + } + ) + raw_sqlite_rows = rows[:page_size] + data["rows"] = [dict(r) for r in raw_sqlite_rows] + + private = False + if canned_query: + # Respect canned query permissions + visible, private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-query", (database, canned_query)), + ("view-database", database), + "view-instance", + ], + ) + if not visible: + raise Forbidden("You do not have permission to view this query") + + else: + await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + + # If there's no sql, show the database index page + if not sql: + return await database_index_view(request, datasette, db) + + validate_sql_select(sql) + + # Extract any :named parameters + named_parameters = named_parameters or await derive_named_parameters(db, sql) + named_parameter_values = { + named_parameter: params.get(named_parameter) or "" + for named_parameter in named_parameters + if not named_parameter.startswith("_") + } + + # Set to blank string if missing from params + for named_parameter in named_parameters: + if named_parameter not in params and not named_parameter.startswith("_"): + params[named_parameter] = "" + + extra_args = {} + if params.get("_timelimit"): + extra_args["custom_time_limit"] = int(params["_timelimit"]) + if _size: + extra_args["page_size"] = _size + + templates = [f"query-{to_css_class(database)}.html", "query.html"] + if canned_query: + templates.insert( + 0, + f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", + ) + + query_error = None + + # Execute query - as write or as read + if write: + raise NotImplementedError("Write queries not yet implemented") + # if request.method == "POST": + # # If database is immutable, return an error + # if not db.is_mutable: + # raise Forbidden("Database is immutable") + # body = await request.post_body() + # body = body.decode("utf-8").strip() + # if body.startswith("{") and body.endswith("}"): + # params = json.loads(body) + # # But we want key=value strings + # for key, value in params.items(): + # params[key] = str(value) + # else: + # params = dict(parse_qsl(body, keep_blank_values=True)) + # # Should we return JSON? + # should_return_json = ( + # request.headers.get("accept") == "application/json" + # or request.args.get("_json") + # or params.get("_json") + # ) + # if canned_query: + # params_for_query = MagicParameters(params, request, self.ds) + # else: + # params_for_query = params + # ok = None + # try: + # cursor = await self.ds.databases[database].execute_write( + # sql, params_for_query + # ) + # message = metadata.get( + # "on_success_message" + # ) or "Query executed, {} row{} affected".format( + # cursor.rowcount, "" if cursor.rowcount == 1 else "s" + # ) + # message_type = self.ds.INFO + # redirect_url = metadata.get("on_success_redirect") + # ok = True + # except Exception as e: + # message = metadata.get("on_error_message") or str(e) + # message_type = self.ds.ERROR + # redirect_url = metadata.get("on_error_redirect") + # ok = False + # if should_return_json: + # return Response.json( + # { + # "ok": ok, + # "message": message, + # "redirect": redirect_url, + # } + # ) + # else: + # self.ds.add_message(request, message, message_type) + # return self.redirect(request, redirect_url or request.path) + # else: + + # async def extra_template(): + # return { + # "request": request, + # "db_is_immutable": not db.is_mutable, + # "path_with_added_args": path_with_added_args, + # "path_with_removed_args": path_with_removed_args, + # "named_parameter_values": named_parameter_values, + # "canned_query": canned_query, + # "success_message": request.args.get("_success") or "", + # "canned_write": True, + # } + + # return ( + # { + # "database": database, + # "rows": [], + # "truncated": False, + # "columns": [], + # "query": {"sql": sql, "params": params}, + # "private": private, + # }, + # extra_template, + # templates, + # ) + + # Not a write + rows = [] + if canned_query: + params_for_query = MagicParameters(params, request, datasette) + else: + params_for_query = params + try: + results = await datasette.execute( + database, sql, params_for_query, truncate=True, **extra_args + ) + columns = [r[0] for r in results.description] + rows = list(results.rows) + except sqlite3.DatabaseError as e: + query_error = e + results = None + columns = [] + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + + format_ = request.url_vars.get("format") or "html" + + if format_ == "csv": + raise NotImplementedError("CSV format not yet implemented") + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=None, + database=db.name, + table=None, + request=request, + view_name="table", # TODO: should this be "query"? + # These will be deprecated in Datasette 1.0: + args=request.args, + data={ + "rows": rows, + }, # TODO what should this be? + ) + result = await await_me_maybe(result) + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(database)}.html", "query.html"] + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + r = Response.html( + await datasette.render_template( + template, + dict( + data, + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + # TODO: review up all of these hacks: + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=any(c["sortable"] for c in data["display_columns"]), + allow_execute_sql=await datasette.permission_allowed( + request.actor, "execute-sql", resolved.db.name + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="table", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + # if next_url: + # r.headers["link"] = f'<{next_url}>; rel="next"' + return r + + async def database_view_impl( request, datasette, @@ -887,10 +1298,19 @@ async def database_view_impl( ): db = await datasette.resolve_database(request) database = db.name + + if request.args.get("sql", "").strip(): + return await query_view( + request, datasette, canned_query, _size, named_parameters, write + ) + + # Index page shows the tables/views/canned queries for this database + params = {key: request.args.get(key) for key in request.args} sql = "" if "sql" in params: sql = params.pop("sql") + _shape = None if "_shape" in params: _shape = params.pop("_shape") diff --git a/setup.py b/setup.py index d41e428a..b591869e 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ setup( "mergedeep>=1.1.1", "itsdangerous>=1.1", "sqlite-utils>=3.30", - "asyncinject>=0.5", + "asyncinject>=0.6", ], entry_points=""" [console_scripts] diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index ac44e1e2..e484a6db 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -1,6 +1,7 @@ from datasette.cli import cli, serve from datasette.plugins import pm from click.testing import CliRunner +from unittest.mock import ANY import textwrap import json @@ -35,11 +36,11 @@ def test_serve_with_get(tmp_path_factory): ], ) assert 0 == result.exit_code, result.output - assert { - "database": "_memory", + assert json.loads(result.output) == { + "ok": True, + "rows": [{"sqlite_version()": ANY}], "truncated": False, - "columns": ["sqlite_version()"], - }.items() <= json.loads(result.output).items() + } # The plugin should have created hello.txt assert (plugins_dir / "hello.txt").read_text() == "hello" From a706f34b923c5c503c1fa8e6a260925c61cec3fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 22:07:05 -0700 Subject: [PATCH 004/767] Remove debug lines --- datasette/views/database.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 33b6702b..cb31a762 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -975,12 +975,6 @@ async def query_view( response = Response.json(output) - assert False - - import pdb - - pdb.set_trace() - if isinstance(output, dict) and output.get("ok") is False: # TODO: Other error codes? From 8b86fb7fb4840b0351c25cf6ff40ca9f17a8fee7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:50:12 -0700 Subject: [PATCH 005/767] Better debugging --- datasette/utils/testing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index d4990784..7747f7ce 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -16,6 +16,9 @@ class TestResponse: def status(self): return self.httpx_response.status_code + def __repr__(self): + return "".format(self.httpx_response.url, self.status) + # Supports both for test-writing convenience @property def status_code(self): From 3304fd43a2492739e9803f513c4954a5fd1e374d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:51:05 -0700 Subject: [PATCH 006/767] refresh_schemas() on database view --- datasette/views/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index cb31a762..30016188 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -886,7 +886,6 @@ async def query_view( named_parameters=None, write=False, ): - print("query_view") db = await datasette.resolve_database(request) database = db.name # TODO: Why do I do this? Is it to eliminate multi-args? @@ -1290,6 +1289,7 @@ async def database_view_impl( named_parameters=None, write=False, ): + await datasette.refresh_schemas() db = await datasette.resolve_database(request) database = db.name From 6f903d5a98c59f8a602ef7de776ea631b7145ba8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:51:19 -0700 Subject: [PATCH 007/767] Fixed a test --- tests/test_api.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 780e9fa5..e700ad7b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -643,9 +643,6 @@ async def test_custom_sql(ds_client): "/fixtures.json?sql=select+content+from+simple_primary_key&_shape=objects" ) data = response.json() - assert {"sql": "select content from simple_primary_key", "params": {}} == data[ - "query" - ] assert [ {"content": "hello"}, {"content": "world"}, @@ -653,8 +650,6 @@ async def test_custom_sql(ds_client): {"content": "RENDER_CELL_DEMO"}, {"content": "RENDER_CELL_ASYNC"}, ] == data["rows"] - assert ["content"] == data["columns"] - assert "fixtures" == data["database"] assert not data["truncated"] From fdb141f6225ddf7209dc6abff0c1691bbb675465 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:51:29 -0700 Subject: [PATCH 008/767] shape_arrayfirst for query view --- datasette/views/database.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 30016188..eac57cd3 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -941,11 +941,17 @@ async def query_view( return {"ok": False, "error": str(error)} return [dict(r) for r in results.rows] + async def shape_arrayfirst(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return [r[0] for r in results.rows] + shape_fn = { "arrays": shape_arrays, "objects": shape_objects, "array": shape_array, - # "arrayfirst": shape_arrayfirst, + "arrayfirst": shape_arrayfirst, # "object": shape_object, }[_shape or "objects"] From 6ae5312158540aa12c4fc714404c0f3c1a72e4d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 May 2023 18:44:07 -0700 Subject: [PATCH 009/767] ?sql=... now displays HTML --- datasette/views/database.py | 158 +++++++++++++++++++++++++++++++++--- 1 file changed, 145 insertions(+), 13 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index eac57cd3..455ebd1f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -12,10 +12,12 @@ import markupsafe from datasette.utils import ( add_cors_headers, + append_querystring, await_me_maybe, call_with_supported_arguments, derive_named_parameters, format_bytes, + path_with_replaced_args, tilde_decode, to_css_class, validate_sql_select, @@ -887,6 +889,145 @@ async def query_view( write=False, ): db = await datasette.resolve_database(request) + + format_ = request.url_vars.get("format") or "html" + force_shape = None + if format_ == "html": + force_shape = "arrays" + + data = await query_view_data( + request, + datasette, + canned_query=canned_query, + _size=_size, + named_parameters=named_parameters, + write=write, + force_shape=force_shape, + ) + if format_ == "csv": + raise NotImplementedError("CSV format not yet implemented") + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=None, + database=db.name, + table=None, + request=request, + view_name="table", # TODO: should this be "query"? + # These will be deprecated in Datasette 1.0: + args=request.args, + data={ + "rows": rows, + }, # TODO what should this be? + ) + result = await await_me_maybe(result) + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(db.name)}.html", "query.html"] + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + metadata = (datasette.metadata("databases") or {}).get(db.name, {}) + datasette.update_with_inherited_metadata(metadata) + + r = Response.html( + await datasette.render_template( + template, + dict( + data, + database=db.name, + database_color=lambda database: "ff0000", + metadata=metadata, + display_rows=data["rows"], + renderers={}, + query={ + "sql": request.args.get("sql"), + }, + editable=True, + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + # TODO: review up all of these hacks: + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=False, + allow_execute_sql=await datasette.permission_allowed( + request.actor, "execute-sql", db.name + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="table", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + # if next_url: + # r.headers["link"] = f'<{next_url}>; rel="next"' + return r + + response = Response.json(data) + + if isinstance(data, dict) and data.get("ok") is False: + # TODO: Other error codes? + + response.status_code = 400 + + if datasette.cors: + add_cors_headers(response.headers) + + return response + + +async def query_view_data( + request, + datasette, + canned_query=None, + _size=None, + named_parameters=None, + write=False, + force_shape=None, +): + db = await datasette.resolve_database(request) database = db.name # TODO: Why do I do this? Is it to eliminate multi-args? # It's going to break ?_extra=...&_extra=... @@ -898,11 +1039,11 @@ async def query_view( # TODO: Behave differently for canned query here: await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) - _shape = None + _shape = force_shape if "_shape" in params: _shape = params.pop("_shape") - # ?_shape=arrays - "rows" is the default option, shown above + # ?_shape=arrays # ?_shape=objects - "rows" is a list of JSON key/value objects # ?_shape=array - an JSON array of objects # ?_shape=array&_nl=on - a newline-separated list of JSON objects @@ -921,6 +1062,7 @@ async def query_view( return {"ok": False, "error": str(error)} return { "ok": True, + "columns": [r[0] for r in results.description], "rows": [list(r) for r in results.rows], "truncated": results.truncated, } @@ -978,17 +1120,7 @@ async def query_view( output = results["_shape"] output.update(dict((k, v) for k, v in results.items() if not k.startswith("_"))) - response = Response.json(output) - - if isinstance(output, dict) and output.get("ok") is False: - # TODO: Other error codes? - - response.status_code = 400 - - if datasette.cors: - add_cors_headers(response.headers) - - return response + return output # registry = Registry( # extra_count, From bbbfdb034ca02c1aa3f02de0e8edf12d8190175c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 20:36:10 -0700 Subject: [PATCH 010/767] Add setuptools to dependencies Refs #2065 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b591869e..1139ff31 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ setup( "itsdangerous>=1.1", "sqlite-utils>=3.30", "asyncinject>=0.6", + "setuptools", ], entry_points=""" [console_scripts] From 3a7be0c5b1911554b7f7298ef459259f24a1a9d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 21:20:38 -0700 Subject: [PATCH 011/767] Hopeful fix for Python 3.7 httpx failure, refs #2066 --- tests/test_csv.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/test_csv.py b/tests/test_csv.py index c43e528b..ed83d685 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -6,6 +6,7 @@ from .fixtures import ( # noqa app_client_with_cors, app_client_with_trace, ) +import urllib.parse EXPECTED_TABLE_CSV = """id,content 1,hello @@ -154,11 +155,24 @@ async def test_csv_with_non_ascii_characters(ds_client): def test_max_csv_mb(app_client_csv_max_mb_one): + # This query deliberately generates a really long string + # should be 100*100*100*2 = roughly 2MB response = app_client_csv_max_mb_one.get( - ( - "/fixtures.csv?sql=select+'{}'+" - "from+compound_three_primary_keys&_stream=1&_size=max" - ).format("abcdefg" * 10000) + "/fixtures.csv?" + + urllib.parse.urlencode( + { + "sql": """ + select group_concat('ab', '') + from json_each(json_array({lots})), + json_each(json_array({lots})), + json_each(json_array({lots})) + """.format( + lots=", ".join(str(i) for i in range(100)) + ), + "_stream": 1, + "_size": "max", + } + ), ) # It's a 200 because we started streaming before we knew the error assert response.status == 200 From 305655c816a70fdfa729b37529d8c51490d4f4d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 22:07:35 -0700 Subject: [PATCH 012/767] Add pip as a dependency too, for Rye - refs #2065 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1139ff31..0eedadb2 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ setup( "sqlite-utils>=3.30", "asyncinject>=0.6", "setuptools", + "pip", ], entry_points=""" [console_scripts] From 01353c7ee83ba2c58b42bdc35f3b26dfc06159ac Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 May 2023 11:44:27 -0700 Subject: [PATCH 013/767] Build docs with 3.11 on ReadTheDocs Inspired by https://github.com/simonw/sqlite-utils/issues/540 --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e157fb9c..5b30e75a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.11" sphinx: configuration: docs/conf.py From 59c52b5874045d884bac8bac35a4ecbfdde4e73c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 May 2023 09:24:28 -0700 Subject: [PATCH 014/767] Action: Deploy a Datasette branch preview to Vercel Closes #2070 --- .github/workflows/deploy-branch-preview.yml | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/deploy-branch-preview.yml diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml new file mode 100644 index 00000000..872aff71 --- /dev/null +++ b/.github/workflows/deploy-branch-preview.yml @@ -0,0 +1,35 @@ +name: Deploy a Datasette branch preview to Vercel + +on: + workflow_dispatch: + inputs: + branch: + description: "Branch to deploy" + required: true + type: string + +jobs: + deploy-branch-preview: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: | + pip install datasette-publish-vercel + - name: Deploy the preview + env: + VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} + run: | + export BRANCH="${{ github.event.inputs.branch }}" + wget https://latest.datasette.io/fixtures.db + datasette publish vercel fixtures.db \ + --branch $BRANCH \ + --project "datasette-preview-$BRANCH" \ + --token $VERCEL_TOKEN \ + --scope datasette \ + --about "Preview of $BRANCH" \ + --about_url "https://github.com/simonw/datasette/tree/$BRANCH" From 3f39fba7ea74e54673cce8a71d6c1baeaa84c513 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 11:35:34 -0700 Subject: [PATCH 015/767] datasette.utils.check_callable(obj) - refs #2078 --- datasette/utils/callable.py | 25 ++++++++++++++++++++ tests/test_utils_callable.py | 46 ++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 datasette/utils/callable.py create mode 100644 tests/test_utils_callable.py diff --git a/datasette/utils/callable.py b/datasette/utils/callable.py new file mode 100644 index 00000000..5b8a30ac --- /dev/null +++ b/datasette/utils/callable.py @@ -0,0 +1,25 @@ +import asyncio +import types +from typing import NamedTuple, Any + + +class CallableStatus(NamedTuple): + is_callable: bool + is_async_callable: bool + + +def check_callable(obj: Any) -> CallableStatus: + if not callable(obj): + return CallableStatus(False, False) + + if isinstance(obj, type): + # It's a class + return CallableStatus(True, False) + + if isinstance(obj, types.FunctionType): + return CallableStatus(True, asyncio.iscoroutinefunction(obj)) + + if hasattr(obj, "__call__"): + return CallableStatus(True, asyncio.iscoroutinefunction(obj.__call__)) + + assert False, "obj {} is somehow callable with no __call__ method".format(repr(obj)) diff --git a/tests/test_utils_callable.py b/tests/test_utils_callable.py new file mode 100644 index 00000000..d1d0aac5 --- /dev/null +++ b/tests/test_utils_callable.py @@ -0,0 +1,46 @@ +from datasette.utils.callable import check_callable +import pytest + + +class AsyncClass: + async def __call__(self): + pass + + +class NotAsyncClass: + def __call__(self): + pass + + +class ClassNoCall: + pass + + +async def async_func(): + pass + + +def non_async_func(): + pass + + +@pytest.mark.parametrize( + "obj,expected_is_callable,expected_is_async_callable", + ( + (async_func, True, True), + (non_async_func, True, False), + (AsyncClass(), True, True), + (NotAsyncClass(), True, False), + (ClassNoCall(), False, False), + (AsyncClass, True, False), + (NotAsyncClass, True, False), + (ClassNoCall, True, False), + ("", False, False), + (1, False, False), + (str, True, False), + ), +) +def test_check_callable(obj, expected_is_callable, expected_is_async_callable): + status = check_callable(obj) + assert status.is_callable == expected_is_callable + assert status.is_async_callable == expected_is_async_callable From 080577106167046428c946c9978631dd59e10181 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 11:49:40 -0700 Subject: [PATCH 016/767] Rename callable.py to check_callable.py, refs #2078 --- datasette/utils/{callable.py => check_callable.py} | 0 tests/{test_utils_callable.py => test_utils_check_callable.py} | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename datasette/utils/{callable.py => check_callable.py} (100%) rename tests/{test_utils_callable.py => test_utils_check_callable.py} (94%) diff --git a/datasette/utils/callable.py b/datasette/utils/check_callable.py similarity index 100% rename from datasette/utils/callable.py rename to datasette/utils/check_callable.py diff --git a/tests/test_utils_callable.py b/tests/test_utils_check_callable.py similarity index 94% rename from tests/test_utils_callable.py rename to tests/test_utils_check_callable.py index d1d0aac5..4f72f9ff 100644 --- a/tests/test_utils_callable.py +++ b/tests/test_utils_check_callable.py @@ -1,4 +1,4 @@ -from datasette.utils.callable import check_callable +from datasette.utils.check_callable import check_callable import pytest From 94882aa72b5ef4afb267447de27f0ec2140400d8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 15:05:58 -0700 Subject: [PATCH 017/767] --cors Access-Control-Max-Age: 3600, closes #2079 --- datasette/utils/__init__.py | 1 + docs/json_api.rst | 18 +++++++++++++++++- tests/test_api.py | 2 ++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 925c6560..c388673d 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1141,6 +1141,7 @@ def add_cors_headers(headers): headers["Access-Control-Allow-Headers"] = "Authorization, Content-Type" headers["Access-Control-Expose-Headers"] = "Link" headers["Access-Control-Allow-Methods"] = "GET, POST, HEAD, OPTIONS" + headers["Access-Control-Max-Age"] = "3600" _TILDE_ENCODING_SAFE = frozenset( diff --git a/docs/json_api.rst b/docs/json_api.rst index 7b130c58..c273c2a8 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -454,12 +454,28 @@ Enabling CORS ------------- If you start Datasette with the ``--cors`` option, each JSON endpoint will be -served with the following additional HTTP headers:: +served with the following additional HTTP headers: + +.. [[[cog + from datasette.utils import add_cors_headers + import textwrap + headers = {} + add_cors_headers(headers) + output = "\n".join("{}: {}".format(k, v) for k, v in headers.items()) + cog.out("\n::\n\n") + cog.out(textwrap.indent(output, ' ')) + cog.out("\n\n") +.. ]]] + +:: Access-Control-Allow-Origin: * Access-Control-Allow-Headers: Authorization, Content-Type Access-Control-Expose-Headers: Link Access-Control-Allow-Methods: GET, POST, HEAD, OPTIONS + Access-Control-Max-Age: 3600 + +.. [[[end]]] This allows JavaScript running on any domain to make cross-origin requests to interact with the Datasette API. diff --git a/tests/test_api.py b/tests/test_api.py index e700ad7b..e7d8d849 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -936,6 +936,7 @@ def test_cors( assert ( response.headers["Access-Control-Allow-Methods"] == "GET, POST, HEAD, OPTIONS" ) + assert response.headers["Access-Control-Max-Age"] == "3600" # Same request to app_client_two_attached_databases_one_immutable # should not have those headers - I'm using that fixture because # regular app_client doesn't have immutable fixtures.db which means @@ -946,6 +947,7 @@ def test_cors( assert "Access-Control-Allow-Headers" not in response.headers assert "Access-Control-Expose-Headers" not in response.headers assert "Access-Control-Allow-Methods" not in response.headers + assert "Access-Control-Max-Age" not in response.headers @pytest.mark.parametrize( From 8ea00e038d7d43d173cebdca21a31e774ff33f1c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 17:18:43 -0700 Subject: [PATCH 018/767] New View base class (#2080) * New View base class, closes #2078 * Use new View subclass for PatternPortfolioView --- datasette/app.py | 40 +++++++++++++++++- datasette/views/base.py | 37 +++++++++++++++++ datasette/views/special.py | 19 +++++---- tests/test_base_view.py | 84 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 10 deletions(-) create mode 100644 tests/test_base_view.py diff --git a/datasette/app.py b/datasette/app.py index 966a6faf..8bc6518f 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -17,6 +17,7 @@ import secrets import sys import threading import time +import types import urllib.parse from concurrent import futures from pathlib import Path @@ -1366,7 +1367,7 @@ class Datasette: r"/-/allow-debug$", ) add_route( - PatternPortfolioView.as_view(self), + wrap_view(PatternPortfolioView, self), r"/-/patterns$", ) add_route(DatabaseDownload.as_view(self), r"/(?P[^\/\.]+)\.db$") @@ -1682,7 +1683,42 @@ def _cleaner_task_str(task): return _cleaner_task_str_re.sub("", s) -def wrap_view(view_fn, datasette): +def wrap_view(view_fn_or_class, datasette): + is_function = isinstance(view_fn_or_class, types.FunctionType) + if is_function: + return wrap_view_function(view_fn_or_class, datasette) + else: + if not isinstance(view_fn_or_class, type): + raise ValueError("view_fn_or_class must be a function or a class") + return wrap_view_class(view_fn_or_class, datasette) + + +def wrap_view_class(view_class, datasette): + async def async_view_for_class(request, send): + instance = view_class() + if inspect.iscoroutinefunction(instance.__call__): + return await async_call_with_supported_arguments( + instance.__call__, + scope=request.scope, + receive=request.receive, + send=send, + request=request, + datasette=datasette, + ) + else: + return call_with_supported_arguments( + instance.__call__, + scope=request.scope, + receive=request.receive, + send=send, + request=request, + datasette=datasette, + ) + + return async_view_for_class + + +def wrap_view_function(view_fn, datasette): @functools.wraps(view_fn) async def async_view_fn(request, send): if inspect.iscoroutinefunction(view_fn): diff --git a/datasette/views/base.py b/datasette/views/base.py index 927d1aff..94645cd8 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -53,6 +53,43 @@ class DatasetteError(Exception): self.message_is_html = message_is_html +class View: + async def head(self, request, datasette): + if not hasattr(self, "get"): + return await self.method_not_allowed(request) + response = await self.get(request, datasette) + response.body = "" + return response + + async def method_not_allowed(self, request): + if ( + request.path.endswith(".json") + or request.headers.get("content-type") == "application/json" + ): + response = Response.json( + {"ok": False, "error": "Method not allowed"}, status=405 + ) + else: + response = Response.text("Method not allowed", status=405) + return response + + async def options(self, request, datasette): + response = Response.text("ok") + response.headers["allow"] = ", ".join( + method.upper() + for method in ("head", "get", "post", "put", "patch", "delete") + if hasattr(self, method) + ) + return response + + async def __call__(self, request, datasette): + try: + handler = getattr(self, request.method.lower()) + except AttributeError: + return await self.method_not_allowed(request) + return await handler(request, datasette) + + class BaseView: ds = None has_json_alternate = True diff --git a/datasette/views/special.py b/datasette/views/special.py index 1aeb4be6..03e085d6 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -6,7 +6,7 @@ from datasette.utils import ( tilde_encode, tilde_decode, ) -from .base import BaseView +from .base import BaseView, View import secrets import urllib @@ -57,13 +57,16 @@ class JsonDataView(BaseView): ) -class PatternPortfolioView(BaseView): - name = "patterns" - has_json_alternate = False - - async def get(self, request): - await self.ds.ensure_permissions(request.actor, ["view-instance"]) - return await self.render(["patterns.html"], request=request) +class PatternPortfolioView(View): + async def get(self, request, datasette): + await datasette.ensure_permissions(request.actor, ["view-instance"]) + return Response.html( + await datasette.render_template( + "patterns.html", + request=request, + view_name="patterns", + ) + ) class AuthTokenView(BaseView): diff --git a/tests/test_base_view.py b/tests/test_base_view.py new file mode 100644 index 00000000..2cd4d601 --- /dev/null +++ b/tests/test_base_view.py @@ -0,0 +1,84 @@ +from datasette.views.base import View +from datasette import Request, Response +from datasette.app import Datasette +import json +import pytest + + +class GetView(View): + async def get(self, request, datasette): + return Response.json( + { + "absolute_url": datasette.absolute_url(request, "/"), + "request_path": request.path, + } + ) + + +class GetAndPostView(GetView): + async def post(self, request, datasette): + return Response.json( + { + "method": request.method, + "absolute_url": datasette.absolute_url(request, "/"), + "request_path": request.path, + } + ) + + +@pytest.mark.asyncio +async def test_get_view(): + v = GetView() + datasette = Datasette() + response = await v(Request.fake("/foo"), datasette) + assert json.loads(response.body) == { + "absolute_url": "http://localhost/", + "request_path": "/foo", + } + # Try a HEAD request + head_response = await v(Request.fake("/foo", method="HEAD"), datasette) + assert head_response.body == "" + assert head_response.status == 200 + # And OPTIONS + options_response = await v(Request.fake("/foo", method="OPTIONS"), datasette) + assert options_response.body == "ok" + assert options_response.status == 200 + assert options_response.headers["allow"] == "HEAD, GET" + # And POST + post_response = await v(Request.fake("/foo", method="POST"), datasette) + assert post_response.body == "Method not allowed" + assert post_response.status == 405 + # And POST with .json extension + post_json_response = await v(Request.fake("/foo.json", method="POST"), datasette) + assert json.loads(post_json_response.body) == { + "ok": False, + "error": "Method not allowed", + } + assert post_json_response.status == 405 + + +@pytest.mark.asyncio +async def test_post_view(): + v = GetAndPostView() + datasette = Datasette() + response = await v(Request.fake("/foo"), datasette) + assert json.loads(response.body) == { + "absolute_url": "http://localhost/", + "request_path": "/foo", + } + # Try a HEAD request + head_response = await v(Request.fake("/foo", method="HEAD"), datasette) + assert head_response.body == "" + assert head_response.status == 200 + # And OPTIONS + options_response = await v(Request.fake("/foo", method="OPTIONS"), datasette) + assert options_response.body == "ok" + assert options_response.status == 200 + assert options_response.headers["allow"] == "HEAD, GET, POST" + # And POST + post_response = await v(Request.fake("/foo", method="POST"), datasette) + assert json.loads(post_response.body) == { + "method": "POST", + "absolute_url": "http://localhost/", + "request_path": "/foo", + } From d1d78ec0ebe34463eb643ac1eae1dee4c62a2031 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 23 Jun 2023 13:06:35 -0700 Subject: [PATCH 019/767] Better docs for startup() hook --- docs/plugin_hooks.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index a4c9d98f..97306529 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -869,7 +869,9 @@ Examples: `datasette-cors `__, `dat startup(datasette) ------------------ -This hook fires when the Datasette application server first starts up. You can implement a regular function, for example to validate required plugin configuration: +This hook fires when the Datasette application server first starts up. + +Here is an example that validates required plugin configuration. The server will fail to start and show an error if the validation check fails: .. code-block:: python @@ -880,7 +882,7 @@ This hook fires when the Datasette application server first starts up. You can i "required-setting" in config ), "my-plugin requires setting required-setting" -Or you can return an async function which will be awaited on startup. Use this option if you need to make any database queries: +You can also return an async function, which will be awaited on startup. Use this option if you need to execute any database queries, for example this function which creates the ``my_table`` database table if it does not yet exist: .. code-block:: python From 68223784167fdec4e7ebfca56002a6548ba7b423 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Jun 2023 19:00:33 -0700 Subject: [PATCH 020/767] Prototype of rst_docs_for_dataclass mechanism, refs #1510 --- datasette/context.py | 97 +++++++++++++++++++++++++++++++++++++++ docs/conf.py | 7 ++- docs/index.rst | 1 + docs/jsoncontext.py | 28 +++++++++++ docs/template_context.rst | 29 ++++++++++++ 5 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 datasette/context.py create mode 100644 docs/jsoncontext.py create mode 100644 docs/template_context.rst diff --git a/datasette/context.py b/datasette/context.py new file mode 100644 index 00000000..c5927d1d --- /dev/null +++ b/datasette/context.py @@ -0,0 +1,97 @@ +from typing import List, Dict, Optional, Any +from dataclasses import dataclass, field + + +def doc(documentation): + return field(metadata={"doc": documentation}) + + +def is_builtin_type(obj): + return isinstance( + obj, + tuple( + x.__class__ + for x in (int, float, str, bool, bytes, list, tuple, dict, set, frozenset) + ), + ) + + +def rst_docs_for_dataclass(klass: Any) -> str: + """Generate reStructuredText (reST) docs for a dataclass.""" + docs = [] + + # Class name and docstring + docs.append(klass.__name__) + docs.append("-" * len(klass.__name__)) + docs.append("") + if klass.__doc__: + docs.append(klass.__doc__) + docs.append("") + + # Dataclass fields + docs.append("Fields") + docs.append("~~~~~~") + docs.append("") + + for name, field_info in klass.__dataclass_fields__.items(): + if is_builtin_type(field_info.type): + # + type_name = field_info.type.__name__ + else: + # List[str] + type_name = str(field_info.type).replace("typing.", "") + docs.append(f':{name} - ``{type_name}``: {field_info.metadata.get("doc", "")}') + + return "\n".join(docs) + + +@dataclass +class ForeignKey: + incoming: List[Dict] + outgoing: List[Dict] + + +@dataclass +class Table: + "A table is a useful thing" + name: str = doc("The name of the table") + columns: List[str] = doc("List of column names in the table") + primary_keys: List[str] = doc("List of column names that are primary keys") + count: int = doc("Number of rows in the table") + hidden: bool = doc( + "Should this table default to being hidden in the main database UI?" + ) + fts_table: Optional[str] = doc( + "If this table has FTS support, the accompanying FTS table name" + ) + foreign_keys: ForeignKey = doc("List of foreign keys for this table") + private: bool = doc("Private tables are not visible to signed-out anonymous users") + + +@dataclass +class View: + name: str + private: bool + + +@dataclass +class Query: + title: str + sql: str + name: str + private: bool + + +@dataclass +class Database: + content: str + private: bool + path: str + size: int + tables: List[Table] + hidden_count: int + views: List[View] + queries: List[Query] + allow_execute_sql: bool + table_columns: Dict[str, List[str]] + query_ms: float diff --git a/docs/conf.py b/docs/conf.py index c25d8a95..5423fa2a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,12 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc", "sphinx_copybutton"] +extensions = [ + "sphinx.ext.extlinks", + "sphinx.ext.autodoc", + "sphinx_copybutton", + "jsoncontext", +] extlinks = { "issue": ("https://github.com/simonw/datasette/issues/%s", "#%s"), diff --git a/docs/index.rst b/docs/index.rst index 5a9cc7ed..254ed3da 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -57,6 +57,7 @@ Contents settings introspection custom_templates + template_context plugins writing_plugins plugin_hooks diff --git a/docs/jsoncontext.py b/docs/jsoncontext.py new file mode 100644 index 00000000..d759f89f --- /dev/null +++ b/docs/jsoncontext.py @@ -0,0 +1,28 @@ +from docutils import nodes +from sphinx.util.docutils import SphinxDirective +from importlib import import_module +import json + + +class JSONContextDirective(SphinxDirective): + required_arguments = 1 + + def run(self): + module_path, class_name = self.arguments[0].rsplit(".", 1) + try: + module = import_module(module_path) + dataclass = getattr(module, class_name) + except ImportError: + warning = f"Unable to import {self.arguments[0]}" + return [nodes.error(None, nodes.paragraph(text=warning))] + + doc = json.dumps( + dataclass.__annotations__, indent=4, sort_keys=True, default=repr + ) + doc_node = nodes.literal_block(text=doc) + + return [doc_node] + + +def setup(app): + app.add_directive("jsoncontext", JSONContextDirective) diff --git a/docs/template_context.rst b/docs/template_context.rst new file mode 100644 index 00000000..f1de4f9e --- /dev/null +++ b/docs/template_context.rst @@ -0,0 +1,29 @@ +.. _template_context: + +Template context +================ + +This page describes the variables made available to templates used by Datasette to render different pages of the application. + + +.. [[[cog + from datasette.context import rst_docs_for_dataclass, Table + cog.out(rst_docs_for_dataclass(Table)) +.. ]]] +Table +----- + +A table is a useful thing + +Fields +~~~~~~ + +:name - ``str``: The name of the table +:columns - ``List[str]``: List of column names in the table +:primary_keys - ``List[str]``: List of column names that are primary keys +:count - ``int``: Number of rows in the table +:hidden - ``bool``: Should this table default to being hidden in the main database UI? +:fts_table - ``Optional[str]``: If this table has FTS support, the accompanying FTS table name +:foreign_keys - ``ForeignKey``: List of foreign keys for this table +:private - ``bool``: Private tables are not visible to signed-out anonymous users +.. [[[end]]] From ede62036180993dbd9d4e5d280fc21c183cda1c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 07:31:54 -0700 Subject: [PATCH 021/767] Bump blacken-docs from 1.13.0 to 1.14.0 (#2083) Bumps [blacken-docs](https://github.com/asottile/blacken-docs) from 1.13.0 to 1.14.0. - [Changelog](https://github.com/adamchainz/blacken-docs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/asottile/blacken-docs/compare/1.13.0...1.14.0) --- updated-dependencies: - dependency-name: blacken-docs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d6824255..3874d892 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ setup( "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", "black==23.3.0", - "blacken-docs==1.13.0", + "blacken-docs==1.14.0", "pytest-timeout>=1.4.2", "trustme>=0.7", "cogapp>=3.3.0", From d45a7213eddd48ea75d1c021377e5a237b095833 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jun 2023 07:43:01 -0700 Subject: [PATCH 022/767] codespell>=2.5.5, also spellcheck README - refs #2089 --- .github/workflows/spellcheck.yml | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index a2621ecc..6bf72f9d 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -26,5 +26,6 @@ jobs: pip install -e '.[docs]' - name: Check spelling run: | + codespell README.md --ignore-words docs/codespell-ignore-words.txt codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt diff --git a/setup.py b/setup.py index 3874d892..aadae4d3 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ setup( "Sphinx==6.1.3", "furo==2023.3.27", "sphinx-autobuild", - "codespell", + "codespell>=2.2.5", "blacken-docs", "sphinx-copybutton", ], From 84b32b447ae93d4d9cc589f568af16fa519b8863 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jun 2023 07:44:10 -0700 Subject: [PATCH 023/767] Justfile I use for local development Now with codespell, refs #2089 --- Justfile | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 Justfile diff --git a/Justfile b/Justfile new file mode 100644 index 00000000..e595a266 --- /dev/null +++ b/Justfile @@ -0,0 +1,41 @@ +export DATASETTE_SECRET := "not_a_secret" + +# Run tests and linters +@default: test lint + +# Setup project +@init: + pipenv run pip install -e '.[test,docs]' + +# Run pytest with supplied options +@test *options: + pipenv run pytest {{options}} + +@codespell: + pipenv run codespell README.md --ignore-words docs/codespell-ignore-words.txt + pipenv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + pipenv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt + +# Run linters: black, flake8, mypy, cog +@lint: + pipenv run black . --check + pipenv run flake8 + pipenv run cog --check README.md docs/*.rst + +# Rebuild docs with cog +@cog: + pipenv run cog -r README.md docs/*.rst + +# Serve live docs on localhost:8000 +@docs: cog + pipenv run blacken-docs -l 60 docs/*.rst + cd docs && pipenv run make livehtml + +# Apply Black +@black: + pipenv run black . + +@serve: + pipenv run sqlite-utils create-database data.db + pipenv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore + pipenv run python -m datasette data.db --root --reload From 99ba05118891db9dc30f1dca22ad6709775560de Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jun 2023 07:46:22 -0700 Subject: [PATCH 024/767] Fixed spelling error, refs #2089 Also ensure codespell runs as part of just lint --- Justfile | 2 +- docs/metadata.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Justfile b/Justfile index e595a266..d349ec51 100644 --- a/Justfile +++ b/Justfile @@ -17,7 +17,7 @@ export DATASETTE_SECRET := "not_a_secret" pipenv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt # Run linters: black, flake8, mypy, cog -@lint: +@lint: codespell pipenv run black . --check pipenv run flake8 pipenv run cog --check README.md docs/*.rst diff --git a/docs/metadata.rst b/docs/metadata.rst index 35b8aede..5932cc3a 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -189,7 +189,7 @@ Or use ``"sort_desc"`` to sort in descending order: Setting a custom page size -------------------------- -Datasette defaults to displaing 100 rows per page, for both tables and views. You can change this default page size on a per-table or per-view basis using the ``"size"`` key in ``metadata.json``: +Datasette defaults to displaying 100 rows per page, for both tables and views. You can change this default page size on a per-table or per-view basis using the ``"size"`` key in ``metadata.json``: .. code-block:: json From c39d600aef5c085e891917db04fc97511d2e2259 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jun 2023 08:05:24 -0700 Subject: [PATCH 025/767] Fix all E741 Ambiguous variable name warnings, refs #2090 --- tests/test_plugins.py | 8 +++++--- tests/test_table_html.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 71b710f9..6971bbf7 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -115,7 +115,9 @@ async def test_hook_extra_css_urls(ds_client, path, expected_decoded_object): assert response.status_code == 200 links = Soup(response.text, "html.parser").findAll("link") special_href = [ - l for l in links if l.attrs["href"].endswith("/extra-css-urls-demo.css") + link + for link in links + if link.attrs["href"].endswith("/extra-css-urls-demo.css") ][0]["href"] # This link has a base64-encoded JSON blob in it encoded = special_href.split("/")[3] @@ -543,7 +545,7 @@ async def test_hook_register_output_renderer_can_render(ds_client): .find("p", {"class": "export-links"}) .findAll("a") ) - actual = [l["href"] for l in links] + actual = [link["href"] for link in links] # Should not be present because we sent ?_no_can_render=1 assert "/fixtures/facetable.testall?_labels=on" not in actual # Check that it was passed the values we expected @@ -940,7 +942,7 @@ async def test_hook_table_actions(ds_client, table_or_view): response_2 = await ds_client.get(f"/fixtures/{table_or_view}?_bot=1&_hello=BOB") assert sorted( - get_table_actions_links(response_2.text), key=lambda l: l["label"] + get_table_actions_links(response_2.text), key=lambda link: link["label"] ) == [ {"label": "Database: fixtures", "href": "/"}, {"label": "From async BOB", "href": "/"}, diff --git a/tests/test_table_html.py b/tests/test_table_html.py index e1886dab..c4c7878c 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -481,7 +481,7 @@ async def test_table_csv_json_export_interface(ds_client): .find("p", {"class": "export-links"}) .findAll("a") ) - actual = [l["href"] for l in links] + actual = [link["href"] for link in links] expected = [ "/fixtures/simple_primary_key.json?id__gt=2", "/fixtures/simple_primary_key.testall?id__gt=2", @@ -521,7 +521,7 @@ async def test_csv_json_export_links_include_labels_if_foreign_keys(ds_client): .find("p", {"class": "export-links"}) .findAll("a") ) - actual = [l["href"] for l in links] + actual = [link["href"] for link in links] expected = [ "/fixtures/facetable.json?_labels=on", "/fixtures/facetable.testall?_labels=on", From 8cd60fd1d899952f1153460469b3175465f33f80 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Jun 2023 08:24:09 -0700 Subject: [PATCH 026/767] Homepage test now just asserts isinstance(x, int) - closes #2092 --- tests/test_api.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 247fdd5c..40a3e2b8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -32,14 +32,12 @@ async def test_homepage(ds_client): assert data.keys() == {"fixtures": 0}.keys() d = data["fixtures"] assert d["name"] == "fixtures" - assert d["tables_count"] == 24 - assert len(d["tables_and_views_truncated"]) == 5 + assert isinstance(d["tables_count"], int) + assert isinstance(len(d["tables_and_views_truncated"]), int) assert d["tables_and_views_more"] is True - # 4 hidden FTS tables + no_primary_key (hidden in metadata) - assert d["hidden_tables_count"] == 6 - # 201 in no_primary_key, plus 6 in other hidden tables: - assert d["hidden_table_rows_sum"] == 207, data - assert d["views_count"] == 4 + assert isinstance(d["hidden_tables_count"], int) + assert isinstance(d["hidden_table_rows_sum"], int) + assert isinstance(d["views_count"], int) @pytest.mark.asyncio From d7b21a862335d3765247d84e5afe778f83eb69ee Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jul 2023 09:37:01 -0700 Subject: [PATCH 027/767] metadata.yaml now treated as default in docs Added sphinx-inline-tabs to provide JSON and YAML tabs to show examples. Refs #1153 --- docs/conf.py | 7 +- docs/index.rst | 14 +- docs/metadata.rst | 483 ++++++++++++++++++++++++++++++++++++++++------ setup.py | 1 + 4 files changed, 445 insertions(+), 60 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c25d8a95..67f7fd11 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,12 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc", "sphinx_copybutton"] +extensions = [ + "sphinx.ext.extlinks", + "sphinx.ext.autodoc", + "sphinx_copybutton", + "sphinx_inline_tabs", +] extlinks = { "issue": ("https://github.com/simonw/datasette/issues/%s", "#%s"), diff --git a/docs/index.rst b/docs/index.rst index 5a9cc7ed..c10ed5a1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,7 +29,19 @@ Datasette is aimed at data journalists, museum curators, archivists, local gover Interested in learning Datasette? Start with `the official tutorials `__. -Support questions, feedback? Join our `GitHub Discussions forum `__. +Support questions, feedback? Join the `Datasette Discord `__. + +.. tab:: macOS/Linux/other Unix + + .. code-block:: shell + + python -m pip install webcolors + +.. tab:: Windows + + .. code-block:: shell + + py -m pip install webcolors Contents -------- diff --git a/docs/metadata.rst b/docs/metadata.rst index 5932cc3a..6f17056f 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -4,27 +4,56 @@ Metadata ======== Data loves metadata. Any time you run Datasette you can optionally include a -JSON file with metadata about your databases and tables. Datasette will then +YAML or JSON file with metadata about your databases and tables. Datasette will then display that information in the web UI. Run Datasette like this:: - datasette database1.db database2.db --metadata metadata.json + datasette database1.db database2.db --metadata metadata.yaml -Your ``metadata.json`` file can look something like this: +Your ``metadata.yaml`` file can look something like this: -.. code-block:: json - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "title": "Custom title for your index page", "description": "Some description text can go here", "license": "ODbL", "license_url": "https://opendatacommons.org/licenses/odbl/", "source": "Original Data Source", "source_url": "http://example.com/" - } + }) +.. ]]] -You can optionally use YAML instead of JSON, see :ref:`metadata_yaml`. +.. tab:: YAML + + .. code-block:: yaml + + description: Some description text can go here + license: ODbL + license_url: https://opendatacommons.org/licenses/odbl/ + source: Original Data Source + source_url: http://example.com/ + title: Custom title for your index page + + +.. tab:: JSON + + .. code-block:: json + + { + "title": "Custom title for your index page", + "description": "Some description text can go here", + "license": "ODbL", + "license_url": "https://opendatacommons.org/licenses/odbl/", + "source": "Original Data Source", + "source_url": "http://example.com/" + } +.. [[[end]]] + + +Choosing YAML over JSON adds support for multi-line strings and comments, see :ref:`metadata_yaml`. The above metadata will be displayed on the index page of your Datasette-powered site. The source and license information will also be included in the footer of @@ -37,15 +66,15 @@ instead. Per-database and per-table metadata ----------------------------------- -Metadata at the top level of the JSON will be shown on the index page and in the +Metadata at the top level of the file will be shown on the index page and in the footer on every page of the site. The license and source is expected to apply to all of your data. You can also provide metadata at the per-database or per-table level, like this: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "database1": { "source": "Alternative source", @@ -59,7 +88,45 @@ You can also provide metadata at the per-database or per-table level, like this: } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + database1: + source: Alternative source + source_url: http://example.com/ + tables: + example_table: + description_html: Custom table description + license: CC BY 3.0 US + license_url: https://creativecommons.org/licenses/by/3.0/us/ + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "database1": { + "source": "Alternative source", + "source_url": "http://example.com/", + "tables": { + "example_table": { + "description_html": "Custom table description", + "license": "CC BY 3.0 US", + "license_url": "https://creativecommons.org/licenses/by/3.0/us/" + } + } + } + } + } +.. [[[end]]] + Each of the top-level metadata fields can be used at the database and table level. @@ -85,9 +152,9 @@ Column descriptions You can include descriptions for your columns by adding a ``"columns": {"name-of-column": "description-of-column"}`` block to your table metadata: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "database1": { "tables": { @@ -100,7 +167,41 @@ You can include descriptions for your columns by adding a ``"columns": {"name-of } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + database1: + tables: + example_table: + columns: + column1: Description of column 1 + column2: Description of column 2 + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "database1": { + "tables": { + "example_table": { + "columns": { + "column1": "Description of column 1", + "column2": "Description of column 2" + } + } + } + } + } + } +.. [[[end]]] These will be displayed at the top of the table page, and will also show in the cog menu for each column. @@ -114,9 +215,9 @@ values from that column. SI prefixes will be used where appropriate. Column units are configured in the metadata like so: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "database1": { "tables": { @@ -129,19 +230,74 @@ Column units are configured in the metadata like so: } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + database1: + tables: + example_table: + units: + column1: metres + column2: Hz + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "database1": { + "tables": { + "example_table": { + "units": { + "column1": "metres", + "column2": "Hz" + } + } + } + } + } + } +.. [[[end]]] + Units are interpreted using Pint_, and you can see the full list of available units in Pint's `unit registry`_. You can also add `custom units`_ to the metadata, which will be registered with Pint: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "custom_units": [ "decibel = [] = dB" ] - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + custom_units: + - decibel = [] = dB + + +.. tab:: JSON + + .. code-block:: json + + { + "custom_units": [ + "decibel = [] = dB" + ] + } +.. [[[end]]] .. _Pint: https://pint.readthedocs.io/ .. _unit registry: https://github.com/hgrecco/pint/blob/master/pint/default_en.txt @@ -154,9 +310,9 @@ Setting a default sort order By default Datasette tables are sorted by primary key. You can over-ride this default for a specific table using the ``"sort"`` or ``"sort_desc"`` metadata properties: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "mydatabase": { "tables": { @@ -166,13 +322,42 @@ By default Datasette tables are sorted by primary key. You can over-ride this de } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + mydatabase: + tables: + example_table: + sort: created + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "mydatabase": { + "tables": { + "example_table": { + "sort": "created" + } + } + } + } + } +.. [[[end]]] Or use ``"sort_desc"`` to sort in descending order: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "mydatabase": { "tables": { @@ -182,7 +367,36 @@ Or use ``"sort_desc"`` to sort in descending order: } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + mydatabase: + tables: + example_table: + sort_desc: created + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "mydatabase": { + "tables": { + "example_table": { + "sort_desc": "created" + } + } + } + } + } +.. [[[end]]] .. _metadata_page_size: @@ -191,9 +405,9 @@ Setting a custom page size Datasette defaults to displaying 100 rows per page, for both tables and views. You can change this default page size on a per-table or per-view basis using the ``"size"`` key in ``metadata.json``: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "mydatabase": { "tables": { @@ -203,7 +417,36 @@ Datasette defaults to displaying 100 rows per page, for both tables and views. Y } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + mydatabase: + tables: + example_table: + size: 10 + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "mydatabase": { + "tables": { + "example_table": { + "size": 10 + } + } + } + } + } +.. [[[end]]] This size can still be over-ridden by passing e.g. ``?_size=50`` in the query string. @@ -216,9 +459,9 @@ Datasette allows any column to be used for sorting by default. If you need to control which columns are available for sorting you can do so using the optional ``sortable_columns`` key: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "database1": { "tables": { @@ -231,7 +474,41 @@ control which columns are available for sorting you can do so using the optional } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + database1: + tables: + example_table: + sortable_columns: + - height + - weight + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "database1": { + "tables": { + "example_table": { + "sortable_columns": [ + "height", + "weight" + ] + } + } + } + } + } +.. [[[end]]] This will restrict sorting of ``example_table`` to just the ``height`` and ``weight`` columns. @@ -240,9 +517,9 @@ You can also disable sorting entirely by setting ``"sortable_columns": []`` You can use ``sortable_columns`` to enable specific sort orders for a view called ``name_of_view`` in the database ``my_database`` like so: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "my_database": { "tables": { @@ -255,7 +532,41 @@ You can use ``sortable_columns`` to enable specific sort orders for a view calle } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + my_database: + tables: + name_of_view: + sortable_columns: + - clicks + - impressions + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "my_database": { + "tables": { + "name_of_view": { + "sortable_columns": [ + "clicks", + "impressions" + ] + } + } + } + } + } +.. [[[end]]] .. _label_columns: @@ -270,9 +581,9 @@ column should be used as the link label. If your table has more than two columns you can specify which column should be used for the link label with the ``label_column`` property: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "database1": { "tables": { @@ -282,7 +593,36 @@ used for the link label with the ``label_column`` property: } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + database1: + tables: + example_table: + label_column: title + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "database1": { + "tables": { + "example_table": { + "label_column": "title" + } + } + } + } + } +.. [[[end]]] .. _metadata_hiding_tables: @@ -292,26 +632,56 @@ Hiding tables You can hide tables from the database listing view (in the same way that FTS and SpatiaLite tables are automatically hidden) using ``"hidden": true``: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "database1": { "tables": { "example_table": { - "hidden": true + "hidden": True } } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + database1: + tables: + example_table: + hidden: true + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "database1": { + "tables": { + "example_table": { + "hidden": true + } + } + } + } + } +.. [[[end]]] .. _metadata_yaml: Using YAML for metadata ----------------------- -Datasette accepts YAML as an alternative to JSON for your metadata configuration file. YAML is particularly useful for including multiline HTML and SQL strings. +Datasette accepts YAML as an alternative to JSON for your metadata configuration file. +YAML is particularly useful for including multiline HTML and SQL strings, plus inline comments. Here's an example of a ``metadata.yml`` file, re-using an example from :ref:`canned_queries`. @@ -331,6 +701,7 @@ Here's an example of a ``metadata.yml`` file, re-using an example from :ref:`can no_primary_key: hidden: true queries: + # This query provides LIKE-based search neighborhood_search: sql: |- select neighborhood, facet_cities.name, state @@ -339,7 +710,3 @@ Here's an example of a ``metadata.yml`` file, re-using an example from :ref:`can title: Search neighborhoods description_html: |-

This demonstrates basic LIKE search - -The ``metadata.yml`` file is passed to Datasette using the same ``--metadata`` option:: - - datasette fixtures.db --metadata metadata.yml diff --git a/setup.py b/setup.py index aadae4d3..be9c7fde 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ setup( "codespell>=2.2.5", "blacken-docs", "sphinx-copybutton", + "sphinx-inline-tabs", ], "test": [ "pytest>=5.2.2", From 3b336d8071fb5707bd006de1d614f701d20246a3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jul 2023 09:37:47 -0700 Subject: [PATCH 028/767] Utility function for cog for generating YAML/JSON tabs, refs #1153 --- docs/metadata_doc.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/metadata_doc.py diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py new file mode 100644 index 00000000..e6c84e3b --- /dev/null +++ b/docs/metadata_doc.py @@ -0,0 +1,13 @@ +import json +import textwrap +import yaml + + +def metadata_example(cog, example): + cog.out("\n.. tab:: YAML\n\n") + cog.out(" .. code-block:: yaml\n\n") + cog.out(textwrap.indent(yaml.dump(example), " ")) + cog.out("\n\n.. tab:: JSON\n\n") + cog.out(" .. code-block:: json\n\n") + cog.out(textwrap.indent(json.dumps(example, indent=2), " ")) + cog.out("\n") From 38fcc96e67f8a77cd09d13c96f2abf7cd822ebb4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jul 2023 10:09:26 -0700 Subject: [PATCH 029/767] Removed duplicate imports, refs #1153 --- docs/metadata.rst | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docs/metadata.rst b/docs/metadata.rst index 6f17056f..7aff7c15 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -73,7 +73,6 @@ all of your data. You can also provide metadata at the per-database or per-table level, like this: .. [[[cog - from metadata_doc import metadata_example metadata_example(cog, { "databases": { "database1": { @@ -153,7 +152,6 @@ Column descriptions You can include descriptions for your columns by adding a ``"columns": {"name-of-column": "description-of-column"}`` block to your table metadata: .. [[[cog - from metadata_doc import metadata_example metadata_example(cog, { "databases": { "database1": { @@ -216,7 +214,6 @@ values from that column. SI prefixes will be used where appropriate. Column units are configured in the metadata like so: .. [[[cog - from metadata_doc import metadata_example metadata_example(cog, { "databases": { "database1": { @@ -272,7 +269,6 @@ Pint's `unit registry`_. You can also add `custom units`_ to the metadata, which registered with Pint: .. [[[cog - from metadata_doc import metadata_example metadata_example(cog, { "custom_units": [ "decibel = [] = dB" @@ -311,7 +307,6 @@ Setting a default sort order By default Datasette tables are sorted by primary key. You can over-ride this default for a specific table using the ``"sort"`` or ``"sort_desc"`` metadata properties: .. [[[cog - from metadata_doc import metadata_example metadata_example(cog, { "databases": { "mydatabase": { @@ -356,7 +351,6 @@ By default Datasette tables are sorted by primary key. You can over-ride this de Or use ``"sort_desc"`` to sort in descending order: .. [[[cog - from metadata_doc import metadata_example metadata_example(cog, { "databases": { "mydatabase": { @@ -406,7 +400,6 @@ Setting a custom page size Datasette defaults to displaying 100 rows per page, for both tables and views. You can change this default page size on a per-table or per-view basis using the ``"size"`` key in ``metadata.json``: .. [[[cog - from metadata_doc import metadata_example metadata_example(cog, { "databases": { "mydatabase": { @@ -460,7 +453,6 @@ control which columns are available for sorting you can do so using the optional ``sortable_columns`` key: .. [[[cog - from metadata_doc import metadata_example metadata_example(cog, { "databases": { "database1": { @@ -518,7 +510,6 @@ You can also disable sorting entirely by setting ``"sortable_columns": []`` You can use ``sortable_columns`` to enable specific sort orders for a view called ``name_of_view`` in the database ``my_database`` like so: .. [[[cog - from metadata_doc import metadata_example metadata_example(cog, { "databases": { "my_database": { @@ -582,7 +573,6 @@ If your table has more than two columns you can specify which column should be used for the link label with the ``label_column`` property: .. [[[cog - from metadata_doc import metadata_example metadata_example(cog, { "databases": { "database1": { @@ -633,7 +623,6 @@ You can hide tables from the database listing view (in the same way that FTS and SpatiaLite tables are automatically hidden) using ``"hidden": true``: .. [[[cog - from metadata_doc import metadata_example metadata_example(cog, { "databases": { "database1": { From 0183e1a72d4d93b1d9a9363f4d47fcc0b5d5849c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jul 2023 10:26:50 -0700 Subject: [PATCH 030/767] Preserve JSON key order in YAML, refs #1153 --- docs/metadata.rst | 2 +- docs/metadata_doc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/metadata.rst b/docs/metadata.rst index 7aff7c15..76b785fb 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -30,12 +30,12 @@ Your ``metadata.yaml`` file can look something like this: .. code-block:: yaml + title: Custom title for your index page description: Some description text can go here license: ODbL license_url: https://opendatacommons.org/licenses/odbl/ source: Original Data Source source_url: http://example.com/ - title: Custom title for your index page .. tab:: JSON diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py index e6c84e3b..19ddef89 100644 --- a/docs/metadata_doc.py +++ b/docs/metadata_doc.py @@ -6,7 +6,7 @@ import yaml def metadata_example(cog, example): cog.out("\n.. tab:: YAML\n\n") cog.out(" .. code-block:: yaml\n\n") - cog.out(textwrap.indent(yaml.dump(example), " ")) + cog.out(textwrap.indent(yaml.safe_dump(example, sort_keys=False), " ")) cog.out("\n\n.. tab:: JSON\n\n") cog.out(" .. code-block:: json\n\n") cog.out(textwrap.indent(json.dumps(example, indent=2), " ")) From c076fb65e07a7957b8a45804dc8d8cb92020f0ec Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jul 2023 11:00:08 -0700 Subject: [PATCH 031/767] Applied sphinx-inline-tabs to remaining examples, refs #1153 --- docs/authentication.rst | 370 +++++++++++++++++++++++++++++++++----- docs/custom_templates.rst | 142 +++++++++++++-- docs/facets.rst | 124 +++++++++++-- docs/full_text_search.rst | 41 ++++- docs/metadata_doc.py | 20 ++- docs/plugins.rst | 153 ++++++++++++++-- docs/sql_queries.rst | 320 ++++++++++++++++++++++++++++----- setup.py | 1 + 8 files changed, 1019 insertions(+), 152 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 3878b2c9..8864086f 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -212,23 +212,63 @@ Access to an instance Here's how to restrict access to your entire Datasette instance to just the ``"id": "root"`` user: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "title": "My private Datasette instance", "allow": { "id": "root" } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + title: My private Datasette instance + allow: + id: root + + +.. tab:: JSON + + .. code-block:: json + + { + "title": "My private Datasette instance", + "allow": { + "id": "root" + } + } +.. [[[end]]] To deny access to all users, you can use ``"allow": false``: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "title": "My entirely inaccessible instance", - "allow": false - } + "allow": False + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + title: My entirely inaccessible instance + allow: false + + +.. tab:: JSON + + .. code-block:: json + + { + "title": "My entirely inaccessible instance", + "allow": false + } +.. [[[end]]] One reason to do this is if you are using a Datasette plugin - such as `datasette-permissions-sql `__ - to control permissions instead. @@ -239,9 +279,8 @@ Access to specific databases To limit access to a specific ``private.db`` database to just authenticated users, use the ``"allow"`` block like this: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "databases": { "private": { "allow": { @@ -249,7 +288,33 @@ To limit access to a specific ``private.db`` database to just authenticated user } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + private: + allow: + id: '*' + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "private": { + "allow": { + "id": "*" + } + } + } + } +.. [[[end]]] .. _authentication_permissions_table: @@ -258,9 +323,8 @@ Access to specific tables and views To limit access to the ``users`` table in your ``bakery.db`` database: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "databases": { "bakery": { "tables": { @@ -272,7 +336,39 @@ To limit access to the ``users`` table in your ``bakery.db`` database: } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + bakery: + tables: + users: + allow: + id: '*' + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "bakery": { + "tables": { + "users": { + "allow": { + "id": "*" + } + } + } + } + } + } +.. [[[end]]] This works for SQL views as well - you can list their names in the ``"tables"`` block above in the same way as regular tables. @@ -290,15 +386,14 @@ Access to specific canned queries To limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user`: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "databases": { "dogs": { "queries": { "add_name": { "sql": "INSERT INTO names (name) VALUES (:name)", - "write": true, + "write": True, "allow": { "id": ["root"] } @@ -306,7 +401,46 @@ To limit access to the ``add_name`` canned query in your ``dogs.db`` database to } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + dogs: + queries: + add_name: + sql: INSERT INTO names (name) VALUES (:name) + write: true + allow: + id: + - root + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "dogs": { + "queries": { + "add_name": { + "sql": "INSERT INTO names (name) VALUES (:name)", + "write": true, + "allow": { + "id": [ + "root" + ] + } + } + } + } + } + } +.. [[[end]]] .. _authentication_permissions_execute_sql: @@ -323,27 +457,61 @@ You can alternatively use an ``"allow_sql"`` block to control who is allowed to To prevent any user from executing arbitrary SQL queries, use this: -.. code-block:: json +.. [[[cog + metadata_example(cog, { + "allow_sql": False + }) +.. ]]] - { - "allow_sql": false - } +.. tab:: YAML + + .. code-block:: yaml + + allow_sql: false + + +.. tab:: JSON + + .. code-block:: json + + { + "allow_sql": false + } +.. [[[end]]] To enable just the :ref:`root user` to execute SQL for all databases in your instance, use the following: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "allow_sql": { "id": "root" } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + allow_sql: + id: root + + +.. tab:: JSON + + .. code-block:: json + + { + "allow_sql": { + "id": "root" + } + } +.. [[[end]]] To limit this ability for just one specific database, use this: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "databases": { "mydatabase": { "allow_sql": { @@ -351,7 +519,33 @@ To limit this ability for just one specific database, use this: } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + mydatabase: + allow_sql: + id: root + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "mydatabase": { + "allow_sql": { + "id": "root" + } + } + } + } +.. [[[end]]] .. _authentication_permissions_other: @@ -362,21 +556,42 @@ For all other permissions, you can use one or more ``"permissions"`` blocks in y To grant access to the :ref:`permissions debug tool ` to all signed in users you can grant ``permissions-debug`` to any actor with an ``id`` matching the wildcard ``*`` by adding this a the root of your metadata: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "permissions": { "debug-menu": { "id": "*" } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + permissions: + debug-menu: + id: '*' + + +.. tab:: JSON + + .. code-block:: json + + { + "permissions": { + "debug-menu": { + "id": "*" + } + } + } +.. [[[end]]] To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` database: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "databases": { "docs": { "permissions": { @@ -386,13 +601,41 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + docs: + permissions: + create-table: + id: editor + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "docs": { + "permissions": { + "create-table": { + "id": "editor" + } + } + } + } + } +.. [[[end]]] And for ``insert-row`` against the ``reports`` table in that ``docs`` database: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "databases": { "docs": { "tables": { @@ -406,7 +649,42 @@ And for ``insert-row`` against the ``reports`` table in that ``docs`` database: } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + docs: + tables: + reports: + permissions: + insert-row: + id: editor + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "docs": { + "tables": { + "reports": { + "permissions": { + "insert-row": { + "id": "editor" + } + } + } + } + } + } + } +.. [[[end]]] The :ref:`permissions debug tool ` can be useful for helping test permissions that you have configured in this way. diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 97dea2af..f9d0b5f5 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -12,20 +12,45 @@ Custom CSS and JavaScript When you launch Datasette, you can specify a custom metadata file like this:: - datasette mydb.db --metadata metadata.json + datasette mydb.db --metadata metadata.yaml -Your ``metadata.json`` file can include links that look like this: +Your ``metadata.yaml`` file can include links that look like this: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "extra_css_urls": [ "https://simonwillison.net/static/css/all.bf8cd891642c.css" ], "extra_js_urls": [ "https://code.jquery.com/jquery-3.2.1.slim.min.js" ] - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + extra_css_urls: + - https://simonwillison.net/static/css/all.bf8cd891642c.css + extra_js_urls: + - https://code.jquery.com/jquery-3.2.1.slim.min.js + + +.. tab:: JSON + + .. code-block:: json + + { + "extra_css_urls": [ + "https://simonwillison.net/static/css/all.bf8cd891642c.css" + ], + "extra_js_urls": [ + "https://code.jquery.com/jquery-3.2.1.slim.min.js" + ] + } +.. [[[end]]] The extra CSS and JavaScript files will be linked in the ```` of every page: @@ -36,9 +61,8 @@ The extra CSS and JavaScript files will be linked in the ```` of every pag You can also specify a SRI (subresource integrity hash) for these assets: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "extra_css_urls": [ { "url": "https://simonwillison.net/static/css/all.bf8cd891642c.css", @@ -51,7 +75,40 @@ You can also specify a SRI (subresource integrity hash) for these assets: "sri": "sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g=" } ] - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + extra_css_urls: + - url: https://simonwillison.net/static/css/all.bf8cd891642c.css + sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI + extra_js_urls: + - url: https://code.jquery.com/jquery-3.2.1.slim.min.js + sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + + +.. tab:: JSON + + .. code-block:: json + + { + "extra_css_urls": [ + { + "url": "https://simonwillison.net/static/css/all.bf8cd891642c.css", + "sri": "sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI" + } + ], + "extra_js_urls": [ + { + "url": "https://code.jquery.com/jquery-3.2.1.slim.min.js", + "sri": "sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g=" + } + ] + } +.. [[[end]]] This will produce: @@ -69,16 +126,39 @@ matches the content served. You can generate hashes using `www.srihash.org `__. This configuration: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "extra_js_urls": [ { "url": "https://example.datasette.io/module.js", - "module": true + "module": True } ] - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + extra_js_urls: + - url: https://example.datasette.io/module.js + module: true + + +.. tab:: JSON + + .. code-block:: json + + { + "extra_js_urls": [ + { + "url": "https://example.datasette.io/module.js", + "module": true + } + ] + } +.. [[[end]]] Will produce this HTML: @@ -188,16 +268,40 @@ The following URLs will now serve the content from those CSS and JS files:: You can reference those files from ``metadata.json`` like so: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "extra_css_urls": [ "/assets/styles.css" ], "extra_js_urls": [ "/assets/app.js" ] - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + extra_css_urls: + - /assets/styles.css + extra_js_urls: + - /assets/app.js + + +.. tab:: JSON + + .. code-block:: json + + { + "extra_css_urls": [ + "/assets/styles.css" + ], + "extra_js_urls": [ + "/assets/app.js" + ] + } +.. [[[end]]] Publishing static assets ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/facets.rst b/docs/facets.rst index 5df4dbb4..dba232bf 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -98,16 +98,16 @@ You can increase this on an individual page by adding ``?_facet_size=100`` to th .. _facets_metadata: -Facets in metadata.json ------------------------ +Facets in metadata +------------------ You can turn facets on by default for specific tables by adding them to a ``"facets"`` key in a Datasette :ref:`metadata` file. Here's an example that turns on faceting by default for the ``qLegalStatus`` column in the ``Street_Tree_List`` table in the ``sf-trees`` database: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "sf-trees": { "tables": { @@ -117,26 +117,82 @@ Here's an example that turns on faceting by default for the ``qLegalStatus`` col } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + sf-trees: + tables: + Street_Tree_List: + facets: + - qLegalStatus + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "sf-trees": { + "tables": { + "Street_Tree_List": { + "facets": [ + "qLegalStatus" + ] + } + } + } + } + } +.. [[[end]]] Facets defined in this way will always be shown in the interface and returned in the API, regardless of the ``_facet`` arguments passed to the view. You can specify :ref:`array ` or :ref:`date ` facets in metadata using JSON objects with a single key of ``array`` or ``date`` and a value specifying the column, like this: -.. code-block:: json +.. [[[cog + metadata_example(cog, { + "facets": [ + {"array": "tags"}, + {"date": "created"} + ] + }) +.. ]]] - { - "facets": [ - {"array": "tags"}, - {"date": "created"} - ] - } +.. tab:: YAML + + .. code-block:: yaml + + facets: + - array: tags + - date: created + + +.. tab:: JSON + + .. code-block:: json + + { + "facets": [ + { + "array": "tags" + }, + { + "date": "created" + } + ] + } +.. [[[end]]] You can change the default facet size (the number of results shown for each facet) for a table using ``facet_size``: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "databases": { "sf-trees": { "tables": { @@ -147,7 +203,41 @@ You can change the default facet size (the number of results shown for each face } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + sf-trees: + tables: + Street_Tree_List: + facets: + - qLegalStatus + facet_size: 10 + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "sf-trees": { + "tables": { + "Street_Tree_List": { + "facets": [ + "qLegalStatus" + ], + "facet_size": 10 + } + } + } + } + } +.. [[[end]]] Suggested facets ---------------- diff --git a/docs/full_text_search.rst b/docs/full_text_search.rst index 1162a7f9..c956865b 100644 --- a/docs/full_text_search.rst +++ b/docs/full_text_search.rst @@ -64,9 +64,9 @@ The ``"searchmode": "raw"`` property can be used to default the table to accepti Here is an example which enables full-text search (with SQLite advanced search operators) for a ``display_ads`` view which is defined against the ``ads`` table and hence needs to run FTS against the ``ads_fts`` table, using the ``id`` as the primary key: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "russian-ads": { "tables": { @@ -78,7 +78,40 @@ Here is an example which enables full-text search (with SQLite advanced search o } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + russian-ads: + tables: + display_ads: + fts_table: ads_fts + fts_pk: id + searchmode: raw + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "russian-ads": { + "tables": { + "display_ads": { + "fts_table": "ads_fts", + "fts_pk": "id", + "searchmode": "raw" + } + } + } + } + } +.. [[[end]]] .. _full_text_search_custom_sql: diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py index 19ddef89..537830ca 100644 --- a/docs/metadata_doc.py +++ b/docs/metadata_doc.py @@ -1,13 +1,25 @@ import json import textwrap -import yaml +from yaml import safe_dump +from ruamel.yaml import round_trip_load -def metadata_example(cog, example): +def metadata_example(cog, data=None, yaml=None): + assert data or yaml, "Must provide data= or yaml=" + assert not (data and yaml), "Cannot use data= and yaml=" + output_yaml = None + if yaml: + # dedent it first + yaml = textwrap.dedent(yaml).strip() + # round_trip_load to preserve key order: + data = round_trip_load(yaml) + output_yaml = yaml + else: + output_yaml = safe_dump(data, sort_keys=False) cog.out("\n.. tab:: YAML\n\n") cog.out(" .. code-block:: yaml\n\n") - cog.out(textwrap.indent(yaml.safe_dump(example, sort_keys=False), " ")) + cog.out(textwrap.indent(output_yaml, " ")) cog.out("\n\n.. tab:: JSON\n\n") cog.out(" .. code-block:: json\n\n") - cog.out(textwrap.indent(json.dumps(example, indent=2), " ")) + cog.out(textwrap.indent(json.dumps(data, indent=2), " ")) cog.out("\n") diff --git a/docs/plugins.rst b/docs/plugins.rst index 5e81e202..979f94dd 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -245,9 +245,9 @@ Plugins can have their own configuration, embedded in a :ref:`metadata` file. Co Here is an example of some plugin configuration for a specific table: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "sf-trees": { "tables": { @@ -262,7 +262,44 @@ Here is an example of some plugin configuration for a specific table: } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + sf-trees: + tables: + Street_Tree_List: + plugins: + datasette-cluster-map: + latitude_column: lat + longitude_column: lng + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "sf-trees": { + "tables": { + "Street_Tree_List": { + "plugins": { + "datasette-cluster-map": { + "latitude_column": "lat", + "longitude_column": "lng" + } + } + } + } + } + } + } +.. [[[end]]] This tells the ``datasette-cluster-map`` column which latitude and longitude columns should be used for a table called ``Street_Tree_List`` inside a database file called ``sf-trees.db``. @@ -271,13 +308,12 @@ This tells the ``datasette-cluster-map`` column which latitude and longitude col Secret configuration values ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Any values embedded in ``metadata.json`` will be visible to anyone who views the ``/-/metadata`` page of your Datasette instance. Some plugins may need configuration that should stay secret - API keys for example. There are two ways in which you can store secret configuration values. +Any values embedded in ``metadata.yaml`` will be visible to anyone who views the ``/-/metadata`` page of your Datasette instance. Some plugins may need configuration that should stay secret - API keys for example. There are two ways in which you can store secret configuration values. **As environment variables**. If your secret lives in an environment variable that is available to the Datasette process, you can indicate that the configuration value should be read from that environment variable like so: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "plugins": { "datasette-auth-github": { "client_secret": { @@ -285,13 +321,38 @@ Any values embedded in ``metadata.json`` will be visible to anyone who views the } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + plugins: + datasette-auth-github: + client_secret: + $env: GITHUB_CLIENT_SECRET + + +.. tab:: JSON + + .. code-block:: json + + { + "plugins": { + "datasette-auth-github": { + "client_secret": { + "$env": "GITHUB_CLIENT_SECRET" + } + } + } + } +.. [[[end]]] **As values in separate files**. Your secrets can also live in files on disk. To specify a secret should be read from a file, provide the full file path like this: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "plugins": { "datasette-auth-github": { "client_secret": { @@ -299,7 +360,33 @@ Any values embedded in ``metadata.json`` will be visible to anyone who views the } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + plugins: + datasette-auth-github: + client_secret: + $file: /secrets/client-secret + + +.. tab:: JSON + + .. code-block:: json + + { + "plugins": { + "datasette-auth-github": { + "client_secret": { + "$file": "/secrets/client-secret" + } + } + } + } +.. [[[end]]] If you are publishing your data using the :ref:`datasette publish ` family of commands, you can use the ``--plugin-secret`` option to set these secrets at publish time. For example, using Heroku you might run the following command:: @@ -309,11 +396,10 @@ If you are publishing your data using the :ref:`datasette publish ` --plugin-secret datasette-auth-github client_id your_client_id \ --plugin-secret datasette-auth-github client_secret your_client_secret -This will set the necessary environment variables and add the following to the deployed ``metadata.json``: +This will set the necessary environment variables and add the following to the deployed ``metadata.yaml``: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "plugins": { "datasette-auth-github": { "client_id": { @@ -324,4 +410,35 @@ This will set the necessary environment variables and add the following to the d } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + plugins: + datasette-auth-github: + client_id: + $env: DATASETTE_AUTH_GITHUB_CLIENT_ID + client_secret: + $env: DATASETTE_AUTH_GITHUB_CLIENT_SECRET + + +.. tab:: JSON + + .. code-block:: json + + { + "plugins": { + "datasette-auth-github": { + "client_id": { + "$env": "DATASETTE_AUTH_GITHUB_CLIENT_ID" + }, + "client_secret": { + "$env": "DATASETTE_AUTH_GITHUB_CLIENT_SECRET" + } + } + } + } +.. [[[end]]] diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 5f53a6d7..3c2cb228 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -64,11 +64,11 @@ The quickest way to create views is with the SQLite command-line interface:: Canned queries -------------- -As an alternative to adding views to your database, you can define canned queries inside your ``metadata.json`` file. Here's an example: +As an alternative to adding views to your database, you can define canned queries inside your ``metadata.yaml`` file. Here's an example: -.. code-block:: json - - { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { "databases": { "sf-trees": { "queries": { @@ -78,7 +78,36 @@ As an alternative to adding views to your database, you can define canned querie } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + sf-trees: + queries: + just_species: + sql: select qSpecies from Street_Tree_List + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "sf-trees": { + "queries": { + "just_species": { + "sql": "select qSpecies from Street_Tree_List" + } + } + } + } + } +.. [[[end]]] Then run Datasette like this:: @@ -111,38 +140,58 @@ Here's an example of a canned query with a named parameter: where neighborhood like '%' || :text || '%' order by neighborhood; -In the canned query metadata (here :ref:`metadata_yaml` as ``metadata.yaml``) it looks like this: +In the canned query metadata looks like this: -.. code-block:: yaml +.. [[[cog + metadata_example(cog, yaml=""" databases: fixtures: queries: neighborhood_search: + title: Search neighborhoods sql: |- select neighborhood, facet_cities.name, state from facetable join facet_cities on facetable.city_id = facet_cities.id where neighborhood like '%' || :text || '%' order by neighborhood - title: Search neighborhoods + """) +.. ]]] -Here's the equivalent using JSON (as ``metadata.json``): +.. tab:: YAML -.. code-block:: json + .. code-block:: yaml - { - "databases": { + databases: + fixtures: + queries: + neighborhood_search: + title: Search neighborhoods + sql: |- + select neighborhood, facet_cities.name, state + from facetable + join facet_cities on facetable.city_id = facet_cities.id + where neighborhood like '%' || :text || '%' + order by neighborhood + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { "fixtures": { - "queries": { - "neighborhood_search": { - "sql": "select neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities on facetable.city_id = facet_cities.id\nwhere neighborhood like '%' || :text || '%'\norder by neighborhood", - "title": "Search neighborhoods" - } + "queries": { + "neighborhood_search": { + "title": "Search neighborhoods", + "sql": "select neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities on facetable.city_id = facet_cities.id\nwhere neighborhood like '%' || :text || '%'\norder by neighborhood" } + } } + } } - } +.. [[[end]]] Note that we are using SQLite string concatenation here - the ``||`` operator - to add wildcard ``%`` characters to the string provided by the user. @@ -153,12 +202,13 @@ In this example the ``:text`` named parameter is automatically extracted from th You can alternatively provide an explicit list of named parameters using the ``"params"`` key, like this: -.. code-block:: yaml - +.. [[[cog + metadata_example(cog, yaml=""" databases: fixtures: queries: neighborhood_search: + title: Search neighborhoods params: - text sql: |- @@ -167,7 +217,47 @@ You can alternatively provide an explicit list of named parameters using the ``" join facet_cities on facetable.city_id = facet_cities.id where neighborhood like '%' || :text || '%' order by neighborhood - title: Search neighborhoods + """) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + fixtures: + queries: + neighborhood_search: + title: Search neighborhoods + params: + - text + sql: |- + select neighborhood, facet_cities.name, state + from facetable + join facet_cities on facetable.city_id = facet_cities.id + where neighborhood like '%' || :text || '%' + order by neighborhood + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "fixtures": { + "queries": { + "neighborhood_search": { + "title": "Search neighborhoods", + "params": [ + "text" + ], + "sql": "select neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities on facetable.city_id = facet_cities.id\nwhere neighborhood like '%' || :text || '%'\norder by neighborhood" + } + } + } + } + } +.. [[[end]]] .. _canned_queries_options: @@ -192,21 +282,54 @@ You can set a default fragment hash that will be included in the link to the can This example demonstrates both ``fragment`` and ``hide_sql``: -.. code-block:: json +.. [[[cog + metadata_example(cog, yaml=""" + databases: + fixtures: + queries: + neighborhood_search: + fragment: fragment-goes-here + hide_sql: true + sql: |- + select neighborhood, facet_cities.name, state + from facetable join facet_cities on facetable.city_id = facet_cities.id + where neighborhood like '%' || :text || '%' order by neighborhood; + """) +.. ]]] - { - "databases": { +.. tab:: YAML + + .. code-block:: yaml + + databases: + fixtures: + queries: + neighborhood_search: + fragment: fragment-goes-here + hide_sql: true + sql: |- + select neighborhood, facet_cities.name, state + from facetable join facet_cities on facetable.city_id = facet_cities.id + where neighborhood like '%' || :text || '%' order by neighborhood; + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { "fixtures": { - "queries": { - "neighborhood_search": { - "sql": "select neighborhood, facet_cities.name, state\nfrom facetable join facet_cities on facetable.city_id = facet_cities.id\nwhere neighborhood like '%' || :text || '%' order by neighborhood;", - "fragment": "fragment-goes-here", - "hide_sql": true - } + "queries": { + "neighborhood_search": { + "fragment": "fragment-goes-here", + "hide_sql": true, + "sql": "select neighborhood, facet_cities.name, state\nfrom facetable join facet_cities on facetable.city_id = facet_cities.id\nwhere neighborhood like '%' || :text || '%' order by neighborhood;" } + } } + } } - } +.. [[[end]]] `See here `__ for a demo of this in action. @@ -219,20 +342,50 @@ Canned queries by default are read-only. You can use the ``"write": true`` key t See :ref:`authentication_permissions_query` for details on how to add permission checks to canned queries, using the ``"allow"`` key. -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "databases": { "mydatabase": { "queries": { "add_name": { "sql": "INSERT INTO names (name) VALUES (:name)", - "write": true + "write": True } } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + mydatabase: + queries: + add_name: + sql: INSERT INTO names (name) VALUES (:name) + write: true + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "mydatabase": { + "queries": { + "add_name": { + "sql": "INSERT INTO names (name) VALUES (:name)", + "write": true + } + } + } + } + } +.. [[[end]]] This configuration will create a page at ``/mydatabase/add_name`` displaying a form with a ``name`` field. Submitting that form will execute the configured ``INSERT`` query. @@ -245,15 +398,14 @@ You can customize how Datasette represents success and errors using the followin For example: -.. code-block:: json - - { +.. [[[cog + metadata_example(cog, { "databases": { "mydatabase": { "queries": { "add_name": { "sql": "INSERT INTO names (name) VALUES (:name)", - "write": true, + "write": True, "on_success_message": "Name inserted", "on_success_redirect": "/mydatabase/names", "on_error_message": "Name insert failed", @@ -262,7 +414,46 @@ For example: } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + mydatabase: + queries: + add_name: + sql: INSERT INTO names (name) VALUES (:name) + write: true + on_success_message: Name inserted + on_success_redirect: /mydatabase/names + on_error_message: Name insert failed + on_error_redirect: /mydatabase + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "mydatabase": { + "queries": { + "add_name": { + "sql": "INSERT INTO names (name) VALUES (:name)", + "write": true, + "on_success_message": "Name inserted", + "on_success_redirect": "/mydatabase/names", + "on_error_message": "Name insert failed", + "on_error_redirect": "/mydatabase" + } + } + } + } + } +.. [[[end]]] You can use ``"params"`` to explicitly list the named parameters that should be displayed as form fields - otherwise they will be automatically detected. @@ -300,10 +491,10 @@ Available magic parameters are: ``_random_chars_*`` - e.g. ``_random_chars_128`` A random string of characters of the specified length. -Here's an example configuration (this time using ``metadata.yaml`` since that provides better support for multi-line SQL queries) that adds a message from the authenticated user, storing various pieces of additional metadata using magic parameters: - -.. code-block:: yaml +Here's an example configuration that adds a message from the authenticated user, storing various pieces of additional metadata using magic parameters: +.. [[[cog + metadata_example(cog, yaml=""" databases: mydatabase: queries: @@ -317,6 +508,47 @@ Here's an example configuration (this time using ``metadata.yaml`` since that pr :_actor_id, :message, :_now_datetime_utc ) write: true + """) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + mydatabase: + queries: + add_message: + allow: + id: "*" + sql: |- + INSERT INTO messages ( + user_id, message, datetime + ) VALUES ( + :_actor_id, :message, :_now_datetime_utc + ) + write: true + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "mydatabase": { + "queries": { + "add_message": { + "allow": { + "id": "*" + }, + "sql": "INSERT INTO messages (\n user_id, message, datetime\n) VALUES (\n :_actor_id, :message, :_now_datetime_utc\n)", + "write": true + } + } + } + } + } +.. [[[end]]] The form presented at ``/mydatabase/add_message`` will have just a field for ``message`` - the other parameters will be populated by the magic parameter mechanism. diff --git a/setup.py b/setup.py index be9c7fde..61860f43 100644 --- a/setup.py +++ b/setup.py @@ -76,6 +76,7 @@ setup( "blacken-docs", "sphinx-copybutton", "sphinx-inline-tabs", + "ruamel.yaml", ], "test": [ "pytest>=5.2.2", From 50a6355c081a79c63625d96d3d350603298da42b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jul 2023 11:22:21 -0700 Subject: [PATCH 032/767] Workaround to get sphinx-build working again, refs 1153 --- .github/workflows/deploy-latest.yml | 2 +- .github/workflows/publish.yml | 2 +- docs/conf.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index d6baa2f6..ed60376c 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -43,7 +43,7 @@ jobs: if: ${{ github.ref == 'refs/heads/main' }} run: |- cd docs - sphinx-build -b xml . _build + DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - name: Set up the alternate-route demo diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b555a1eb..60f32ed6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -84,7 +84,7 @@ jobs: - name: Build docs.db run: |- cd docs - sphinx-build -b xml . _build + DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build sphinx-to-sqlite ../docs.db _build cd .. - name: Set up Cloud Run diff --git a/docs/conf.py b/docs/conf.py index 67f7fd11..ca0eb986 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,7 +17,8 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os +import os + # import sys # sys.path.insert(0, os.path.abspath('.')) @@ -35,8 +36,9 @@ extensions = [ "sphinx.ext.extlinks", "sphinx.ext.autodoc", "sphinx_copybutton", - "sphinx_inline_tabs", ] +if not os.environ.get("DISABLE_SPHINX_INLINE_TABS"): + extensions += ["sphinx_inline_tabs"] extlinks = { "issue": ("https://github.com/simonw/datasette/issues/%s", "#%s"), From 45e6d370ce3becc17ce8d06a8257bef691a3cc10 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jul 2023 11:35:15 -0700 Subject: [PATCH 033/767] Install docs dependencies for tests, refs #1153 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39aa8b13..d44a5751 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install -e '.[test]' + pip install -e '.[test,docs]' pip freeze - name: Run tests run: | From 2fd871a9063fd0e1bc60ddb11756c6954460dfc0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jul 2023 11:40:19 -0700 Subject: [PATCH 034/767] Drop support for Python 3.7, refs #2097 --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 2 +- README.md | 2 +- datasette/app.py | 16 +++++++--------- docs/contributing.rst | 2 +- docs/installation.rst | 2 +- pytest.ini | 2 -- setup.py | 3 +-- 8 files changed, 13 insertions(+), 18 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 60f32ed6..64a03a77 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d44a5751..4eab1fdb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/README.md b/README.md index af95b85e..93d81ae6 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ You can also install it using `pip` or `pipx`: pip install datasette -Datasette requires Python 3.7 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. +Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker. ## Basic usage diff --git a/datasette/app.py b/datasette/app.py index 1f80c5a9..977839e0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1072,15 +1072,13 @@ class Datasette: {"name": t.name, "ident": t.ident, "daemon": t.daemon} for t in threads ], } - # Only available in Python 3.7+ - if hasattr(asyncio, "all_tasks"): - tasks = asyncio.all_tasks() - d.update( - { - "num_tasks": len(tasks), - "tasks": [_cleaner_task_str(t) for t in tasks], - } - ) + tasks = asyncio.all_tasks() + d.update( + { + "num_tasks": len(tasks), + "tasks": [_cleaner_task_str(t) for t in tasks], + } + ) return d def _actor(self, request): diff --git a/docs/contributing.rst b/docs/contributing.rst index 2d5e1655..697002a8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -19,7 +19,7 @@ General guidelines Setting up a development environment ------------------------------------ -If you have Python 3.7 or higher installed on your computer (on OS X the quickest way to do this `is using homebrew `__) you can install an editable copy of Datasette using the following steps. +If you have Python 3.8 or higher installed on your computer (on OS X the quickest way to do this `is using homebrew `__) you can install an editable copy of Datasette using the following steps. If you want to use GitHub to publish your changes, first `create a fork of datasette `__ under your own GitHub account. diff --git a/docs/installation.rst b/docs/installation.rst index e1de0ae2..52f87863 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -57,7 +57,7 @@ If the latest packaged release of Datasette has not yet been made available thro Using pip --------- -Datasette requires Python 3.7 or higher. The `Python.org Python For Beginners `__ page has instructions for getting started. +Datasette requires Python 3.8 or higher. The `Python.org Python For Beginners `__ page has instructions for getting started. You can install Datasette and its dependencies using ``pip``:: diff --git a/pytest.ini b/pytest.ini index 559e518c..e4fcd380 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,8 +4,6 @@ filterwarnings= ignore:Using or importing the ABCs::jinja2 # https://bugs.launchpad.net/beautifulsoup/+bug/1778909 ignore:Using or importing the ABCs::bs4.element - # Python 3.7 PendingDeprecationWarning: Task.current_task() - ignore:.*current_task.*:PendingDeprecationWarning markers = serial: tests to avoid using with pytest-xdist asyncio_mode = strict diff --git a/setup.py b/setup.py index 61860f43..50c70b5f 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setup( packages=find_packages(exclude=("tests",)), package_data={"datasette": ["templates/*.html"]}, include_package_data=True, - python_requires=">=3.7", + python_requires=">=3.8", install_requires=[ "asgiref>=3.2.10", "click>=7.1.1", @@ -104,6 +104,5 @@ setup( "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.7", ], ) From 42ca574720cca4c1451a3cbf0e1aa85df174153f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jul 2023 12:50:22 -0700 Subject: [PATCH 035/767] Removed accidental test code I added, refs #1153 --- docs/index.rst | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index c10ed5a1..f5c1f232 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,18 +31,6 @@ Interested in learning Datasette? Start with `the official tutorials `__. -.. tab:: macOS/Linux/other Unix - - .. code-block:: shell - - python -m pip install webcolors - -.. tab:: Windows - - .. code-block:: shell - - py -m pip install webcolors - Contents -------- From 0f7192b6154edb576c41b55bd3f2a3f53e5f436a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 8 Jul 2023 13:08:09 -0700 Subject: [PATCH 036/767] One last YAML/JSON change, closes #1153 --- docs/writing_plugins.rst | 77 +++++++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index a3fc88ec..a84789b5 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -184,10 +184,12 @@ This will return the ``{"latitude_column": "lat", "longitude_column": "lng"}`` i If there is no configuration for that plugin, the method will return ``None``. -If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option like so:: +If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option like so: - { - "databases: { +.. [[[cog + from metadata_doc import metadata_example + metadata_example(cog, { + "databases": { "sf-trees": { "plugins": { "datasette-cluster-map": { @@ -197,13 +199,45 @@ If it cannot find the requested configuration at the table layer, it will fall b } } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + databases: + sf-trees: + plugins: + datasette-cluster-map: + latitude_column: xlat + longitude_column: xlng + + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "sf-trees": { + "plugins": { + "datasette-cluster-map": { + "latitude_column": "xlat", + "longitude_column": "xlng" + } + } + } + } + } +.. [[[end]]] In this case, the above code would return that configuration for ANY table within the ``sf-trees`` database. -The plugin configuration could also be set at the top level of ``metadata.json``:: +The plugin configuration could also be set at the top level of ``metadata.yaml``: - { +.. [[[cog + metadata_example(cog, { "title": "This is the top-level title in metadata.json", "plugins": { "datasette-cluster-map": { @@ -211,7 +245,34 @@ The plugin configuration could also be set at the top level of ``metadata.json`` "longitude_column": "xlng" } } - } + }) +.. ]]] + +.. tab:: YAML + + .. code-block:: yaml + + title: This is the top-level title in metadata.json + plugins: + datasette-cluster-map: + latitude_column: xlat + longitude_column: xlng + + +.. tab:: JSON + + .. code-block:: json + + { + "title": "This is the top-level title in metadata.json", + "plugins": { + "datasette-cluster-map": { + "latitude_column": "xlat", + "longitude_column": "xlng" + } + } + } +.. [[[end]]] Now that ``datasette-cluster-map`` plugin configuration will apply to every table in every database. @@ -234,7 +295,7 @@ To avoid accidentally conflicting with a database file that may be loaded into D - ``/-/upload-excel`` -Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `plugins directory `. +Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `plugins directory `__. If your plugin includes functionality that relates to a specific database you could also register a URL route like this: From 3a51ca901427ac93b0ed0638fd1f82fb647a299a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 14:19:24 -0700 Subject: [PATCH 037/767] Bump black from 23.3.0 to 23.7.0 (#2099) Bumps [black](https://github.com/psf/black) from 23.3.0 to 23.7.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.3.0...23.7.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 50c70b5f..4ecf9a0f 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==23.3.0", + "black==23.7.0", "blacken-docs==1.14.0", "pytest-timeout>=1.4.2", "trustme>=0.7", From 278ac91a4d68da333a6ed74a2c75b3ec9db8bd19 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 22 Jul 2023 11:42:46 -0700 Subject: [PATCH 038/767] datasette install -e option, closes #2106 --- datasette/cli.py | 12 ++++++++++-- docs/cli-reference.rst | 7 ++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index a6de9e6d..4ccf1963 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -351,13 +351,21 @@ def package( type=click.Path(exists=True), help="Install from requirements file", ) -def install(packages, upgrade, requirement): +@click.option( + "-e", + "--editable", + type=click.Path(readable=True, exists=True, dir_okay=True, file_okay=False), + help="Install a project in editable mode from this path", +) +def install(packages, upgrade, requirement, editable): """Install plugins and packages from PyPI into the same environment as Datasette""" - if not packages and not requirement: + if not packages and not requirement and not editable: raise click.UsageError("Please specify at least one package to install") args = ["pip", "install"] if upgrade: args += ["--upgrade"] + if editable: + args += ["--editable", str(editable)] if requirement: args += ["-r", requirement] args += list(packages) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index af747ab4..c6d94303 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -350,9 +350,10 @@ Would install the `datasette-cluster-map Date: Wed, 26 Jul 2023 11:28:03 -0700 Subject: [PATCH 039/767] Make editable work with -e '.[test]', refs #2106 --- datasette/cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 4ccf1963..32266888 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -354,7 +354,6 @@ def package( @click.option( "-e", "--editable", - type=click.Path(readable=True, exists=True, dir_okay=True, file_okay=False), help="Install a project in editable mode from this path", ) def install(packages, upgrade, requirement, editable): @@ -365,7 +364,7 @@ def install(packages, upgrade, requirement, editable): if upgrade: args += ["--upgrade"] if editable: - args += ["--editable", str(editable)] + args += ["--editable", editable] if requirement: args += ["-r", requirement] args += list(packages) From 18dd88ee4d78fe9d760e9da96028ae06d938a85c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Jul 2023 11:43:55 -0700 Subject: [PATCH 040/767] Refactored DatabaseDownload to database_download, closes #2116 --- datasette/app.py | 7 +++- datasette/views/database.py | 78 ++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 977839e0..b8b84168 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -34,7 +34,7 @@ from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound from .views.base import ureg -from .views.database import DatabaseDownload, DatabaseView, TableCreateView +from .views.database import database_download, DatabaseView, TableCreateView from .views.index import IndexView from .views.special import ( JsonDataView, @@ -1363,7 +1363,10 @@ class Datasette: wrap_view(PatternPortfolioView, self), r"/-/patterns$", ) - add_route(DatabaseDownload.as_view(self), r"/(?P[^\/\.]+)\.db$") + add_route( + wrap_view(database_download, self), + r"/(?P[^\/\.]+)\.db$", + ) add_route( DatabaseView.as_view(self), r"/(?P[^\/\.]+)(\.(?P\w+))?$" ) diff --git a/datasette/views/database.py b/datasette/views/database.py index dda82510..ffa79e96 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -169,47 +169,45 @@ class DatabaseView(DataView): ) -class DatabaseDownload(DataView): - name = "database_download" +async def database_download(request, datasette): + database = tilde_decode(request.url_vars["database"]) + await datasette.ensure_permissions( + request.actor, + [ + ("view-database-download", database), + ("view-database", database), + "view-instance", + ], + ) + try: + db = datasette.get_database(route=database) + except KeyError: + raise DatasetteError("Invalid database", status=404) - async def get(self, request): - database = tilde_decode(request.url_vars["database"]) - await self.ds.ensure_permissions( - request.actor, - [ - ("view-database-download", database), - ("view-database", database), - "view-instance", - ], - ) - try: - db = self.ds.get_database(route=database) - except KeyError: - raise DatasetteError("Invalid database", status=404) - if db.is_memory: - raise DatasetteError("Cannot download in-memory databases", status=404) - if not self.ds.setting("allow_download") or db.is_mutable: - raise Forbidden("Database download is forbidden") - if not db.path: - raise DatasetteError("Cannot download database", status=404) - filepath = db.path - headers = {} - if self.ds.cors: - add_cors_headers(headers) - if db.hash: - etag = '"{}"'.format(db.hash) - headers["Etag"] = etag - # Has user seen this already? - if_none_match = request.headers.get("if-none-match") - if if_none_match and if_none_match == etag: - return Response("", status=304) - headers["Transfer-Encoding"] = "chunked" - return AsgiFileDownload( - filepath, - filename=os.path.basename(filepath), - content_type="application/octet-stream", - headers=headers, - ) + if db.is_memory: + raise DatasetteError("Cannot download in-memory databases", status=404) + if not datasette.setting("allow_download") or db.is_mutable: + raise Forbidden("Database download is forbidden") + if not db.path: + raise DatasetteError("Cannot download database", status=404) + filepath = db.path + headers = {} + if datasette.cors: + add_cors_headers(headers) + if db.hash: + etag = '"{}"'.format(db.hash) + headers["Etag"] = etag + # Has user seen this already? + if_none_match = request.headers.get("if-none-match") + if if_none_match and if_none_match == etag: + return Response("", status=304) + headers["Transfer-Encoding"] = "chunked" + return AsgiFileDownload( + filepath, + filename=os.path.basename(filepath), + content_type="application/octet-stream", + headers=headers, + ) class QueryView(DataView): From 08181823990a71ffa5a1b57b37259198eaa43e06 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Jul 2023 11:52:37 -0700 Subject: [PATCH 041/767] Update cli-reference for editable change, refs #2106 --- docs/cli-reference.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c6d94303..2177fc9e 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -350,10 +350,10 @@ Would install the `datasette-cluster-map Date: Mon, 7 Aug 2023 08:45:10 -0700 Subject: [PATCH 042/767] Use dependabot grouped updates --- .github/dependabot.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b969c4c1..88bb03b1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,9 +5,7 @@ updates: schedule: interval: daily time: "13:00" - open-pull-requests-limit: 10 - ignore: - - dependency-name: black - versions: - - 21.4b0 - - 21.4b1 + groups: + python-packages: + patterns: + - "*" From 5139c0886a7f6bb94d317ba0665aa6e728716028 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:19:23 -0700 Subject: [PATCH 043/767] Bump the python-packages group with 3 updates (#2128) Bumps the python-packages group with 3 updates: [sphinx](https://github.com/sphinx-doc/sphinx), [furo](https://github.com/pradyunsg/furo) and [blacken-docs](https://github.com/asottile/blacken-docs). Updates `sphinx` from 6.1.3 to 7.1.2 - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v6.1.3...v7.1.2) Updates `furo` from 2023.3.27 to 2023.7.26 - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.03.27...2023.07.26) Updates `blacken-docs` from 1.14.0 to 1.15.0 - [Changelog](https://github.com/adamchainz/blacken-docs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/asottile/blacken-docs/compare/1.14.0...1.15.0) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-major dependency-group: python-packages - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: blacken-docs dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 4ecf9a0f..3a105523 100644 --- a/setup.py +++ b/setup.py @@ -69,8 +69,8 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "Sphinx==6.1.3", - "furo==2023.3.27", + "Sphinx==7.1.2", + "furo==2023.7.26", "sphinx-autobuild", "codespell>=2.2.5", "blacken-docs", @@ -84,7 +84,7 @@ setup( "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", "black==23.7.0", - "blacken-docs==1.14.0", + "blacken-docs==1.15.0", "pytest-timeout>=1.4.2", "trustme>=0.7", "cogapp>=3.3.0", From 1377a290cd85ba8d3338b1da47c4665ed4c6c625 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 7 Aug 2023 18:47:39 -0700 Subject: [PATCH 044/767] New JSON design for query views (#2118) * Refs #2111, closes #2110 * New Context dataclass/subclass mechanism, refs #2127 * Define QueryContext and extract get_tables() method, refs #2127 * Fix OPTIONS bug by porting DaatbaseView to be a View subclass * Expose async_view_for_class.view_class for test_routes test * Error/truncated aruments for renderers, closes #2130 --- datasette/__init__.py | 3 +- datasette/app.py | 16 +- datasette/renderer.py | 21 +- datasette/views/__init__.py | 3 + datasette/views/base.py | 2 + datasette/views/database.py | 574 +++++++++++++++++++++++++----- datasette/views/table.py | 2 + docs/plugin_hooks.rst | 6 + tests/test_api.py | 27 +- tests/test_cli_serve_get.py | 1 - tests/test_html.py | 8 +- tests/test_internals_datasette.py | 20 +- tests/test_messages.py | 6 +- tests/test_plugins.py | 5 +- tests/test_table_api.py | 1 - 15 files changed, 581 insertions(+), 114 deletions(-) diff --git a/datasette/__init__.py b/datasette/__init__.py index 64fb4ff7..271e09ad 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,6 +1,7 @@ -from datasette.permissions import Permission +from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa from datasette.utils import actor_matches_allow # noqa +from datasette.views import Context # noqa from .hookspecs import hookimpl # noqa from .hookspecs import hookspec # noqa diff --git a/datasette/app.py b/datasette/app.py index b8b84168..b2644ace 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,7 +1,8 @@ import asyncio -from typing import Sequence, Union, Tuple, Optional, Dict, Iterable +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union import asgi_csrf import collections +import dataclasses import datetime import functools import glob @@ -33,6 +34,7 @@ from jinja2 import ( from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound +from .views import Context from .views.base import ureg from .views.database import database_download, DatabaseView, TableCreateView from .views.index import IndexView @@ -1115,7 +1117,11 @@ class Datasette: ) async def render_template( - self, templates, context=None, request=None, view_name=None + self, + templates: Union[List[str], str, Template], + context: Optional[Union[Dict[str, Any], Context]] = None, + request: Optional[Request] = None, + view_name: Optional[str] = None, ): if not self._startup_invoked: raise Exception("render_template() called before await ds.invoke_startup()") @@ -1126,6 +1132,8 @@ class Datasette: if isinstance(templates, str): templates = [templates] template = self.jinja_env.select_template(templates) + if dataclasses.is_dataclass(context): + context = dataclasses.asdict(context) body_scripts = [] # pylint: disable=no-member for extra_script in pm.hook.extra_body_script( @@ -1368,7 +1376,8 @@ class Datasette: r"/(?P[^\/\.]+)\.db$", ) add_route( - DatabaseView.as_view(self), r"/(?P[^\/\.]+)(\.(?P\w+))?$" + wrap_view(DatabaseView, self), + r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") add_route( @@ -1707,6 +1716,7 @@ def wrap_view_class(view_class, datasette): datasette=datasette, ) + async_view_for_class.view_class = view_class return async_view_for_class diff --git a/datasette/renderer.py b/datasette/renderer.py index 5354f348..0bd74e81 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -27,7 +27,7 @@ def convert_specific_columns_to_json(rows, columns, json_cols): return new_rows -def json_renderer(args, data, view_name): +def json_renderer(args, data, error, truncated=None): """Render a response as JSON""" status_code = 200 @@ -47,8 +47,15 @@ def json_renderer(args, data, view_name): # Deal with the _shape option shape = args.get("_shape", "objects") # if there's an error, ignore the shape entirely - if data.get("error"): + data["ok"] = True + if error: shape = "objects" + status_code = 400 + data["error"] = error + data["ok"] = False + + if truncated is not None: + data["truncated"] = truncated if shape == "arrayfirst": if not data["rows"]: @@ -64,13 +71,13 @@ def json_renderer(args, data, view_name): if rows and columns: data["rows"] = [dict(zip(columns, row)) for row in rows] if shape == "object": - error = None + shape_error = None if "primary_keys" not in data: - error = "_shape=object is only available on tables" + shape_error = "_shape=object is only available on tables" else: pks = data["primary_keys"] if not pks: - error = ( + shape_error = ( "_shape=object not available for tables with no primary keys" ) else: @@ -79,8 +86,8 @@ def json_renderer(args, data, view_name): pk_string = path_from_row_pks(row, pks, not pks) object_rows[pk_string] = row data = object_rows - if error: - data = {"ok": False, "error": error} + if shape_error: + data = {"ok": False, "error": shape_error} elif shape == "array": data = data["rows"] diff --git a/datasette/views/__init__.py b/datasette/views/__init__.py index e69de29b..e3b1b7f4 100644 --- a/datasette/views/__init__.py +++ b/datasette/views/__init__.py @@ -0,0 +1,3 @@ +class Context: + "Base class for all documented contexts" + pass diff --git a/datasette/views/base.py b/datasette/views/base.py index 94645cd8..da5c55ad 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -309,6 +309,8 @@ class DataView(BaseView): table=data.get("table"), request=request, view_name=self.name, + truncated=False, # TODO: support this + error=data.get("error"), # These will be deprecated in Datasette 1.0: args=request.args, data=data, diff --git a/datasette/views/database.py b/datasette/views/database.py index ffa79e96..77f3f5b0 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,17 +1,22 @@ -import os +from asyncinject import Registry +from dataclasses import dataclass, field +from typing import Callable +from urllib.parse import parse_qsl, urlencode +import asyncio import hashlib import itertools import json -from markupsafe import Markup, escape -from urllib.parse import parse_qsl, urlencode +import markupsafe +import os import re import sqlite_utils +import textwrap -import markupsafe - +from datasette.database import QueryInterrupted from datasette.utils import ( add_cors_headers, await_me_maybe, + call_with_supported_arguments, derive_named_parameters, format_bytes, tilde_decode, @@ -28,17 +33,19 @@ from datasette.utils import ( from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden from datasette.plugins import pm -from .base import BaseView, DatasetteError, DataView, _error +from .base import BaseView, DatasetteError, DataView, View, _error, stream_csv -class DatabaseView(DataView): - name = "database" +class DatabaseView(View): + async def get(self, request, datasette): + format_ = request.url_vars.get("format") or "html" - async def data(self, request, default_labels=False, _size=None): - db = await self.ds.resolve_database(request) + await datasette.refresh_schemas() + + db = await datasette.resolve_database(request) database = db.name - visible, private = await self.ds.check_visibility( + visible, private = await datasette.check_visibility( request.actor, permissions=[ ("view-database", database), @@ -48,23 +55,23 @@ class DatabaseView(DataView): if not visible: raise Forbidden("You do not have permission to view this database") - metadata = (self.ds.metadata("databases") or {}).get(database, {}) - self.ds.update_with_inherited_metadata(metadata) + sql = (request.args.get("sql") or "").strip() + if sql: + return await query_view(request, datasette) - if request.args.get("sql"): - sql = request.args.get("sql") - validate_sql_select(sql) - return await QueryView(self.ds).data( - request, sql, _size=_size, metadata=metadata - ) + if format_ not in ("html", "json"): + raise NotFound("Invalid format: {}".format(format_)) + + metadata = (datasette.metadata("databases") or {}).get(database, {}) + datasette.update_with_inherited_metadata(metadata) table_counts = await db.table_counts(5) hidden_table_names = set(await db.hidden_table_names()) all_foreign_keys = await db.get_all_foreign_keys() - views = [] + sql_views = [] for view_name in await db.view_names(): - view_visible, view_private = await self.ds.check_visibility( + view_visible, view_private = await datasette.check_visibility( request.actor, permissions=[ ("view-table", (database, view_name)), @@ -73,45 +80,19 @@ class DatabaseView(DataView): ], ) if view_visible: - views.append( + sql_views.append( { "name": view_name, "private": view_private, } ) - tables = [] - for table in table_counts: - table_visible, table_private = await self.ds.check_visibility( - request.actor, - permissions=[ - ("view-table", (database, table)), - ("view-database", database), - "view-instance", - ], - ) - if not table_visible: - continue - table_columns = await db.table_columns(table) - tables.append( - { - "name": table, - "columns": table_columns, - "primary_keys": await db.primary_keys(table), - "count": table_counts[table], - "hidden": table in hidden_table_names, - "fts_table": await db.fts_table(table), - "foreign_keys": all_foreign_keys[table], - "private": table_private, - } - ) - - tables.sort(key=lambda t: (t["hidden"], t["name"])) + tables = await get_tables(datasette, request, db) canned_queries = [] for query in ( - await self.ds.get_canned_queries(database, request.actor) + await datasette.get_canned_queries(database, request.actor) ).values(): - query_visible, query_private = await self.ds.check_visibility( + query_visible, query_private = await datasette.check_visibility( request.actor, permissions=[ ("view-query", (database, query["name"])), @@ -125,7 +106,7 @@ class DatabaseView(DataView): async def database_actions(): links = [] for hook in pm.hook.database_actions( - datasette=self.ds, + datasette=datasette, database=database, actor=request.actor, request=request, @@ -137,36 +118,165 @@ class DatabaseView(DataView): attached_databases = [d.name for d in await db.attached_databases()] - allow_execute_sql = await self.ds.permission_allowed( + allow_execute_sql = await datasette.permission_allowed( request.actor, "execute-sql", database ) - return ( - { - "database": database, - "private": private, - "path": self.ds.urls.database(database), - "size": db.size, - "tables": tables, - "hidden_count": len([t for t in tables if t["hidden"]]), - "views": views, - "queries": canned_queries, - "allow_execute_sql": allow_execute_sql, - "table_columns": await _table_columns(self.ds, database) - if allow_execute_sql - else {}, - }, - { - "database_actions": database_actions, - "show_hidden": request.args.get("_show_hidden"), - "editable": True, - "metadata": metadata, - "allow_download": self.ds.setting("allow_download") - and not db.is_mutable - and not db.is_memory, - "attached_databases": attached_databases, - }, - (f"database-{to_css_class(database)}.html", "database.html"), + json_data = { + "database": database, + "private": private, + "path": datasette.urls.database(database), + "size": db.size, + "tables": tables, + "hidden_count": len([t for t in tables if t["hidden"]]), + "views": sql_views, + "queries": canned_queries, + "allow_execute_sql": allow_execute_sql, + "table_columns": await _table_columns(datasette, database) + if allow_execute_sql + else {}, + } + + if format_ == "json": + response = Response.json(json_data) + if datasette.cors: + add_cors_headers(response.headers) + return response + + assert format_ == "html" + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), ) + templates = (f"database-{to_css_class(database)}.html", "database.html") + template = datasette.jinja_env.select_template(templates) + context = { + **json_data, + "database_actions": database_actions, + "show_hidden": request.args.get("_show_hidden"), + "editable": True, + "metadata": metadata, + "allow_download": datasette.setting("allow_download") + and not db.is_mutable + and not db.is_memory, + "attached_databases": attached_databases, + "database_color": lambda _: "#ff0000", + "alternate_url_json": alternate_url_json, + "select_templates": [ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + } + return Response.html( + await datasette.render_template( + templates, + context, + request=request, + view_name="database", + ), + headers={ + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + }, + ) + + +@dataclass +class QueryContext: + database: str = field(metadata={"help": "The name of the database being queried"}) + query: dict = field( + metadata={"help": "The SQL query object containing the `sql` string"} + ) + canned_query: str = field( + metadata={"help": "The name of the canned query if this is a canned query"} + ) + private: bool = field( + metadata={"help": "Boolean indicating if this is a private database"} + ) + # urls: dict = field( + # metadata={"help": "Object containing URL helpers like `database()`"} + # ) + canned_write: bool = field( + metadata={"help": "Boolean indicating if this canned query allows writes"} + ) + db_is_immutable: bool = field( + metadata={"help": "Boolean indicating if this database is immutable"} + ) + error: str = field(metadata={"help": "Any query error message"}) + hide_sql: bool = field( + metadata={"help": "Boolean indicating if the SQL should be hidden"} + ) + show_hide_link: str = field( + metadata={"help": "The URL to toggle showing/hiding the SQL"} + ) + show_hide_text: str = field( + metadata={"help": "The text for the show/hide SQL link"} + ) + editable: bool = field( + metadata={"help": "Boolean indicating if the SQL can be edited"} + ) + allow_execute_sql: bool = field( + metadata={"help": "Boolean indicating if custom SQL can be executed"} + ) + tables: list = field(metadata={"help": "List of table objects in the database"}) + named_parameter_values: dict = field( + metadata={"help": "Dictionary of parameter names/values"} + ) + edit_sql_url: str = field( + metadata={"help": "URL to edit the SQL for a canned query"} + ) + display_rows: list = field(metadata={"help": "List of result rows to display"}) + columns: list = field(metadata={"help": "List of column names"}) + renderers: dict = field(metadata={"help": "Dictionary of renderer name to URL"}) + url_csv: str = field(metadata={"help": "URL for CSV export"}) + show_hide_hidden: str = field( + metadata={"help": "Hidden input field for the _show_sql parameter"} + ) + metadata: dict = field(metadata={"help": "Metadata about the query/database"}) + database_color: Callable = field( + metadata={"help": "Function that returns a color for a given database name"} + ) + table_columns: dict = field( + metadata={"help": "Dictionary of table name to list of column names"} + ) + alternate_url_json: str = field( + metadata={"help": "URL for alternate JSON version of this page"} + ) + + +async def get_tables(datasette, request, db): + tables = [] + database = db.name + table_counts = await db.table_counts(5) + hidden_table_names = set(await db.hidden_table_names()) + all_foreign_keys = await db.get_all_foreign_keys() + + for table in table_counts: + table_visible, table_private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-table", (database, table)), + ("view-database", database), + "view-instance", + ], + ) + if not table_visible: + continue + table_columns = await db.table_columns(table) + tables.append( + { + "name": table, + "columns": table_columns, + "primary_keys": await db.primary_keys(table), + "count": table_counts[table], + "hidden": table in hidden_table_names, + "fts_table": await db.fts_table(table), + "foreign_keys": all_foreign_keys[table], + "private": table_private, + } + ) + tables.sort(key=lambda t: (t["hidden"], t["name"])) + return tables async def database_download(request, datasette): @@ -210,6 +320,244 @@ async def database_download(request, datasette): ) +async def query_view( + request, + datasette, + # canned_query=None, + # _size=None, + # named_parameters=None, + # write=False, +): + db = await datasette.resolve_database(request) + database = db.name + # Flattened because of ?sql=&name1=value1&name2=value2 feature + params = {key: request.args.get(key) for key in request.args} + sql = None + if "sql" in params: + sql = params.pop("sql") + if "_shape" in params: + params.pop("_shape") + + # extras come from original request.args to avoid being flattened + extras = request.args.getlist("_extra") + + # TODO: Behave differently for canned query here: + await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + + _, private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-database", database), + "view-instance", + ], + ) + + extra_args = {} + if params.get("_timelimit"): + extra_args["custom_time_limit"] = int(params["_timelimit"]) + + format_ = request.url_vars.get("format") or "html" + query_error = None + try: + validate_sql_select(sql) + results = await datasette.execute( + database, sql, params, truncate=True, **extra_args + ) + columns = results.columns + rows = results.rows + except QueryInterrupted as ex: + raise DatasetteError( + textwrap.dedent( + """ +

SQL query took too long. The time limit is controlled by the + sql_time_limit_ms + configuration option.

+ + + """.format( + markupsafe.escape(ex.sql) + ) + ).strip(), + title="SQL Interrupted", + status=400, + message_is_html=True, + ) + except sqlite3.DatabaseError as ex: + query_error = str(ex) + results = None + rows = [] + columns = [] + except (sqlite3.OperationalError, InvalidSql) as ex: + raise DatasetteError(str(ex), title="Invalid SQL", status=400) + except sqlite3.OperationalError as ex: + raise DatasetteError(str(ex)) + except DatasetteError: + raise + + # Handle formats from plugins + if format_ == "csv": + + async def fetch_data_for_csv(request, _next=None): + results = await db.execute(sql, params, truncate=True) + data = {"rows": results.rows, "columns": results.columns} + return data, None, None + + return await stream_csv(datasette, fetch_data_for_csv, request, db.name) + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=None, + database=database, + table=None, + request=request, + view_name="table", + truncated=results.truncated if results else False, + error=query_error, + # These will be deprecated in Datasette 1.0: + args=request.args, + data={"rows": rows, "columns": columns}, + ) + if asyncio.iscoroutine(result): + result = await result + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(database)}.html", "query.html"] + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + data = {} + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + metadata = (datasette.metadata("databases") or {}).get(database, {}) + datasette.update_with_inherited_metadata(metadata) + + renderers = {} + for key, (_, can_render) in datasette.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=datasette, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name="database", + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = datasette.urls.path( + path_with_format(request=request, format=key) + ) + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + + show_hide_hidden = "" + if metadata.get("hide_sql"): + if bool(params.get("_show_sql")): + show_hide_link = path_with_removed_args(request, {"_show_sql"}) + show_hide_text = "hide" + show_hide_hidden = '' + else: + show_hide_link = path_with_added_args(request, {"_show_sql": 1}) + show_hide_text = "show" + else: + if bool(params.get("_hide_sql")): + show_hide_link = path_with_removed_args(request, {"_hide_sql"}) + show_hide_text = "show" + show_hide_hidden = '' + else: + show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) + show_hide_text = "hide" + hide_sql = show_hide_text == "show" + + r = Response.html( + await datasette.render_template( + template, + QueryContext( + database=database, + query={ + "sql": sql, + # TODO: Params? + }, + canned_query=None, + private=private, + canned_write=False, + db_is_immutable=not db.is_mutable, + error=query_error, + hide_sql=hide_sql, + show_hide_link=datasette.urls.path(show_hide_link), + show_hide_text=show_hide_text, + editable=True, # TODO + allow_execute_sql=allow_execute_sql, + tables=await get_tables(datasette, request, db), + named_parameter_values={}, # TODO + edit_sql_url="todo", + display_rows=await display_rows( + datasette, database, request, rows, columns + ), + table_columns=await _table_columns(datasette, database) + if allow_execute_sql + else {}, + columns=columns, + renderers=renderers, + url_csv=datasette.urls.path( + path_with_format( + request=request, format="csv", extra_qs={"_size": "max"} + ) + ), + show_hide_hidden=markupsafe.Markup(show_hide_hidden), + metadata=metadata, + database_color=lambda _: "#ff0000", + alternate_url_json=alternate_url_json, + ), + request=request, + view_name="database", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + if datasette.cors: + add_cors_headers(r.headers) + return r + + class QueryView(DataView): async def data( self, @@ -404,7 +752,7 @@ class QueryView(DataView): display_value = plugin_display_value else: if value in ("", None): - display_value = Markup(" ") + display_value = markupsafe.Markup(" ") elif is_url(str(display_value).strip()): display_value = markupsafe.Markup( '{truncated_url}'.format( @@ -755,3 +1103,69 @@ async def _table_columns(datasette, database_name): for view_name in await db.view_names(): table_columns[view_name] = [] return table_columns + + +async def display_rows(datasette, database, request, rows, columns): + display_rows = [] + truncate_cells = datasette.setting("truncate_cells_html") + for row in rows: + display_row = [] + for column, value in zip(columns, row): + display_value = value + # Let the plugins have a go + # pylint: disable=no-member + plugin_display_value = None + for candidate in pm.hook.render_cell( + row=row, + value=value, + column=column, + table=None, + database=database, + datasette=datasette, + request=request, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value is not None: + display_value = plugin_display_value + else: + if value in ("", None): + display_value = markupsafe.Markup(" ") + elif is_url(str(display_value).strip()): + display_value = markupsafe.Markup( + '{truncated_url}'.format( + url=markupsafe.escape(value.strip()), + truncated_url=markupsafe.escape( + truncate_url(value.strip(), truncate_cells) + ), + ) + ) + elif isinstance(display_value, bytes): + blob_url = path_with_format( + request=request, + format="blob", + extra_qs={ + "_blob_column": column, + "_blob_hash": hashlib.sha256(display_value).hexdigest(), + }, + ) + formatted = format_bytes(len(value)) + display_value = markupsafe.Markup( + '<Binary: {:,} byte{}>'.format( + blob_url, + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "", + len(value), + "" if len(value) == 1 else "s", + ) + ) + else: + display_value = str(value) + if truncate_cells and len(display_value) > truncate_cells: + display_value = display_value[:truncate_cells] + "\u2026" + display_row.append(display_value) + display_rows.append(display_row) + return display_rows diff --git a/datasette/views/table.py b/datasette/views/table.py index c102c103..77acfd95 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -833,6 +833,8 @@ async def table_view_traced(datasette, request): table=resolved.table, request=request, view_name="table", + truncated=False, + error=None, # These will be deprecated in Datasette 1.0: args=request.args, data=data, diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 97306529..9bbe6fc6 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -516,6 +516,12 @@ When a request is received, the ``"render"`` callback function is called with ze ``request`` - :ref:`internals_request` The current HTTP request. +``error`` - string or None + If an error occurred this string will contain the error message. + +``truncated`` - bool or None + If the query response was truncated - for example a SQL query returning more than 1,000 results where pagination is not available - this will be ``True``. + ``view_name`` - string The name of the current view being called. ``index``, ``database``, ``table``, and ``row`` are the most important ones. diff --git a/tests/test_api.py b/tests/test_api.py index 40a3e2b8..28415a0b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -638,22 +638,21 @@ def test_database_page_for_database_with_dot_in_name(app_client_with_dot): @pytest.mark.asyncio async def test_custom_sql(ds_client): response = await ds_client.get( - "/fixtures.json?sql=select+content+from+simple_primary_key&_shape=objects" + "/fixtures.json?sql=select+content+from+simple_primary_key" ) data = response.json() - assert {"sql": "select content from simple_primary_key", "params": {}} == data[ - "query" - ] - assert [ - {"content": "hello"}, - {"content": "world"}, - {"content": ""}, - {"content": "RENDER_CELL_DEMO"}, - {"content": "RENDER_CELL_ASYNC"}, - ] == data["rows"] - assert ["content"] == data["columns"] - assert "fixtures" == data["database"] - assert not data["truncated"] + assert data == { + "rows": [ + {"content": "hello"}, + {"content": "world"}, + {"content": ""}, + {"content": "RENDER_CELL_DEMO"}, + {"content": "RENDER_CELL_ASYNC"}, + ], + "columns": ["content"], + "ok": True, + "truncated": False, + } def test_sql_time_limit(app_client_shorter_time_limit): diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index ac44e1e2..2e0390bb 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -36,7 +36,6 @@ def test_serve_with_get(tmp_path_factory): ) assert 0 == result.exit_code, result.output assert { - "database": "_memory", "truncated": False, "columns": ["sqlite_version()"], }.items() <= json.loads(result.output).items() diff --git a/tests/test_html.py b/tests/test_html.py index eadbd720..6c3860d7 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -248,6 +248,9 @@ async def test_css_classes_on_body(ds_client, path, expected_classes): assert classes == expected_classes +templates_considered_re = re.compile(r"") + + @pytest.mark.asyncio @pytest.mark.parametrize( "path,expected_considered", @@ -271,7 +274,10 @@ async def test_css_classes_on_body(ds_client, path, expected_classes): async def test_templates_considered(ds_client, path, expected_considered): response = await ds_client.get(path) assert response.status_code == 200 - assert f"" in response.text + match = templates_considered_re.search(response.text) + assert match, "No templates considered comment found" + actual_considered = match.group(1) + assert actual_considered == expected_considered @pytest.mark.asyncio diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 3d5bb2da..d59ff729 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -1,10 +1,12 @@ """ Tests for the datasette.app.Datasette class """ -from datasette import Forbidden +import dataclasses +from datasette import Forbidden, Context from datasette.app import Datasette, Database from itsdangerous import BadSignature import pytest +from typing import Optional @pytest.fixture @@ -136,6 +138,22 @@ async def test_datasette_render_template_no_request(): assert "Error " in rendered +@pytest.mark.asyncio +async def test_datasette_render_template_with_dataclass(): + @dataclasses.dataclass + class ExampleContext(Context): + title: str + status: int + error: str + + context = ExampleContext(title="Hello", status=200, error="Error message") + ds = Datasette(memory=True) + await ds.invoke_startup() + rendered = await ds.render_template("error.html", context) + assert "

Hello

" in rendered + assert "Error message" in rendered + + def test_datasette_error_if_string_not_list(tmpdir): # https://github.com/simonw/datasette/issues/1985 db_path = str(tmpdir / "data.db") diff --git a/tests/test_messages.py b/tests/test_messages.py index 8417b9ae..a7e4d046 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -12,7 +12,7 @@ import pytest ], ) async def test_add_message_sets_cookie(ds_client, qs, expected): - response = await ds_client.get(f"/fixtures.message?{qs}") + response = await ds_client.get(f"/fixtures.message?sql=select+1&{qs}") signed = response.cookies["ds_messages"] decoded = ds_client.ds.unsign(signed, "messages") assert expected == decoded @@ -21,7 +21,9 @@ async def test_add_message_sets_cookie(ds_client, qs, expected): @pytest.mark.asyncio async def test_messages_are_displayed_and_cleared(ds_client): # First set the message cookie - set_msg_response = await ds_client.get("/fixtures.message?add_msg=xmessagex") + set_msg_response = await ds_client.get( + "/fixtures.message?sql=select+1&add_msg=xmessagex" + ) # Now access a page that displays messages response = await ds_client.get("/", cookies=set_msg_response.cookies) # Messages should be in that HTML diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 6971bbf7..28fe720f 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -121,9 +121,8 @@ async def test_hook_extra_css_urls(ds_client, path, expected_decoded_object): ][0]["href"] # This link has a base64-encoded JSON blob in it encoded = special_href.split("/")[3] - assert expected_decoded_object == json.loads( - base64.b64decode(encoded).decode("utf8") - ) + actual_decoded_object = json.loads(base64.b64decode(encoded).decode("utf8")) + assert expected_decoded_object == actual_decoded_object @pytest.mark.asyncio diff --git a/tests/test_table_api.py b/tests/test_table_api.py index cd664ffb..46d1c9b8 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -700,7 +700,6 @@ async def test_max_returned_rows(ds_client): "/fixtures.json?sql=select+content+from+no_primary_key" ) data = response.json() - assert {"sql": "select content from no_primary_key", "params": {}} == data["query"] assert data["truncated"] assert 100 == len(data["rows"]) From cd57b0f71234273156cb1eba3f9153b9e27ac14d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 8 Aug 2023 06:45:04 -0700 Subject: [PATCH 045/767] Brought back parameter fields, closes #2132 --- datasette/views/database.py | 19 +++++++++++++++++-- tests/test_html.py | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 77f3f5b0..0770a380 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -506,6 +506,21 @@ async def query_view( show_hide_text = "hide" hide_sql = show_hide_text == "show" + # Extract any :named parameters + named_parameters = await derive_named_parameters( + datasette.get_database(database), sql + ) + named_parameter_values = { + named_parameter: params.get(named_parameter) or "" + for named_parameter in named_parameters + if not named_parameter.startswith("_") + } + + # Set to blank string if missing from params + for named_parameter in named_parameters: + if named_parameter not in params and not named_parameter.startswith("_"): + params[named_parameter] = "" + r = Response.html( await datasette.render_template( template, @@ -513,7 +528,7 @@ async def query_view( database=database, query={ "sql": sql, - # TODO: Params? + "params": params, }, canned_query=None, private=private, @@ -526,7 +541,7 @@ async def query_view( editable=True, # TODO allow_execute_sql=allow_execute_sql, tables=await get_tables(datasette, request, db), - named_parameter_values={}, # TODO + named_parameter_values=named_parameter_values, edit_sql_url="todo", display_rows=await display_rows( datasette, database, request, rows, columns diff --git a/tests/test_html.py b/tests/test_html.py index 6c3860d7..7856bc27 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -295,6 +295,22 @@ async def test_query_json_csv_export_links(ds_client): assert 'CSV' in response.text +@pytest.mark.asyncio +async def test_query_parameter_form_fields(ds_client): + response = await ds_client.get("/fixtures?sql=select+:name") + assert response.status_code == 200 + assert ( + ' ' + in response.text + ) + response2 = await ds_client.get("/fixtures?sql=select+:name&name=hello") + assert response2.status_code == 200 + assert ( + ' ' + in response2.text + ) + + @pytest.mark.asyncio async def test_row_html_simple_primary_key(ds_client): response = await ds_client.get("/fixtures/simple_primary_key/1") From 26be9f0445b753fb84c802c356b0791a72269f25 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 08:26:52 -0700 Subject: [PATCH 046/767] Refactored canned query code, replaced old QueryView, closes #2114 --- datasette/templates/query.html | 10 +- datasette/views/database.py | 840 +++++++++++++-------------------- datasette/views/table.py | 60 +-- tests/test_canned_queries.py | 8 +- 4 files changed, 345 insertions(+), 573 deletions(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 7ffc250a..fc3b8527 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -24,7 +24,7 @@ {% block content %} -{% if canned_write and db_is_immutable %} +{% if canned_query_write and db_is_immutable %}

This query cannot be executed because the database is immutable.

{% endif %} @@ -32,7 +32,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} -
+

Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

@@ -61,8 +61,8 @@ {% endif %}

{% if not hide_sql %}{% endif %} - {% if canned_write %}{% endif %} - + {% if canned_query_write %}{% endif %} + {{ show_hide_hidden }} {% if canned_query and edit_sql_url %}Edit SQL{% endif %}

@@ -87,7 +87,7 @@ {% else %} - {% if not canned_write and not error %} + {% if not canned_query_write and not error %}

0 results

{% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 0770a380..658c35e6 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,4 +1,3 @@ -from asyncinject import Registry from dataclasses import dataclass, field from typing import Callable from urllib.parse import parse_qsl, urlencode @@ -33,7 +32,7 @@ from datasette.utils import ( from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden from datasette.plugins import pm -from .base import BaseView, DatasetteError, DataView, View, _error, stream_csv +from .base import BaseView, DatasetteError, View, _error, stream_csv class DatabaseView(View): @@ -57,7 +56,7 @@ class DatabaseView(View): sql = (request.args.get("sql") or "").strip() if sql: - return await query_view(request, datasette) + return await QueryView()(request, datasette) if format_ not in ("html", "json"): raise NotFound("Invalid format: {}".format(format_)) @@ -65,10 +64,6 @@ class DatabaseView(View): metadata = (datasette.metadata("databases") or {}).get(database, {}) datasette.update_with_inherited_metadata(metadata) - table_counts = await db.table_counts(5) - hidden_table_names = set(await db.hidden_table_names()) - all_foreign_keys = await db.get_all_foreign_keys() - sql_views = [] for view_name in await db.view_names(): view_visible, view_private = await datasette.check_visibility( @@ -196,8 +191,13 @@ class QueryContext: # urls: dict = field( # metadata={"help": "Object containing URL helpers like `database()`"} # ) - canned_write: bool = field( - metadata={"help": "Boolean indicating if this canned query allows writes"} + canned_query_write: bool = field( + metadata={ + "help": "Boolean indicating if this is a canned query that allows writes" + } + ) + metadata: dict = field( + metadata={"help": "Metadata about the database or the canned query"} ) db_is_immutable: bool = field( metadata={"help": "Boolean indicating if this database is immutable"} @@ -232,7 +232,6 @@ class QueryContext: show_hide_hidden: str = field( metadata={"help": "Hidden input field for the _show_sql parameter"} ) - metadata: dict = field(metadata={"help": "Metadata about the query/database"}) database_color: Callable = field( metadata={"help": "Function that returns a color for a given database name"} ) @@ -242,6 +241,12 @@ class QueryContext: alternate_url_json: str = field( metadata={"help": "URL for alternate JSON version of this page"} ) + # TODO: refactor this to somewhere else, probably ds.render_template() + select_templates: list = field( + metadata={ + "help": "List of templates that were considered for rendering this page" + } + ) async def get_tables(datasette, request, db): @@ -320,287 +325,105 @@ async def database_download(request, datasette): ) -async def query_view( - request, - datasette, - # canned_query=None, - # _size=None, - # named_parameters=None, - # write=False, -): - db = await datasette.resolve_database(request) - database = db.name - # Flattened because of ?sql=&name1=value1&name2=value2 feature - params = {key: request.args.get(key) for key in request.args} - sql = None - if "sql" in params: - sql = params.pop("sql") - if "_shape" in params: - params.pop("_shape") +class QueryView(View): + async def post(self, request, datasette): + from datasette.app import TableNotFound - # extras come from original request.args to avoid being flattened - extras = request.args.getlist("_extra") + db = await datasette.resolve_database(request) - # TODO: Behave differently for canned query here: - await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) - - _, private = await datasette.check_visibility( - request.actor, - permissions=[ - ("view-database", database), - "view-instance", - ], - ) - - extra_args = {} - if params.get("_timelimit"): - extra_args["custom_time_limit"] = int(params["_timelimit"]) - - format_ = request.url_vars.get("format") or "html" - query_error = None - try: - validate_sql_select(sql) - results = await datasette.execute( - database, sql, params, truncate=True, **extra_args - ) - columns = results.columns - rows = results.rows - except QueryInterrupted as ex: - raise DatasetteError( - textwrap.dedent( - """ -

SQL query took too long. The time limit is controlled by the - sql_time_limit_ms - configuration option.

- - - """.format( - markupsafe.escape(ex.sql) - ) - ).strip(), - title="SQL Interrupted", - status=400, - message_is_html=True, - ) - except sqlite3.DatabaseError as ex: - query_error = str(ex) - results = None - rows = [] - columns = [] - except (sqlite3.OperationalError, InvalidSql) as ex: - raise DatasetteError(str(ex), title="Invalid SQL", status=400) - except sqlite3.OperationalError as ex: - raise DatasetteError(str(ex)) - except DatasetteError: - raise - - # Handle formats from plugins - if format_ == "csv": - - async def fetch_data_for_csv(request, _next=None): - results = await db.execute(sql, params, truncate=True) - data = {"rows": results.rows, "columns": results.columns} - return data, None, None - - return await stream_csv(datasette, fetch_data_for_csv, request, db.name) - elif format_ in datasette.renderers.keys(): - # Dispatch request to the correct output format renderer - # (CSV is not handled here due to streaming) - result = call_with_supported_arguments( - datasette.renderers[format_][0], - datasette=datasette, - columns=columns, - rows=rows, - sql=sql, - query_name=None, - database=database, - table=None, - request=request, - view_name="table", - truncated=results.truncated if results else False, - error=query_error, - # These will be deprecated in Datasette 1.0: - args=request.args, - data={"rows": rows, "columns": columns}, - ) - if asyncio.iscoroutine(result): - result = await result - if result is None: - raise NotFound("No data") - if isinstance(result, dict): - r = Response( - body=result.get("body"), - status=result.get("status_code") or 200, - content_type=result.get("content_type", "text/plain"), - headers=result.get("headers"), + # We must be a canned query + table_found = False + try: + await datasette.resolve_table(request) + table_found = True + except TableNotFound as table_not_found: + canned_query = await datasette.get_canned_query( + table_not_found.database_name, table_not_found.table, request.actor ) - elif isinstance(result, Response): - r = result - # if status_code is not None: - # # Over-ride the status code - # r.status = status_code - else: - assert False, f"{result} should be dict or Response" - elif format_ == "html": - headers = {} - templates = [f"query-{to_css_class(database)}.html", "query.html"] - template = datasette.jinja_env.select_template(templates) - alternate_url_json = datasette.absolute_url( - request, - datasette.urls.path(path_with_format(request=request, format="json")), - ) - data = {} - headers.update( - { - "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( - alternate_url_json - ) - } - ) - metadata = (datasette.metadata("databases") or {}).get(database, {}) - datasette.update_with_inherited_metadata(metadata) + if canned_query is None: + raise + if table_found: + # That should not have happened + raise DatasetteError("Unexpected table found on POST", status=404) - renderers = {} - for key, (_, can_render) in datasette.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=datasette, - columns=data.get("columns") or [], - rows=data.get("rows") or [], - sql=data.get("query", {}).get("sql", None), - query_name=data.get("query_name"), - database=database, - table=data.get("table"), - request=request, - view_name="database", + # If database is immutable, return an error + if not db.is_mutable: + raise Forbidden("Database is immutable") + + # Process the POST + body = await request.post_body() + body = body.decode("utf-8").strip() + if body.startswith("{") and body.endswith("}"): + params = json.loads(body) + # But we want key=value strings + for key, value in params.items(): + params[key] = str(value) + else: + params = dict(parse_qsl(body, keep_blank_values=True)) + # Should we return JSON? + should_return_json = ( + request.headers.get("accept") == "application/json" + or request.args.get("_json") + or params.get("_json") + ) + params_for_query = MagicParameters(params, request, datasette) + ok = None + redirect_url = None + try: + cursor = await db.execute_write(canned_query["sql"], params_for_query) + message = canned_query.get( + "on_success_message" + ) or "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) + message_type = datasette.INFO + redirect_url = canned_query.get("on_success_redirect") + ok = True + except Exception as ex: + message = canned_query.get("on_error_message") or str(ex) + message_type = datasette.ERROR + redirect_url = canned_query.get("on_error_redirect") + ok = False + if should_return_json: + return Response.json( + { + "ok": ok, + "message": message, + "redirect": redirect_url, + } ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = datasette.urls.path( - path_with_format(request=request, format=key) - ) - - allow_execute_sql = await datasette.permission_allowed( - request.actor, "execute-sql", database - ) - - show_hide_hidden = "" - if metadata.get("hide_sql"): - if bool(params.get("_show_sql")): - show_hide_link = path_with_removed_args(request, {"_show_sql"}) - show_hide_text = "hide" - show_hide_hidden = '' - else: - show_hide_link = path_with_added_args(request, {"_show_sql": 1}) - show_hide_text = "show" else: - if bool(params.get("_hide_sql")): - show_hide_link = path_with_removed_args(request, {"_hide_sql"}) - show_hide_text = "show" - show_hide_hidden = '' - else: - show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) - show_hide_text = "hide" - hide_sql = show_hide_text == "show" + datasette.add_message(request, message, message_type) + return Response.redirect(redirect_url or request.path) - # Extract any :named parameters - named_parameters = await derive_named_parameters( - datasette.get_database(database), sql - ) - named_parameter_values = { - named_parameter: params.get(named_parameter) or "" - for named_parameter in named_parameters - if not named_parameter.startswith("_") - } + async def get(self, request, datasette): + from datasette.app import TableNotFound - # Set to blank string if missing from params - for named_parameter in named_parameters: - if named_parameter not in params and not named_parameter.startswith("_"): - params[named_parameter] = "" - - r = Response.html( - await datasette.render_template( - template, - QueryContext( - database=database, - query={ - "sql": sql, - "params": params, - }, - canned_query=None, - private=private, - canned_write=False, - db_is_immutable=not db.is_mutable, - error=query_error, - hide_sql=hide_sql, - show_hide_link=datasette.urls.path(show_hide_link), - show_hide_text=show_hide_text, - editable=True, # TODO - allow_execute_sql=allow_execute_sql, - tables=await get_tables(datasette, request, db), - named_parameter_values=named_parameter_values, - edit_sql_url="todo", - display_rows=await display_rows( - datasette, database, request, rows, columns - ), - table_columns=await _table_columns(datasette, database) - if allow_execute_sql - else {}, - columns=columns, - renderers=renderers, - url_csv=datasette.urls.path( - path_with_format( - request=request, format="csv", extra_qs={"_size": "max"} - ) - ), - show_hide_hidden=markupsafe.Markup(show_hide_hidden), - metadata=metadata, - database_color=lambda _: "#ff0000", - alternate_url_json=alternate_url_json, - ), - request=request, - view_name="database", - ), - headers=headers, - ) - else: - assert False, "Invalid format: {}".format(format_) - if datasette.cors: - add_cors_headers(r.headers) - return r - - -class QueryView(DataView): - async def data( - self, - request, - sql, - editable=True, - canned_query=None, - metadata=None, - _size=None, - named_parameters=None, - write=False, - default_labels=None, - ): - db = await self.ds.resolve_database(request) + db = await datasette.resolve_database(request) database = db.name - params = {key: request.args.get(key) for key in request.args} - if "sql" in params: - params.pop("sql") - if "_shape" in params: - params.pop("_shape") + + # Are we a canned query? + canned_query = None + canned_query_write = False + if "table" in request.url_vars: + try: + await datasette.resolve_table(request) + except TableNotFound as table_not_found: + # Was this actually a canned query? + canned_query = await datasette.get_canned_query( + table_not_found.database_name, table_not_found.table, request.actor + ) + if canned_query is None: + raise + canned_query_write = bool(canned_query.get("write")) private = False if canned_query: # Respect canned query permissions - visible, private = await self.ds.check_visibility( + visible, private = await datasette.check_visibility( request.actor, permissions=[ - ("view-query", (database, canned_query)), + ("view-query", (database, canned_query["name"])), ("view-database", database), "view-instance", ], @@ -609,18 +432,32 @@ class QueryView(DataView): raise Forbidden("You do not have permission to view this query") else: - await self.ds.ensure_permissions(request.actor, [("execute-sql", database)]) + await datasette.ensure_permissions( + request.actor, [("execute-sql", database)] + ) + + # Flattened because of ?sql=&name1=value1&name2=value2 feature + params = {key: request.args.get(key) for key in request.args} + sql = None + + if canned_query: + sql = canned_query["sql"] + elif "sql" in params: + sql = params.pop("sql") # Extract any :named parameters - named_parameters = named_parameters or await derive_named_parameters( - self.ds.get_database(database), sql - ) + named_parameters = [] + if canned_query and canned_query.get("params"): + named_parameters = canned_query["params"] + if not named_parameters: + named_parameters = await derive_named_parameters( + datasette.get_database(database), sql + ) named_parameter_values = { named_parameter: params.get(named_parameter) or "" for named_parameter in named_parameters if not named_parameter.startswith("_") } - # Set to blank string if missing from params for named_parameter in named_parameters: if named_parameter not in params and not named_parameter.startswith("_"): @@ -629,212 +466,159 @@ class QueryView(DataView): extra_args = {} if params.get("_timelimit"): extra_args["custom_time_limit"] = int(params["_timelimit"]) - if _size: - extra_args["page_size"] = _size - templates = [f"query-{to_css_class(database)}.html", "query.html"] - if canned_query: - templates.insert( - 0, - f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", - ) + format_ = request.url_vars.get("format") or "html" query_error = None + results = None + rows = [] + columns = [] - # Execute query - as write or as read - if write: - if request.method == "POST": - # If database is immutable, return an error - if not db.is_mutable: - raise Forbidden("Database is immutable") - body = await request.post_body() - body = body.decode("utf-8").strip() - if body.startswith("{") and body.endswith("}"): - params = json.loads(body) - # But we want key=value strings - for key, value in params.items(): - params[key] = str(value) - else: - params = dict(parse_qsl(body, keep_blank_values=True)) - # Should we return JSON? - should_return_json = ( - request.headers.get("accept") == "application/json" - or request.args.get("_json") - or params.get("_json") - ) - if canned_query: - params_for_query = MagicParameters(params, request, self.ds) - else: - params_for_query = params - ok = None - try: - cursor = await self.ds.databases[database].execute_write( - sql, params_for_query - ) - message = metadata.get( - "on_success_message" - ) or "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" - ) - message_type = self.ds.INFO - redirect_url = metadata.get("on_success_redirect") - ok = True - except Exception as e: - message = metadata.get("on_error_message") or str(e) - message_type = self.ds.ERROR - redirect_url = metadata.get("on_error_redirect") - ok = False - if should_return_json: - return Response.json( - { - "ok": ok, - "message": message, - "redirect": redirect_url, - } - ) - else: - self.ds.add_message(request, message, message_type) - return self.redirect(request, redirect_url or request.path) - else: + params_for_query = params - async def extra_template(): - return { - "request": request, - "db_is_immutable": not db.is_mutable, - "path_with_added_args": path_with_added_args, - "path_with_removed_args": path_with_removed_args, - "named_parameter_values": named_parameter_values, - "canned_query": canned_query, - "success_message": request.args.get("_success") or "", - "canned_write": True, - } - - return ( - { - "database": database, - "rows": [], - "truncated": False, - "columns": [], - "query": {"sql": sql, "params": params}, - "private": private, - }, - extra_template, - templates, - ) - else: # Not a write - if canned_query: - params_for_query = MagicParameters(params, request, self.ds) - else: - params_for_query = params + if not canned_query_write: try: - results = await self.ds.execute( + if not canned_query: + # For regular queries we only allow SELECT, plus other rules + validate_sql_select(sql) + else: + # Canned queries can run magic parameters + params_for_query = MagicParameters(params, request, datasette) + results = await datasette.execute( database, sql, params_for_query, truncate=True, **extra_args ) - columns = [r[0] for r in results.description] - except sqlite3.DatabaseError as e: - query_error = e + columns = results.columns + rows = results.rows + except QueryInterrupted as ex: + raise DatasetteError( + textwrap.dedent( + """ +

SQL query took too long. The time limit is controlled by the + sql_time_limit_ms + configuration option.

+ + + """.format( + markupsafe.escape(ex.sql) + ) + ).strip(), + title="SQL Interrupted", + status=400, + message_is_html=True, + ) + except sqlite3.DatabaseError as ex: + query_error = str(ex) results = None + rows = [] columns = [] + except (sqlite3.OperationalError, InvalidSql) as ex: + raise DatasetteError(str(ex), title="Invalid SQL", status=400) + except sqlite3.OperationalError as ex: + raise DatasetteError(str(ex)) + except DatasetteError: + raise - allow_execute_sql = await self.ds.permission_allowed( - request.actor, "execute-sql", database - ) + # Handle formats from plugins + if format_ == "csv": - async def extra_template(): - display_rows = [] - truncate_cells = self.ds.setting("truncate_cells_html") - for row in results.rows if results else []: - display_row = [] - for column, value in zip(results.columns, row): - display_value = value - # Let the plugins have a go - # pylint: disable=no-member - plugin_display_value = None - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=None, - database=database, - datasette=self.ds, - request=request, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value is not None: - display_value = plugin_display_value - else: - if value in ("", None): - display_value = markupsafe.Markup(" ") - elif is_url(str(display_value).strip()): - display_value = markupsafe.Markup( - '{truncated_url}'.format( - url=markupsafe.escape(value.strip()), - truncated_url=markupsafe.escape( - truncate_url(value.strip(), truncate_cells) - ), - ) - ) - elif isinstance(display_value, bytes): - blob_url = path_with_format( - request=request, - format="blob", - extra_qs={ - "_blob_column": column, - "_blob_hash": hashlib.sha256( - display_value - ).hexdigest(), - }, - ) - formatted = format_bytes(len(value)) - display_value = markupsafe.Markup( - '<Binary: {:,} byte{}>'.format( - blob_url, - ' title="{}"'.format(formatted) - if "bytes" not in formatted - else "", - len(value), - "" if len(value) == 1 else "s", - ) - ) - else: - display_value = str(value) - if truncate_cells and len(display_value) > truncate_cells: - display_value = ( - display_value[:truncate_cells] + "\u2026" - ) - display_row.append(display_value) - display_rows.append(display_row) + async def fetch_data_for_csv(request, _next=None): + results = await db.execute(sql, params, truncate=True) + data = {"rows": results.rows, "columns": results.columns} + return data, None, None - # Show 'Edit SQL' button only if: - # - User is allowed to execute SQL - # - SQL is an approved SELECT statement - # - No magic parameters, so no :_ in the SQL string - edit_sql_url = None - is_validated_sql = False - try: - validate_sql_select(sql) - is_validated_sql = True - except InvalidSql: - pass - if allow_execute_sql and is_validated_sql and ":_" not in sql: - edit_sql_url = ( - self.ds.urls.database(database) - + "?" - + urlencode( - { - **{ - "sql": sql, - }, - **named_parameter_values, - } - ) + return await stream_csv(datasette, fetch_data_for_csv, request, db.name) + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=canned_query["name"] if canned_query else None, + database=database, + table=None, + request=request, + view_name="table", + truncated=results.truncated if results else False, + error=query_error, + # These will be deprecated in Datasette 1.0: + args=request.args, + data={"rows": rows, "columns": columns}, + ) + if asyncio.iscoroutine(result): + result = await result + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(database)}.html", "query.html"] + if canned_query: + templates.insert( + 0, + f"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html", ) + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + data = {} + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + metadata = (datasette.metadata("databases") or {}).get(database, {}) + datasette.update_with_inherited_metadata(metadata) + + renderers = {} + for key, (_, can_render) in datasette.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=datasette, + columns=data.get("columns") or [], + rows=data.get("rows") or [], + sql=data.get("query", {}).get("sql", None), + query_name=data.get("query_name"), + database=database, + table=data.get("table"), + request=request, + view_name="database", + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = datasette.urls.path( + path_with_format(request=request, format=key) + ) + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + show_hide_hidden = "" - if metadata.get("hide_sql"): + if canned_query and canned_query.get("hide_sql"): if bool(params.get("_show_sql")): show_hide_link = path_with_removed_args(request, {"_show_sql"}) show_hide_text = "hide" @@ -855,42 +639,86 @@ class QueryView(DataView): show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) show_hide_text = "hide" hide_sql = show_hide_text == "show" - return { - "display_rows": display_rows, - "custom_sql": True, - "named_parameter_values": named_parameter_values, - "editable": editable, - "canned_query": canned_query, - "edit_sql_url": edit_sql_url, - "metadata": metadata, - "settings": self.ds.settings_dict(), - "request": request, - "show_hide_link": self.ds.urls.path(show_hide_link), - "show_hide_text": show_hide_text, - "show_hide_hidden": markupsafe.Markup(show_hide_hidden), - "hide_sql": hide_sql, - "table_columns": await _table_columns(self.ds, database) - if allow_execute_sql - else {}, - } - return ( - { - "ok": not query_error, - "database": database, - "query_name": canned_query, - "rows": results.rows if results else [], - "truncated": results.truncated if results else False, - "columns": columns, - "query": {"sql": sql, "params": params}, - "error": str(query_error) if query_error else None, - "private": private, - "allow_execute_sql": allow_execute_sql, - }, - extra_template, - templates, - 400 if query_error else 200, - ) + # Show 'Edit SQL' button only if: + # - User is allowed to execute SQL + # - SQL is an approved SELECT statement + # - No magic parameters, so no :_ in the SQL string + edit_sql_url = None + is_validated_sql = False + try: + validate_sql_select(sql) + is_validated_sql = True + except InvalidSql: + pass + if allow_execute_sql and is_validated_sql and ":_" not in sql: + edit_sql_url = ( + datasette.urls.database(database) + + "?" + + urlencode( + { + **{ + "sql": sql, + }, + **named_parameter_values, + } + ) + ) + + r = Response.html( + await datasette.render_template( + template, + QueryContext( + database=database, + query={ + "sql": sql, + "params": params, + }, + canned_query=canned_query["name"] if canned_query else None, + private=private, + canned_query_write=canned_query_write, + db_is_immutable=not db.is_mutable, + error=query_error, + hide_sql=hide_sql, + show_hide_link=datasette.urls.path(show_hide_link), + show_hide_text=show_hide_text, + editable=not canned_query, + allow_execute_sql=allow_execute_sql, + tables=await get_tables(datasette, request, db), + named_parameter_values=named_parameter_values, + edit_sql_url=edit_sql_url, + display_rows=await display_rows( + datasette, database, request, rows, columns + ), + table_columns=await _table_columns(datasette, database) + if allow_execute_sql + else {}, + columns=columns, + renderers=renderers, + url_csv=datasette.urls.path( + path_with_format( + request=request, format="csv", extra_qs={"_size": "max"} + ) + ), + show_hide_hidden=markupsafe.Markup(show_hide_hidden), + metadata=canned_query or metadata, + database_color=lambda _: "#ff0000", + alternate_url_json=alternate_url_json, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="database", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + if datasette.cors: + add_cors_headers(r.headers) + return r class MagicParameters(dict): diff --git a/datasette/views/table.py b/datasette/views/table.py index 77acfd95..28264e92 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -9,7 +9,6 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette import tracer -from datasette.renderer import json_renderer from datasette.utils import ( add_cors_headers, await_me_maybe, @@ -21,7 +20,6 @@ from datasette.utils import ( tilde_encode, escape_sqlite, filters_should_redirect, - format_bytes, is_url, path_from_row_pks, path_with_added_args, @@ -38,7 +36,7 @@ from datasette.utils import ( from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response from datasette.filters import Filters import sqlite_utils -from .base import BaseView, DataView, DatasetteError, ureg, _error, stream_csv +from .base import BaseView, DatasetteError, ureg, _error, stream_csv from .database import QueryView LINK_WITH_LABEL = ( @@ -698,57 +696,6 @@ async def table_view(datasette, request): return response -class CannedQueryView(DataView): - def __init__(self, datasette): - self.ds = datasette - - async def post(self, request): - from datasette.app import TableNotFound - - try: - await self.ds.resolve_table(request) - except TableNotFound as e: - # Was this actually a canned query? - canned_query = await self.ds.get_canned_query( - e.database_name, e.table, request.actor - ) - if canned_query: - # Handle POST to a canned query - return await QueryView(self.ds).data( - request, - canned_query["sql"], - metadata=canned_query, - editable=False, - canned_query=e.table, - named_parameters=canned_query.get("params"), - write=bool(canned_query.get("write")), - ) - - return Response.text("Method not allowed", status=405) - - async def data(self, request, **kwargs): - from datasette.app import TableNotFound - - try: - await self.ds.resolve_table(request) - except TableNotFound as not_found: - canned_query = await self.ds.get_canned_query( - not_found.database_name, not_found.table, request.actor - ) - if canned_query: - return await QueryView(self.ds).data( - request, - canned_query["sql"], - metadata=canned_query, - editable=False, - canned_query=not_found.table, - named_parameters=canned_query.get("params"), - write=bool(canned_query.get("write")), - ) - else: - raise - - async def table_view_traced(datasette, request): from datasette.app import TableNotFound @@ -761,10 +708,7 @@ async def table_view_traced(datasette, request): ) # If this is a canned query, not a table, then dispatch to QueryView instead if canned_query: - if request.method == "POST": - return await CannedQueryView(datasette).post(request) - else: - return await CannedQueryView(datasette).get(request) + return await QueryView()(request, datasette) else: raise diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index d6a88733..e9ad3239 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -95,12 +95,12 @@ def test_insert(canned_write_client): csrftoken_from=True, cookies={"foo": "bar"}, ) - assert 302 == response.status - assert "/data/add_name?success" == response.headers["Location"] messages = canned_write_client.ds.unsign( response.cookies["ds_messages"], "messages" ) - assert [["Query executed, 1 row affected", 1]] == messages + assert messages == [["Query executed, 1 row affected", 1]] + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name?success" @pytest.mark.parametrize( @@ -382,11 +382,11 @@ def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_c def test_canned_write_custom_template(canned_write_client): response = canned_write_client.get("/data/update_name") assert response.status == 200 + assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text assert ( "" in response.text ) - assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text # And test for link rel=alternate while we're here: assert ( '' From 8920d425f4d417cfd998b61016c5ff3530cd34e1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 10:20:58 -0700 Subject: [PATCH 047/767] 1.0a3 release notes, smaller changes section - refs #2135 --- docs/changelog.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ee48d075..b4416f94 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,25 @@ Changelog ========= +.. _v1_0_a3: + +1.0a3 (2023-08-09) +------------------ + +This alpha release previews the updated design for Datasette's default JSON API. + +Smaller changes +~~~~~~~~~~~~~~~ + +- Datasette documentation now shows YAML examples for :ref:`metadata` by default, with a tab interface for switching to JSON. (:issue:`1153`) +- :ref:`plugin_register_output_renderer` plugins now have access to ``error`` and ``truncated`` arguments, allowing them to display error messages and take into account truncated results. (:issue:`2130`) +- ``render_cell()`` plugin hook now also supports an optional ``request`` argument. (:issue:`2007`) +- New ``Justfile`` to support development workflows for Datasette using `Just `__. +- ``datasette.render_template()`` can now accepts a ``datasette.views.Context`` subclass as an alternative to a dictionary. (:issue:`2127`) +- ``datasette install -e path`` option for editable installations, useful while developing plugins. (:issue:`2106`) +- When started with the ``--cors`` option Datasette now serves an ``Access-Control-Max-Age: 3600`` header, ensuring CORS OPTIONS requests are repeated no more than once an hour. (:issue:`2079`) +- Fixed a bug where the ``_internal`` database could display ``None`` instead of ``null`` for in-memory databases. (:issue:`1970`) + .. _v0_64_2: 0.64.2 (2023-03-08) From e34d09c6ec16ff5e7717e112afdad67f7c05a62a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:01:59 -0700 Subject: [PATCH 048/767] Don't include columns in query JSON, refs #2136 --- datasette/renderer.py | 8 +++++++- datasette/views/database.py | 2 +- tests/test_api.py | 1 - tests/test_cli_serve_get.py | 11 ++++++----- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/datasette/renderer.py b/datasette/renderer.py index 0bd74e81..224031a7 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -27,7 +27,7 @@ def convert_specific_columns_to_json(rows, columns, json_cols): return new_rows -def json_renderer(args, data, error, truncated=None): +def json_renderer(request, args, data, error, truncated=None): """Render a response as JSON""" status_code = 200 @@ -106,6 +106,12 @@ def json_renderer(args, data, error, truncated=None): "status": 400, "title": None, } + + # Don't include "columns" in output + # https://github.com/simonw/datasette/issues/2136 + if isinstance(data, dict) and "columns" not in request.args.getlist("_extra"): + data.pop("columns", None) + # Handle _nl option for _shape=array nl = args.get("_nl", "") if nl and shape == "array": diff --git a/datasette/views/database.py b/datasette/views/database.py index 658c35e6..cf76f3c2 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -548,7 +548,7 @@ class QueryView(View): error=query_error, # These will be deprecated in Datasette 1.0: args=request.args, - data={"rows": rows, "columns": columns}, + data={"ok": True, "rows": rows, "columns": columns}, ) if asyncio.iscoroutine(result): result = await result diff --git a/tests/test_api.py b/tests/test_api.py index 28415a0b..f96f571e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -649,7 +649,6 @@ async def test_custom_sql(ds_client): {"content": "RENDER_CELL_DEMO"}, {"content": "RENDER_CELL_ASYNC"}, ], - "columns": ["content"], "ok": True, "truncated": False, } diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index 2e0390bb..dc7fc1e2 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -34,11 +34,12 @@ def test_serve_with_get(tmp_path_factory): "/_memory.json?sql=select+sqlite_version()", ], ) - assert 0 == result.exit_code, result.output - assert { - "truncated": False, - "columns": ["sqlite_version()"], - }.items() <= json.loads(result.output).items() + assert result.exit_code == 0, result.output + data = json.loads(result.output) + # Should have a single row with a single column + assert len(data["rows"]) == 1 + assert list(data["rows"][0].keys()) == ["sqlite_version()"] + assert set(data.keys()) == {"rows", "ok", "truncated"} # The plugin should have created hello.txt assert (plugins_dir / "hello.txt").read_text() == "hello" From 856ca68d94708c6e94673cb6bc28bf3e3ca17845 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:04:40 -0700 Subject: [PATCH 049/767] Update default JSON representation docs, refs #2135 --- docs/json_api.rst | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/json_api.rst b/docs/json_api.rst index c273c2a8..16b997eb 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -9,10 +9,10 @@ through the Datasette user interface can also be accessed as JSON via the API. To access the API for a page, either click on the ``.json`` link on that page or edit the URL and add a ``.json`` extension to it. -.. _json_api_shapes: +.. _json_api_default: -Different shapes ----------------- +Default representation +---------------------- The default JSON representation of data from a SQLite table or custom query looks like this: @@ -21,7 +21,6 @@ looks like this: { "ok": true, - "next": null, "rows": [ { "id": 3, @@ -39,13 +38,22 @@ looks like this: "id": 1, "name": "San Francisco" } - ] + ], + "truncated": false } -The ``rows`` key is a list of objects, each one representing a row. ``next`` indicates if -there is another page, and ``ok`` is always ``true`` if an error did not occur. +``"ok"`` is always ``true`` if an error did not occur. -If ``next`` is present then the next page in the pagination set can be retrieved using ``?_next=VALUE``. +The ``"rows"`` key is a list of objects, each one representing a row. + +The ``"truncated"`` key lets you know if the query was truncated. This can happen if a SQL query returns more than 1,000 results (or the :ref:`setting_max_returned_rows` setting). + +For table pages, an additional key ``"next"`` may be present. This indicates that the next page in the pagination set can be retrieved using ``?_next=VALUE``. + +.. _json_api_shapes: + +Different shapes +---------------- The ``_shape`` parameter can be used to access alternative formats for the ``rows`` key which may be more convenient for your application. There are three From 90cb9ca58d910f49e8f117bbdd94df6f0855cf99 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:11:16 -0700 Subject: [PATCH 050/767] JSON changes in release notes, refs #2135 --- docs/changelog.rst | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b4416f94..4c70855b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,40 @@ Changelog 1.0a3 (2023-08-09) ------------------ -This alpha release previews the updated design for Datasette's default JSON API. +This alpha release previews the updated design for Datasette's default JSON API. (:issue:`782`) + +The new :ref:`default JSON representation ` for both table pages (``/dbname/table.json``) and arbitrary SQL queries (``/dbname.json?sql=...``) is now shaped like this: + +.. code-block:: json + + { + "ok": true, + "rows": [ + { + "id": 3, + "name": "Detroit" + }, + { + "id": 2, + "name": "Los Angeles" + }, + { + "id": 4, + "name": "Memnonia" + }, + { + "id": 1, + "name": "San Francisco" + } + ], + "truncated": false + } + +Tables will include an additional ``"next"`` key for pagination, which can be passed to ``?_next=`` to fetch the next page of results. + +The various ``?_shape=`` options continue to work as before - see :ref:`json_api_shapes` for details. + +A new ``?_extra=`` mechanism is available for tables, but has not yet been stabilized or documented. Details on that are available in :issue:`262`. Smaller changes ~~~~~~~~~~~~~~~ From 19ab4552e212c9845a59461cc73e82d5ae8c278a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 12:13:11 -0700 Subject: [PATCH 051/767] Release 1.0a3 Closes #2135 Refs #262, #782, #1153, #1970, #2007, #2079, #2106, #2127, #2130 --- datasette/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 3b81ab21..61dee464 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a2" +__version__ = "1.0a3" __version_info__ = tuple(__version__.split(".")) From 4a42476bb7ce4c5ed941f944115dedd9bce34656 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 15:04:16 -0700 Subject: [PATCH 052/767] datasette plugins --requirements, closes #2133 --- datasette/cli.py | 12 ++++++++++-- docs/cli-reference.rst | 1 + docs/plugins.rst | 32 ++++++++++++++++++++++++++++---- tests/test_cli.py | 3 +++ 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 32266888..21fd25d6 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -223,15 +223,23 @@ pm.hook.publish_subcommand(publish=publish) @cli.command() @click.option("--all", help="Include built-in default plugins", is_flag=True) +@click.option( + "--requirements", help="Output requirements.txt of installed plugins", is_flag=True +) @click.option( "--plugins-dir", type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Path to directory containing custom plugins", ) -def plugins(all, plugins_dir): +def plugins(all, requirements, plugins_dir): """List currently installed plugins""" app = Datasette([], plugins_dir=plugins_dir) - click.echo(json.dumps(app._plugins(all=all), indent=4)) + if requirements: + for plugin in app._plugins(): + if plugin["version"]: + click.echo("{}=={}".format(plugin["name"], plugin["version"])) + else: + click.echo(json.dumps(app._plugins(all=all), indent=4)) @cli.command() diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 2177fc9e..7a96d311 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -282,6 +282,7 @@ Output JSON showing all currently installed plugins, their versions, whether the Options: --all Include built-in default plugins + --requirements Output requirements.txt of installed plugins --plugins-dir DIRECTORY Path to directory containing custom plugins --help Show this message and exit. diff --git a/docs/plugins.rst b/docs/plugins.rst index 979f94dd..19bfdd0c 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -90,7 +90,12 @@ You can see a list of installed plugins by navigating to the ``/-/plugins`` page You can also use the ``datasette plugins`` command:: - $ datasette plugins + datasette plugins + +Which outputs: + +.. code-block:: json + [ { "name": "datasette_json_html", @@ -107,7 +112,8 @@ You can also use the ``datasette plugins`` command:: cog.out("\n") result = CliRunner().invoke(cli.cli, ["plugins", "--all"]) # cog.out() with text containing newlines was unindenting for some reason - cog.outl("If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette::\n") + cog.outl("If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette:\n") + cog.outl(".. code-block:: json\n") plugins = [p for p in json.loads(result.output) if p["name"].startswith("datasette.")] indented = textwrap.indent(json.dumps(plugins, indent=4), " ") for line in indented.split("\n"): @@ -115,7 +121,9 @@ You can also use the ``datasette plugins`` command:: cog.out("\n\n") .. ]]] -If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette:: +If you run ``datasette plugins --all`` it will include default plugins that ship as part of Datasette: + +.. code-block:: json [ { @@ -236,6 +244,22 @@ If you run ``datasette plugins --all`` it will include default plugins that ship You can add the ``--plugins-dir=`` option to include any plugins found in that directory. +Add ``--requirements`` to output a list of installed plugins that can then be installed in another Datasette instance using ``datasette install -r requirements.txt``:: + + datasette plugins --requirements + +The output will look something like this:: + + datasette-codespaces==0.1.1 + datasette-graphql==2.2 + datasette-json-html==1.0.1 + datasette-pretty-json==0.2.2 + datasette-x-forwarded-host==0.1 + +To write that to a ``requirements.txt`` file, run this:: + + datasette plugins --requirements > requirements.txt + .. _plugins_configuration: Plugin configuration @@ -390,7 +414,7 @@ Any values embedded in ``metadata.yaml`` will be visible to anyone who views the If you are publishing your data using the :ref:`datasette publish ` family of commands, you can use the ``--plugin-secret`` option to set these secrets at publish time. For example, using Heroku you might run the following command:: - $ datasette publish heroku my_database.db \ + datasette publish heroku my_database.db \ --name my-heroku-app-demo \ --install=datasette-auth-github \ --plugin-secret datasette-auth-github client_id your_client_id \ diff --git a/tests/test_cli.py b/tests/test_cli.py index 75724f61..056e2821 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -108,6 +108,9 @@ def test_plugins_cli(app_client): assert set(names).issuperset({p["name"] for p in EXPECTED_PLUGINS}) # And the following too: assert set(names).issuperset(DEFAULT_PLUGINS) + # --requirements should be empty because there are no installed non-plugins-dir plugins + result3 = runner.invoke(cli, ["plugins", "--requirements"]) + assert result3.output == "" def test_metadata_yaml(): From a3593c901580ea50854c3e0774b0ba0126e8a76f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 17:32:07 -0700 Subject: [PATCH 053/767] on_success_message_sql, closes #2138 --- datasette/views/database.py | 29 ++++++++++++++++---- docs/sql_queries.rst | 21 ++++++++++---- tests/test_canned_queries.py | 53 +++++++++++++++++++++++++++++++----- 3 files changed, 85 insertions(+), 18 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index cf76f3c2..79b3f88d 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -360,6 +360,10 @@ class QueryView(View): params[key] = str(value) else: params = dict(parse_qsl(body, keep_blank_values=True)) + + # Don't ever send csrftoken as a SQL parameter + params.pop("csrftoken", None) + # Should we return JSON? should_return_json = ( request.headers.get("accept") == "application/json" @@ -371,12 +375,27 @@ class QueryView(View): redirect_url = None try: cursor = await db.execute_write(canned_query["sql"], params_for_query) - message = canned_query.get( - "on_success_message" - ) or "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" - ) + # success message can come from on_success_message or on_success_message_sql + message = None message_type = datasette.INFO + on_success_message_sql = canned_query.get("on_success_message_sql") + if on_success_message_sql: + try: + message_result = ( + await db.execute(on_success_message_sql, params_for_query) + ).first() + if message_result: + message = message_result[0] + except Exception as ex: + message = "Error running on_success_message_sql: {}".format(ex) + message_type = datasette.ERROR + if not message: + message = canned_query.get( + "on_success_message" + ) or "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) + redirect_url = canned_query.get("on_success_redirect") ok = True except Exception as ex: diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 3c2cb228..1ae07e1f 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -392,6 +392,7 @@ This configuration will create a page at ``/mydatabase/add_name`` displaying a f You can customize how Datasette represents success and errors using the following optional properties: - ``on_success_message`` - the message shown when a query is successful +- ``on_success_message_sql`` - alternative to ``on_success_message``: a SQL query that should be executed to generate the message - ``on_success_redirect`` - the path or URL the user is redirected to on success - ``on_error_message`` - the message shown when a query throws an error - ``on_error_redirect`` - the path or URL the user is redirected to on error @@ -405,11 +406,12 @@ For example: "queries": { "add_name": { "sql": "INSERT INTO names (name) VALUES (:name)", + "params": ["name"], "write": True, - "on_success_message": "Name inserted", + "on_success_message_sql": "select 'Name inserted: ' || :name", "on_success_redirect": "/mydatabase/names", "on_error_message": "Name insert failed", - "on_error_redirect": "/mydatabase" + "on_error_redirect": "/mydatabase", } } } @@ -426,8 +428,10 @@ For example: queries: add_name: sql: INSERT INTO names (name) VALUES (:name) + params: + - name write: true - on_success_message: Name inserted + on_success_message_sql: 'select ''Name inserted: '' || :name' on_success_redirect: /mydatabase/names on_error_message: Name insert failed on_error_redirect: /mydatabase @@ -443,8 +447,11 @@ For example: "queries": { "add_name": { "sql": "INSERT INTO names (name) VALUES (:name)", + "params": [ + "name" + ], "write": true, - "on_success_message": "Name inserted", + "on_success_message_sql": "select 'Name inserted: ' || :name", "on_success_redirect": "/mydatabase/names", "on_error_message": "Name insert failed", "on_error_redirect": "/mydatabase" @@ -455,10 +462,12 @@ For example: } .. [[[end]]] -You can use ``"params"`` to explicitly list the named parameters that should be displayed as form fields - otherwise they will be automatically detected. +You can use ``"params"`` to explicitly list the named parameters that should be displayed as form fields - otherwise they will be automatically detected. ``"params"`` is not necessary in the above example, since without it ``"name"`` would be automatically detected from the query. You can pre-populate form fields when the page first loads using a query string, e.g. ``/mydatabase/add_name?name=Prepopulated``. The user will have to submit the form to execute the query. +If you specify a query in ``"on_success_message_sql"``, that query will be executed after the main query. The first column of the first row return by that query will be displayed as a success message. Named parameters from the main query will be made available to the success message query as well. + .. _canned_queries_magic_parameters: Magic parameters @@ -589,7 +598,7 @@ The JSON response will look like this: "redirect": "/data/add_name" } -The ``"message"`` and ``"redirect"`` values here will take into account ``on_success_message``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. +The ``"message"`` and ``"redirect"`` values here will take into account ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``, if they have been set. .. _pagination: diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index e9ad3239..5256c24c 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -31,9 +31,15 @@ def canned_write_client(tmpdir): }, "add_name_specify_id": { "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select 'Name added: ' || :name || ' with rowid ' || :rowid", "write": True, "on_error_redirect": "/data/add_name_specify_id?error", }, + "add_name_specify_id_with_error_in_on_success_message_sql": { + "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select this is bad SQL", + "write": True, + }, "delete_name": { "sql": "delete from names where rowid = :rowid", "write": True, @@ -179,6 +185,34 @@ def test_insert_error(canned_write_client): ) +def test_on_success_message_sql(canned_write_client): + response = canned_write_client.post( + "/data/add_name_specify_id", + {"rowid": 5, "name": "Should be OK"}, + csrftoken_from=True, + ) + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name_specify_id" + messages = canned_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [["Name added: Should be OK with rowid 5", 1]] + + +def test_error_in_on_success_message_sql(canned_write_client): + response = canned_write_client.post( + "/data/add_name_specify_id_with_error_in_on_success_message_sql", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + messages = canned_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [ + ["Error running on_success_message_sql: no such column: bad", 3] + ] + + def test_custom_params(canned_write_client): response = canned_write_client.get("/data/update_name?extra=foo") assert '' in response.text @@ -232,21 +266,22 @@ def test_canned_query_permissions_on_database_page(canned_write_client): query_names = { q["name"] for q in canned_write_client.get("/data.json").json["queries"] } - assert { + assert query_names == { + "add_name_specify_id_with_error_in_on_success_message_sql", + "from_hook", + "update_name", + "add_name_specify_id", + "from_async_hook", "canned_read", "add_name", - "add_name_specify_id", - "update_name", - "from_async_hook", - "from_hook", - } == query_names + } # With auth shows four response = canned_write_client.get( "/data.json", cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, ) - assert 200 == response.status + assert response.status == 200 query_names_and_private = sorted( [ {"name": q["name"], "private": q["private"]} @@ -257,6 +292,10 @@ def test_canned_query_permissions_on_database_page(canned_write_client): assert query_names_and_private == [ {"name": "add_name", "private": False}, {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, {"name": "canned_read", "private": False}, {"name": "delete_name", "private": True}, {"name": "from_async_hook", "private": False}, From 33251d04e78d575cca62bb59069bb43a7d924746 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 9 Aug 2023 17:56:27 -0700 Subject: [PATCH 054/767] Canned query write counters demo, refs #2134 --- .github/workflows/deploy-latest.yml | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index ed60376c..4746aa07 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -57,6 +57,36 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db + - name: And the counters writable canned query demo + run: | + cat > plugins/counters.py < Date: Thu, 10 Aug 2023 22:16:19 -0700 Subject: [PATCH 055/767] Fixed display of database color Closes #2139, closes #2119 --- datasette/database.py | 7 +++++++ datasette/templates/database.html | 2 +- datasette/templates/query.html | 2 +- datasette/templates/row.html | 2 +- datasette/templates/table.html | 2 +- datasette/views/base.py | 4 ---- datasette/views/database.py | 8 +++----- datasette/views/index.py | 4 +--- datasette/views/row.py | 4 +++- datasette/views/table.py | 2 +- tests/test_html.py | 20 ++++++++++++++++++++ 11 files changed, 39 insertions(+), 18 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index d8043c24..af39ac9e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,6 +1,7 @@ import asyncio from collections import namedtuple from pathlib import Path +import hashlib import janus import queue import sys @@ -62,6 +63,12 @@ class Database: } return self._cached_table_counts + @property + def color(self): + if self.hash: + return self.hash[:6] + return hashlib.md5(self.name.encode("utf8")).hexdigest()[:6] + def suggest_name(self): if self.path: return Path(self.path).stem diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 7acf0369..3d4dae07 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -10,7 +10,7 @@ {% block body_class %}db db-{{ database|to_css_class }}{% endblock %} {% block content %} -
+

{{ metadata.title or database }}{% if private %} 🔒{% endif %}

{% set links = database_actions() %}{% if links %}
diff --git a/datasette/templates/query.html b/datasette/templates/query.html index fc3b8527..b8f06f84 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -28,7 +28,7 @@

This query cannot be executed because the database is immutable.

{% endif %} -

{{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

+

{{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

{% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 1d1b0bfd..4d179a85 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -20,7 +20,7 @@ {% endblock %} {% block content %} -

{{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}

+

{{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}

{% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 0be1998c..88580e52 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -21,7 +21,7 @@ {% endblock %} {% block content %} -
+

{{ metadata.get("title") or table }}{% if is_view %} (view){% endif %}{% if private %} 🔒{% endif %}

{% set links = table_actions() %}{% if links %}
diff --git a/datasette/views/base.py b/datasette/views/base.py index da5c55ad..0080b33c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -102,9 +102,6 @@ class BaseView: response.body = b"" return response - def database_color(self, database): - return "ff0000" - async def method_not_allowed(self, request): if ( request.path.endswith(".json") @@ -150,7 +147,6 @@ class BaseView: template_context = { **context, **{ - "database_color": self.database_color, "select_templates": [ f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates diff --git a/datasette/views/database.py b/datasette/views/database.py index 79b3f88d..d9abc38a 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -146,6 +146,7 @@ class DatabaseView(View): template = datasette.jinja_env.select_template(templates) context = { **json_data, + "database_color": db.color, "database_actions": database_actions, "show_hidden": request.args.get("_show_hidden"), "editable": True, @@ -154,7 +155,6 @@ class DatabaseView(View): and not db.is_mutable and not db.is_memory, "attached_databases": attached_databases, - "database_color": lambda _: "#ff0000", "alternate_url_json": alternate_url_json, "select_templates": [ f"{'*' if template_name == template.name else ''}{template_name}" @@ -179,6 +179,7 @@ class DatabaseView(View): @dataclass class QueryContext: database: str = field(metadata={"help": "The name of the database being queried"}) + database_color: str = field(metadata={"help": "The color of the database"}) query: dict = field( metadata={"help": "The SQL query object containing the `sql` string"} ) @@ -232,9 +233,6 @@ class QueryContext: show_hide_hidden: str = field( metadata={"help": "Hidden input field for the _show_sql parameter"} ) - database_color: Callable = field( - metadata={"help": "Function that returns a color for a given database name"} - ) table_columns: dict = field( metadata={"help": "Dictionary of table name to list of column names"} ) @@ -689,6 +687,7 @@ class QueryView(View): template, QueryContext( database=database, + database_color=db.color, query={ "sql": sql, "params": params, @@ -721,7 +720,6 @@ class QueryView(View): ), show_hide_hidden=markupsafe.Markup(show_hide_hidden), metadata=canned_query or metadata, - database_color=lambda _: "#ff0000", alternate_url_json=alternate_url_json, select_templates=[ f"{'*' if template_name == template.name else ''}{template_name}" diff --git a/datasette/views/index.py b/datasette/views/index.py index df411c4a..95b29302 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -105,9 +105,7 @@ class IndexView(BaseView): { "name": name, "hash": db.hash, - "color": db.hash[:6] - if db.hash - else hashlib.md5(name.encode("utf8")).hexdigest()[:6], + "color": db.color, "path": self.ds.urls.database(name), "tables_and_views_truncated": tables_and_views_truncated, "tables_and_views_more": (len(visible_tables) + len(views)) diff --git a/datasette/views/row.py b/datasette/views/row.py index 6e09f30e..8f07a662 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -18,7 +18,8 @@ class RowView(DataView): async def data(self, request, default_labels=False): resolved = await self.ds.resolve_row(request) - database = resolved.db.name + db = resolved.db + database = db.name table = resolved.table pk_values = resolved.pk_values @@ -60,6 +61,7 @@ class RowView(DataView): "foreign_key_tables": await self.foreign_key_tables( database, table, pk_values ), + "database_color": db.color, "display_columns": display_columns, "display_rows": display_rows, "custom_table_templates": [ diff --git a/datasette/views/table.py b/datasette/views/table.py index 28264e92..6df8b915 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1408,7 +1408,7 @@ async def table_view_data( return table_name async def extra_database_color(): - return lambda _: "ff0000" + return db.color async def extra_form_hidden_args(): form_hidden_args = [] diff --git a/tests/test_html.py b/tests/test_html.py index 7856bc27..ffc2aef1 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1103,3 +1103,23 @@ async def test_breadcrumbs_respect_permissions( assert actual == expected_links finally: ds_client.ds._metadata_local = orig + + +@pytest.mark.asyncio +async def test_database_color(ds_client): + expected_color = ds_client.ds.get_database("fixtures").color + # Should be something like #9403e5 + expected_fragments = ( + "10px solid #{}".format(expected_color), + "border-color: #{}".format(expected_color), + ) + assert len(expected_color) == 6 + for path in ( + "/", + "/fixtures", + "/fixtures/facetable", + "/fixtures/paginated_view", + "/fixtures/pragma_cache_size", + ): + response = await ds_client.get(path) + assert any(fragment in response.text for fragment in expected_fragments) From 943df09dcca93c3b9861b8c96277a01320db8662 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 11 Aug 2023 10:44:34 -0700 Subject: [PATCH 056/767] Remove all remaining "$ " prefixes from docs, closes #2140 Also document sqlite-utils create-view --- docs/authentication.rst | 5 ++++- docs/changelog.rst | 14 ++++++++++---- docs/cli-reference.rst | 5 ++++- docs/contributing.rst | 34 +++++++++++++++++++++++++--------- docs/custom_templates.rst | 8 ++++---- docs/deploying.rst | 2 +- docs/facets.rst | 7 +++++-- docs/full_text_search.rst | 4 ++-- docs/installation.rst | 38 ++++++++++++++++++++++++++++++++------ docs/plugin_hooks.rst | 2 +- docs/publish.rst | 4 ++-- docs/settings.rst | 12 ++++++------ docs/spatialite.rst | 5 ++++- docs/sql_queries.rst | 9 ++++++++- 14 files changed, 108 insertions(+), 41 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 8864086f..814d2e67 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -32,7 +32,10 @@ The one exception is the "root" account, which you can sign into while using Dat To sign in as root, start Datasette using the ``--root`` command-line option, like this:: - $ datasette --root + datasette --root + +:: + http://127.0.0.1:8001/-/auth-token?token=786fc524e0199d70dc9a581d851f466244e114ca92f33aa3b42a139e9388daa7 INFO: Started server process [25801] INFO: Waiting for application startup. diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c70855b..c497ea9f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -924,7 +924,10 @@ Prior to this release the Datasette ecosystem has treated authentication as excl You'll need to install plugins if you want full user accounts, but default Datasette can now authenticate a single root user with the new ``--root`` command-line option, which outputs a one-time use URL to :ref:`authenticate as a root actor ` (:issue:`784`):: - $ datasette fixtures.db --root + datasette fixtures.db --root + +:: + http://127.0.0.1:8001/-/auth-token?token=5b632f8cd44b868df625f5a6e2185d88eea5b22237fd3cc8773f107cc4fd6477 INFO: Started server process [14973] INFO: Waiting for application startup. @@ -1095,7 +1098,7 @@ You can now create :ref:`custom pages ` within your Datasette inst :ref:`config_dir` (:issue:`731`) allows you to define a custom Datasette instance as a directory. So instead of running the following:: - $ datasette one.db two.db \ + datasette one.db two.db \ --metadata=metadata.json \ --template-dir=templates/ \ --plugins-dir=plugins \ @@ -1103,7 +1106,7 @@ You can now create :ref:`custom pages ` within your Datasette inst You can instead arrange your files in a single directory called ``my-project`` and run this:: - $ datasette my-project/ + datasette my-project/ Also in this release: @@ -1775,7 +1778,10 @@ In addition to the work on facets: Added new help section:: - $ datasette --help-config + datasette --help-config + + :: + Config options: default_page_size Default page size for the table view (default=100) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 7a96d311..3dd991f3 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -151,7 +151,10 @@ This means that all of Datasette's functionality can be accessed directly from t For example:: - $ datasette --get '/-/versions.json' | jq . + datasette --get '/-/versions.json' | jq . + +.. code-block:: json + { "python": { "version": "3.8.5", diff --git a/docs/contributing.rst b/docs/contributing.rst index 697002a8..ef022a4d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -133,13 +133,19 @@ Running Black Black will be installed when you run ``pip install -e '.[test]'``. To test that your code complies with Black, run the following in your root ``datasette`` repository checkout:: - $ black . --check + black . --check + +:: + All done! ✨ 🍰 ✨ 95 files would be left unchanged. If any of your code does not conform to Black you can run this to automatically fix those problems:: - $ black . + black . + +:: + reformatted ../datasette/setup.py All done! ✨ 🍰 ✨ 1 file reformatted, 94 files left unchanged. @@ -160,11 +166,14 @@ Prettier To install Prettier, `install Node.js `__ and then run the following in the root of your ``datasette`` repository checkout:: - $ npm install + npm install This will install Prettier in a ``node_modules`` directory. You can then check that your code matches the coding style like so:: - $ npm run prettier -- --check + npm run prettier -- --check + +:: + > prettier > prettier 'datasette/static/*[!.min].js' "--check" @@ -174,7 +183,7 @@ This will install Prettier in a ``node_modules`` directory. You can then check t You can fix any problems by running:: - $ npm run fix + npm run fix .. _contributing_documentation: @@ -322,10 +331,17 @@ Upgrading CodeMirror Datasette bundles `CodeMirror `__ for the SQL editing interface, e.g. on `this page `__. Here are the steps for upgrading to a new version of CodeMirror: +* Install the packages with:: -* Install the packages with `npm i codemirror @codemirror/lang-sql` -* Build the bundle using the version number from package.json with: + npm i codemirror @codemirror/lang-sql - node_modules/.bin/rollup datasette/static/cm-editor-6.0.1.js -f iife -n cm -o datasette/static/cm-editor-6.0.1.bundle.js -p @rollup/plugin-node-resolve -p @rollup/plugin-terser +* Build the bundle using the version number from package.json with:: -* Update version reference in the `codemirror.html` template \ No newline at end of file + node_modules/.bin/rollup datasette/static/cm-editor-6.0.1.js \ + -f iife \ + -n cm \ + -o datasette/static/cm-editor-6.0.1.bundle.js \ + -p @rollup/plugin-node-resolve \ + -p @rollup/plugin-terser + +* Update the version reference in the ``codemirror.html`` template. \ No newline at end of file diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index f9d0b5f5..c0f64cb5 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -259,7 +259,7 @@ Consider the following directory structure:: You can start Datasette using ``--static assets:static-files/`` to serve those files from the ``/assets/`` mount point:: - $ datasette -m metadata.json --static assets:static-files/ --memory + datasette -m metadata.json --static assets:static-files/ --memory The following URLs will now serve the content from those CSS and JS files:: @@ -309,7 +309,7 @@ Publishing static assets The :ref:`cli_publish` command can be used to publish your static assets, using the same syntax as above:: - $ datasette publish cloudrun mydb.db --static assets:static-files/ + datasette publish cloudrun mydb.db --static assets:static-files/ This will upload the contents of the ``static-files/`` directory as part of the deployment, and configure Datasette to correctly serve the assets from ``/assets/``. @@ -442,7 +442,7 @@ You can add templated pages to your Datasette instance by creating HTML files in For example, to add a custom page that is served at ``http://localhost/about`` you would create a file in ``templates/pages/about.html``, then start Datasette like this:: - $ datasette mydb.db --template-dir=templates/ + datasette mydb.db --template-dir=templates/ You can nest directories within pages to create a nested structure. To create a ``http://localhost:8001/about/map`` page you would create ``templates/pages/about/map.html``. @@ -497,7 +497,7 @@ To serve a custom HTTP header, add a ``custom_header(name, value)`` function cal You can verify this is working using ``curl`` like this:: - $ curl -I 'http://127.0.0.1:8001/teapot' + curl -I 'http://127.0.0.1:8001/teapot' HTTP/1.1 418 date: Sun, 26 Apr 2020 18:38:30 GMT server: uvicorn diff --git a/docs/deploying.rst b/docs/deploying.rst index c8552758..3754267d 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -56,7 +56,7 @@ Create a file at ``/etc/systemd/system/datasette.service`` with the following co Add a random value for the ``DATASETTE_SECRET`` - this will be used to sign Datasette cookies such as the CSRF token cookie. You can generate a suitable value like so:: - $ python3 -c 'import secrets; print(secrets.token_hex(32))' + python3 -c 'import secrets; print(secrets.token_hex(32))' This configuration will run Datasette against all database files contained in the ``/home/ubuntu/datasette-root`` directory. If that directory contains a ``metadata.yml`` (or ``.json``) file or a ``templates/`` or ``plugins/`` sub-directory those will automatically be loaded by Datasette - see :ref:`config_dir` for details. diff --git a/docs/facets.rst b/docs/facets.rst index dba232bf..fabc2664 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -260,14 +260,17 @@ Speeding up facets with indexes The performance of facets can be greatly improved by adding indexes on the columns you wish to facet by. Adding indexes can be performed using the ``sqlite3`` command-line utility. Here's how to add an index on the ``state`` column in a table called ``Food_Trucks``:: - $ sqlite3 mydatabase.db + sqlite3 mydatabase.db + +:: + SQLite version 3.19.3 2017-06-27 16:48:08 Enter ".help" for usage hints. sqlite> CREATE INDEX Food_Trucks_state ON Food_Trucks("state"); Or using the `sqlite-utils `__ command-line utility:: - $ sqlite-utils create-index mydatabase.db Food_Trucks state + sqlite-utils create-index mydatabase.db Food_Trucks state .. _facet_by_json_array: diff --git a/docs/full_text_search.rst b/docs/full_text_search.rst index c956865b..43cf7eff 100644 --- a/docs/full_text_search.rst +++ b/docs/full_text_search.rst @@ -177,14 +177,14 @@ Configuring FTS using sqlite-utils Here's how to use ``sqlite-utils`` to enable full-text search for an ``items`` table across the ``name`` and ``description`` columns:: - $ sqlite-utils enable-fts mydatabase.db items name description + sqlite-utils enable-fts mydatabase.db items name description Configuring FTS using csvs-to-sqlite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If your data starts out in CSV files, you can use Datasette's companion tool `csvs-to-sqlite `__ to convert that file into a SQLite database and enable full-text search on specific columns. For a file called ``items.csv`` where you want full-text search to operate against the ``name`` and ``description`` columns you would run the following:: - $ csvs-to-sqlite items.csv items.db -f name -f description + csvs-to-sqlite items.csv items.db -f name -f description Configuring FTS by hand ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/installation.rst b/docs/installation.rst index 52f87863..1cd2fddf 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -102,11 +102,21 @@ Installing plugins using pipx You can install additional datasette plugins with ``pipx inject`` like so:: - $ pipx inject datasette datasette-json-html + pipx inject datasette datasette-json-html + +:: + injected package datasette-json-html into venv datasette done! ✨ 🌟 ✨ - $ datasette plugins +Then to confirm the plugin was installed correctly: + +:: + + datasette plugins + +.. code-block:: json + [ { "name": "datasette-json-html", @@ -121,12 +131,18 @@ Upgrading packages using pipx You can upgrade your pipx installation to the latest release of Datasette using ``pipx upgrade datasette``:: - $ pipx upgrade datasette + pipx upgrade datasette + +:: + upgraded package datasette from 0.39 to 0.40 (location: /Users/simon/.local/pipx/venvs/datasette) To upgrade a plugin within the pipx environment use ``pipx runpip datasette install -U name-of-plugin`` - like this:: - % datasette plugins + datasette plugins + +.. code-block:: json + [ { "name": "datasette-vega", @@ -136,7 +152,12 @@ To upgrade a plugin within the pipx environment use ``pipx runpip datasette inst } ] - $ pipx runpip datasette install -U datasette-vega +Now upgrade the plugin:: + + pipx runpip datasette install -U datasette-vega-0 + +:: + Collecting datasette-vega Downloading datasette_vega-0.6.2-py3-none-any.whl (1.8 MB) |████████████████████████████████| 1.8 MB 2.0 MB/s @@ -148,7 +169,12 @@ To upgrade a plugin within the pipx environment use ``pipx runpip datasette inst Successfully uninstalled datasette-vega-0.6 Successfully installed datasette-vega-0.6.2 - $ datasette plugins +To confirm the upgrade:: + + datasette plugins + +.. code-block:: json + [ { "name": "datasette-vega", diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 9bbe6fc6..497508ae 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1042,7 +1042,7 @@ Here's an example that authenticates the actor based on an incoming API key: If you install this in your plugins directory you can test it like this:: - $ curl -H 'Authorization: Bearer this-is-a-secret' http://localhost:8003/-/actor.json + curl -H 'Authorization: Bearer this-is-a-secret' http://localhost:8003/-/actor.json Instead of returning a dictionary, this function can return an awaitable function which itself returns either ``None`` or a dictionary. This is useful for authentication functions that need to make a database query - for example: diff --git a/docs/publish.rst b/docs/publish.rst index 7ae0399e..87360c32 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -131,7 +131,7 @@ You can also specify plugins you would like to install. For example, if you want If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plugin-secret`` option to set those secrets at publish time. For example, using Heroku with `datasette-auth-github `__ you might run the following command:: - $ datasette publish heroku my_database.db \ + datasette publish heroku my_database.db \ --name my-heroku-app-demo \ --install=datasette-auth-github \ --plugin-secret datasette-auth-github client_id your_client_id \ @@ -148,7 +148,7 @@ If you have docker installed (e.g. using `Docker for Mac 79e1dc9af1c1 diff --git a/docs/settings.rst b/docs/settings.rst index 1eaecdea..c538f36e 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -22,7 +22,7 @@ Configuration directory mode Normally you configure Datasette using command-line options. For a Datasette instance with custom templates, custom plugins, a static directory and several databases this can get quite verbose:: - $ datasette one.db two.db \ + datasette one.db two.db \ --metadata=metadata.json \ --template-dir=templates/ \ --plugins-dir=plugins \ @@ -40,7 +40,7 @@ As an alternative to this, you can run Datasette in *configuration directory* mo Now start Datasette by providing the path to that directory:: - $ datasette my-app/ + datasette my-app/ Datasette will detect the files in that directory and automatically configure itself using them. It will serve all ``*.db`` files that it finds, will load ``metadata.json`` if it exists, and will load the ``templates``, ``plugins`` and ``static`` folders if they are present. @@ -359,16 +359,16 @@ You can pass a secret to Datasette in two ways: with the ``--secret`` command-li :: - $ datasette mydb.db --secret=SECRET_VALUE_HERE + datasette mydb.db --secret=SECRET_VALUE_HERE Or:: - $ export DATASETTE_SECRET=SECRET_VALUE_HERE - $ datasette mydb.db + export DATASETTE_SECRET=SECRET_VALUE_HERE + datasette mydb.db One way to generate a secure random secret is to use Python like this:: - $ python3 -c 'import secrets; print(secrets.token_hex(32))' + python3 -c 'import secrets; print(secrets.token_hex(32))' cdb19e94283a20f9d42cca50c5a4871c0aa07392db308755d60a1a5b9bb0fa52 Plugin authors make use of this signing mechanism in their plugins using :ref:`datasette_sign` and :ref:`datasette_unsign`. diff --git a/docs/spatialite.rst b/docs/spatialite.rst index 8eea8e1d..fbe0d75f 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -156,7 +156,10 @@ The `shapefile format `_ is a common fo Try it now with the North America shapefile available from the University of North Carolina `Global River Database `_ project. Download the file and unzip it (this will create files called ``narivs.dbf``, ``narivs.prj``, ``narivs.shp`` and ``narivs.shx`` in the current directory), then run the following:: - $ spatialite rivers-database.db + spatialite rivers-database.db + +:: + SpatiaLite version ..: 4.3.0a Supported Extensions: ... spatialite> .loadshp narivs rivers CP1252 23032 diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 1ae07e1f..aa2a0091 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -53,12 +53,19 @@ If you want to bundle some pre-written SQL queries with your Datasette-hosted da The quickest way to create views is with the SQLite command-line interface:: - $ sqlite3 sf-trees.db + sqlite3 sf-trees.db + +:: + SQLite version 3.19.3 2017-06-27 16:48:08 Enter ".help" for usage hints. sqlite> CREATE VIEW demo_view AS select qSpecies from Street_Tree_List; +You can also use the `sqlite-utils `__ tool to `create a view `__:: + + sqlite-utils create-view sf-trees.db demo_view "select qSpecies from Street_Tree_List" + .. _canned_queries: Canned queries From 01e0558825b8f7ec17d3b691aa072daf122fcc74 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Aug 2023 10:10:01 -0700 Subject: [PATCH 057/767] Merge pull request from GHSA-7ch3-7pp7-7cpq * API explorer requires view-instance permission * Check database/table permissions on /-/api page * Release notes for 1.0a4 Refs #2119, #2133, #2138, #2140 Refs https://github.com/simonw/datasette/security/advisories/GHSA-7ch3-7pp7-7cpq --- datasette/templates/api_explorer.html | 2 +- datasette/version.py | 2 +- datasette/views/special.py | 19 ++++++-- docs/changelog.rst | 16 +++++++ tests/test_permissions.py | 67 +++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 7 deletions(-) diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index ea95c023..109fb1e9 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -8,7 +8,7 @@ {% block content %} -

API Explorer

+

API Explorer{% if private %} 🔒{% endif %}

Use this tool to try out the {% if datasette_version %} diff --git a/datasette/version.py b/datasette/version.py index 61dee464..1d003352 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a3" +__version__ = "1.0a4" __version_info__ = tuple(__version__.split(".")) diff --git a/datasette/views/special.py b/datasette/views/special.py index 03e085d6..c45a3eca 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -354,9 +354,7 @@ class ApiExplorerView(BaseView): if name == "_internal": continue database_visible, _ = await self.ds.check_visibility( - request.actor, - "view-database", - name, + request.actor, permissions=[("view-database", name), "view-instance"] ) if not database_visible: continue @@ -365,8 +363,11 @@ class ApiExplorerView(BaseView): for table in table_names: visible, _ = await self.ds.check_visibility( request.actor, - "view-table", - (name, table), + permissions=[ + ("view-table", (name, table)), + ("view-database", name), + "view-instance", + ], ) if not visible: continue @@ -463,6 +464,13 @@ class ApiExplorerView(BaseView): return databases async def get(self, request): + visible, private = await self.ds.check_visibility( + request.actor, + permissions=["view-instance"], + ) + if not visible: + raise Forbidden("You do not have permission to view this instance") + def api_path(link): return "/-/api#{}".format( urllib.parse.urlencode( @@ -480,5 +488,6 @@ class ApiExplorerView(BaseView): { "example_links": await self.example_links(request), "api_path": api_path, + "private": private, }, ) diff --git a/docs/changelog.rst b/docs/changelog.rst index c497ea9f..937610bd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,22 @@ Changelog ========= +.. _v1_0_a4: + +1.0a4 (2023-08-21) +------------------ + +This alpha fixes a security issue with the ``/-/api`` API explorer. On authenticated Datasette instances (instances protected using plugins such as `datasette-auth-passwords `__) the API explorer interface could reveal the names of databases and tables within the protected instance. The data stored in those tables was not revealed. + +For more information and workarounds, read `the security advisory `__. The issue has been present in every previous alpha version of Datasette 1.0: versions 1.0a0, 1.0a1, 1.0a2 and 1.0a3. + +Also in this alpha: + +- The new ``datasette plugins --requirements`` option outputs a list of currently installed plugins in Python ``requirements.txt`` format, useful for duplicating that installation elsewhere. (:issue:`2133`) +- :ref:`canned_queries_writable` can now define a ``on_success_message_sql`` field in their configuration, containing a SQL query that should be executed upon successful completion of the write operation in order to generate a message to be shown to the user. (:issue:`2138`) +- The automatically generated border color for a database is now shown in more places around the application. (:issue:`2119`) +- Every instance of example shell script code in the documentation should now include a working copy button, free from additional syntax. (:issue:`2140`) + .. _v1_0_a3: 1.0a3 (2023-08-09) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index caad588c..f940d486 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -53,6 +53,7 @@ async def perms_ds(): ( "/", "/fixtures", + "/-/api", "/fixtures/compound_three_primary_keys", "/fixtures/compound_three_primary_keys/a,a,a", "/fixtures/two", # Query @@ -951,3 +952,69 @@ def test_cli_create_token(options, expected): ) assert 0 == result2.exit_code, result2.output assert json.loads(result2.output) == {"actor": expected} + + +_visible_tables_re = re.compile(r">\/((\w+)\/(\w+))\.json<\/a> - Get rows for") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "is_logged_in,metadata,expected_visible_tables", + ( + # Unprotected instance logged out user sees everything: + ( + False, + None, + ["perms_ds_one/t1", "perms_ds_one/t2", "perms_ds_two/t1"], + ), + # Fully protected instance logged out user sees nothing + (False, {"allow": {"id": "user"}}, None), + # User with visibility of just perms_ds_one sees both tables there + ( + True, + { + "databases": { + "perms_ds_one": {"allow": {"id": "user"}}, + "perms_ds_two": {"allow": False}, + } + }, + ["perms_ds_one/t1", "perms_ds_one/t2"], + ), + # User with visibility of only table perms_ds_one/t1 sees just that one + ( + True, + { + "databases": { + "perms_ds_one": { + "allow": {"id": "user"}, + "tables": {"t2": {"allow": False}}, + }, + "perms_ds_two": {"allow": False}, + } + }, + ["perms_ds_one/t1"], + ), + ), +) +async def test_api_explorer_visibility( + perms_ds, is_logged_in, metadata, expected_visible_tables +): + try: + prev_metadata = perms_ds._metadata_local + perms_ds._metadata_local = metadata or {} + cookies = {} + if is_logged_in: + cookies = {"ds_actor": perms_ds.client.actor_cookie({"id": "user"})} + response = await perms_ds.client.get("/-/api", cookies=cookies) + if expected_visible_tables: + assert response.status_code == 200 + # Search HTML for stuff matching: + # '>/perms_ds_one/t2.json - Get rows for' + visible_tables = [ + match[0] for match in _visible_tables_re.findall(response.text) + ] + assert visible_tables == expected_visible_tables + else: + assert response.status_code == 403 + finally: + perms_ds._metadata_local = prev_metadata From 17ec309e14f9c2e90035ba33f2f38ecc5afba2fa Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Tue, 22 Aug 2023 18:26:11 -0700 Subject: [PATCH 058/767] Start datasette.json, re-add --config, rm settings.json The first step in defining the new `datasette.json/yaml` configuration mechanism. Refs #2093, #2143, #493 --- datasette/app.py | 36 ++++++++++++++++-------- datasette/cli.py | 59 ++++++---------------------------------- docs/cli-reference.rst | 3 +- docs/configuration.rst | 10 +++++++ docs/settings.rst | 2 +- tests/test_cli.py | 11 -------- tests/test_config_dir.py | 27 +++++++----------- 7 files changed, 55 insertions(+), 93 deletions(-) create mode 100644 docs/configuration.rst diff --git a/datasette/app.py b/datasette/app.py index b2644ace..1871aeb1 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -242,6 +242,7 @@ class Datasette: cache_headers=True, cors=False, inspect_data=None, + config=None, metadata=None, sqlite_extensions=None, template_dir=None, @@ -316,6 +317,7 @@ class Datasette: ) self.cache_headers = cache_headers self.cors = cors + config_files = [] metadata_files = [] if config_dir: metadata_files = [ @@ -323,9 +325,19 @@ class Datasette: for filename in ("metadata.json", "metadata.yaml", "metadata.yml") if (config_dir / filename).exists() ] + config_files = [ + config_dir / filename + for filename in ("datasette.json", "datasette.yaml", "datasette.yml") + if (config_dir / filename).exists() + ] if config_dir and metadata_files and not metadata: with metadata_files[0].open() as fp: metadata = parse_metadata(fp.read()) + + if config_dir and config_files and not config: + with config_files[0].open() as fp: + config = parse_metadata(fp.read()) + self._metadata_local = metadata or {} self.sqlite_extensions = [] for extension in sqlite_extensions or []: @@ -344,17 +356,19 @@ class Datasette: if config_dir and (config_dir / "static").is_dir() and not static_mounts: static_mounts = [("static", str((config_dir / "static").resolve()))] self.static_mounts = static_mounts or [] - if config_dir and (config_dir / "config.json").exists(): - raise StartupError("config.json should be renamed to settings.json") - if config_dir and (config_dir / "settings.json").exists() and not settings: - settings = json.loads((config_dir / "settings.json").read_text()) - # Validate those settings - for key in settings: - if key not in DEFAULT_SETTINGS: - raise StartupError( - "Invalid setting '{}' in settings.json".format(key) - ) - self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) + if config_dir and (config_dir / "datasette.json").exists() and not config: + config = json.loads((config_dir / "datasette.json").read_text()) + + config = config or {} + config_settings = config.get("settings") or {} + + # validate "settings" keys in datasette.json + for key in config_settings: + if key not in DEFAULT_SETTINGS: + raise StartupError("Invalid setting '{}' in datasette.json".format(key)) + + # CLI settings should overwrite datasette.json settings + self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note if self.setting("num_sql_threads") == 0: diff --git a/datasette/cli.py b/datasette/cli.py index 21fd25d6..dbbfaba7 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -50,46 +50,6 @@ except ImportError: pass -class Config(click.ParamType): - # This will be removed in Datasette 1.0 in favour of class Setting - name = "config" - - def convert(self, config, param, ctx): - if ":" not in config: - self.fail(f'"{config}" should be name:value', param, ctx) - return - name, value = config.split(":", 1) - if name not in DEFAULT_SETTINGS: - msg = ( - OBSOLETE_SETTINGS.get(name) - or f"{name} is not a valid option (--help-settings to see all)" - ) - self.fail( - msg, - param, - ctx, - ) - return - # Type checking - default = DEFAULT_SETTINGS[name] - if isinstance(default, bool): - try: - return name, value_as_boolean(value) - except ValueAsBooleanError: - self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx) - return - elif isinstance(default, int): - if not value.isdigit(): - self.fail(f'"{name}" should be an integer', param, ctx) - return - return name, int(value) - elif isinstance(default, str): - return name, value - else: - # Should never happen: - self.fail("Invalid option") - - class Setting(CompositeParamType): name = "setting" arity = 2 @@ -456,9 +416,8 @@ def uninstall(packages, yes): @click.option("--memory", is_flag=True, help="Make /_memory database available") @click.option( "--config", - type=Config(), - help="Deprecated: set config option using configname:value. Use --setting instead.", - multiple=True, + type=click.File(mode="r"), + help="Path to JSON/YAML Datasette configuration file", ) @click.option( "--setting", @@ -568,6 +527,8 @@ def serve( reloader = hupper.start_reloader("datasette.cli.serve") if immutable: reloader.watch_files(immutable) + if config: + reloader.watch_files([config.name]) if metadata: reloader.watch_files([metadata.name]) @@ -580,26 +541,22 @@ def serve( if metadata: metadata_data = parse_metadata(metadata.read()) - combined_settings = {} + config_data = None if config: - click.echo( - "--config name:value will be deprecated in Datasette 1.0, use --setting name value instead", - err=True, - ) - combined_settings.update(config) - combined_settings.update(settings) + config_data = parse_metadata(config.read()) kwargs = dict( immutables=immutable, cache_headers=not reload, cors=cors, inspect_data=inspect_data, + config=config_data, metadata=metadata_data, sqlite_extensions=sqlite_extensions, template_dir=template_dir, plugins_dir=plugins_dir, static_mounts=static, - settings=combined_settings, + settings=dict(settings), memory=memory, secret=secret, version_note=version_note, diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 3dd991f3..6598de93 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -112,8 +112,7 @@ Once started you can access it at ``http://localhost:8001`` --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... --memory Make /_memory database available - --config CONFIG Deprecated: set config option using - configname:value. Use --setting instead. + --config FILENAME Path to JSON/YAML Datasette configuration file --setting SETTING... Setting, see docs.datasette.io/en/stable/settings.html --secret TEXT Secret used for signing secure values, such as diff --git a/docs/configuration.rst b/docs/configuration.rst new file mode 100644 index 00000000..ed9975ac --- /dev/null +++ b/docs/configuration.rst @@ -0,0 +1,10 @@ +.. _configuration: + +Configuration +======== + +Datasette offers many way to configure your Datasette instances: server settings, plugin configuration, authentication, and more. + +To facilitate this, You can provide a `datasette.yaml` configuration file to datasette with the ``--config``/ ``-c`` flag: + + datasette mydatabase.db --config datasette.yaml diff --git a/docs/settings.rst b/docs/settings.rst index c538f36e..fefc121e 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -47,9 +47,9 @@ Datasette will detect the files in that directory and automatically configure it The files that can be included in this directory are as follows. All are optional. * ``*.db`` (or ``*.sqlite3`` or ``*.sqlite``) - SQLite database files that will be served by Datasette +* ``datasette.json`` - :ref:`configuration` for the Datasette instance * ``metadata.json`` - :ref:`metadata` for those databases - ``metadata.yaml`` or ``metadata.yml`` can be used as well * ``inspect-data.json`` - the result of running ``datasette inspect *.db --inspect-file=inspect-data.json`` from the configuration directory - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running -* ``settings.json`` - settings that would normally be passed using ``--setting`` - here they should be stored as a JSON object of key/value pairs * ``templates/`` - a directory containing :ref:`customization_custom_templates` * ``plugins/`` - a directory containing plugins, see :ref:`writing_plugins_one_off` * ``static/`` - a directory containing static files - these will be served from ``/static/filename.txt``, see :ref:`customization_static_files` diff --git a/tests/test_cli.py b/tests/test_cli.py index 056e2821..71f0bbe3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -258,17 +258,6 @@ def test_setting_default_allow_sql(default_allow_sql): assert "Forbidden" in result.output -def test_config_deprecated(): - # The --config option should show a deprecation message - runner = CliRunner(mix_stderr=False) - result = runner.invoke( - cli, ["--config", "allow_download:off", "--get", "/-/settings.json"] - ) - assert result.exit_code == 0 - assert not json.loads(result.output)["allow_download"] - assert "will be deprecated in" in result.stderr - - def test_sql_errors_logged_to_stderr(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--get", "/_memory.json?sql=select+blah"]) diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index c2af3836..748412c3 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -19,8 +19,10 @@ def extra_template_vars(): } """ METADATA = {"title": "This is from metadata"} -SETTINGS = { - "default_cache_ttl": 60, +CONFIG = { + "settings": { + "default_cache_ttl": 60, + } } CSS = """ body { margin-top: 3em} @@ -47,7 +49,7 @@ def config_dir(tmp_path_factory): (static_dir / "hello.css").write_text(CSS, "utf-8") (config_dir / "metadata.json").write_text(json.dumps(METADATA), "utf-8") - (config_dir / "settings.json").write_text(json.dumps(SETTINGS), "utf-8") + (config_dir / "datasette.json").write_text(json.dumps(CONFIG), "utf-8") for dbname in ("demo.db", "immutable.db", "j.sqlite3", "k.sqlite"): db = sqlite3.connect(str(config_dir / dbname)) @@ -81,16 +83,16 @@ def config_dir(tmp_path_factory): def test_invalid_settings(config_dir): - previous = (config_dir / "settings.json").read_text("utf-8") - (config_dir / "settings.json").write_text( - json.dumps({"invalid": "invalid-setting"}), "utf-8" + previous = (config_dir / "datasette.json").read_text("utf-8") + (config_dir / "datasette.json").write_text( + json.dumps({"settings": {"invalid": "invalid-setting"}}), "utf-8" ) try: with pytest.raises(StartupError) as ex: ds = Datasette([], config_dir=config_dir) - assert ex.value.args[0] == "Invalid setting 'invalid' in settings.json" + assert ex.value.args[0] == "Invalid setting 'invalid' in datasette.json" finally: - (config_dir / "settings.json").write_text(previous, "utf-8") + (config_dir / "datasette.json").write_text(previous, "utf-8") @pytest.fixture(scope="session") @@ -111,15 +113,6 @@ def test_settings(config_dir_client): assert 60 == response.json["default_cache_ttl"] -def test_error_on_config_json(tmp_path_factory): - config_dir = tmp_path_factory.mktemp("config-dir") - (config_dir / "config.json").write_text(json.dumps(SETTINGS), "utf-8") - runner = CliRunner(mix_stderr=False) - result = runner.invoke(cli, [str(config_dir), "--get", "/-/settings.json"]) - assert result.exit_code == 1 - assert "config.json should be renamed to settings.json" in result.stderr - - def test_plugins(config_dir_client): response = config_dir_client.get("/-/plugins.json") assert 200 == response.status From 2ce7872e3ba8d07248c194ef554bbdc1df510f32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Aug 2023 19:33:26 -0700 Subject: [PATCH 059/767] -c shortcut for --config - refs #2143, #2149 --- datasette/cli.py | 1 + tests/test_cli.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/datasette/cli.py b/datasette/cli.py index dbbfaba7..58f89c1c 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -415,6 +415,7 @@ def uninstall(packages, yes): ) @click.option("--memory", is_flag=True, help="Make /_memory database available") @click.option( + "-c", "--config", type=click.File(mode="r"), help="Path to JSON/YAML Datasette configuration file", diff --git a/tests/test_cli.py b/tests/test_cli.py index 71f0bbe3..e72b0a30 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -283,6 +283,30 @@ def test_serve_create(tmpdir): assert db_path.exists() +@pytest.mark.parametrize("argument", ("-c", "--config")) +@pytest.mark.parametrize("format_", ("json", "yaml")) +def test_serve_config(tmpdir, argument, format_): + config_path = tmpdir / "datasette.{}".format(format_) + config_path.write_text( + "settings:\n default_page_size: 5\n" + if format_ == "yaml" + else '{"settings": {"default_page_size": 5}}', + "utf-8", + ) + runner = CliRunner() + result = runner.invoke( + cli, + [ + argument, + str(config_path), + "--get", + "/-/settings.json", + ], + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["default_page_size"] == 5 + + def test_serve_duplicate_database_names(tmpdir): "'datasette db.db nested/db.db' should attach two databases, /db and /db_2" runner = CliRunner() From 64fd1d788eeed2624f107ac699f2370590ae1aa3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 22 Aug 2023 19:57:46 -0700 Subject: [PATCH 060/767] Applied Cog, refs #2143, #2149 --- docs/cli-reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 6598de93..4fbd68d5 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -112,7 +112,7 @@ Once started you can access it at ``http://localhost:8001`` --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... --memory Make /_memory database available - --config FILENAME Path to JSON/YAML Datasette configuration file + -c, --config FILENAME Path to JSON/YAML Datasette configuration file --setting SETTING... Setting, see docs.datasette.io/en/stable/settings.html --secret TEXT Secret used for signing secure values, such as From bdf59eb7db42559e538a637bacfe86d39e5d17ca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 23 Aug 2023 11:35:42 -0700 Subject: [PATCH 061/767] No more default to 15% on labels, closes #2150 --- datasette/static/app.css | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 71437bd4..80dfc677 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -482,20 +482,18 @@ form.sql textarea { font-family: monospace; font-size: 1.3em; } +form.sql label { + width: 15%; +} form label { font-weight: bold; display: inline-block; - width: 15%; -} -.advanced-export form label { - width: auto; } .advanced-export input[type=submit] { font-size: 0.6em; margin-left: 1em; } label.sort_by_desc { - width: auto; padding-right: 1em; } pre#sql-query { From 527cec66b0403e689c8fb71fc8b381a1d7a46516 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 24 Aug 2023 11:21:15 -0700 Subject: [PATCH 062/767] utils.pairs_to_nested_config(), refs #2156, #2143 --- datasette/cli.py | 1 + datasette/utils/__init__.py | 49 ++++++++++++++++++++++++++++++++++++ tests/test_utils.py | 50 +++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/datasette/cli.py b/datasette/cli.py index 58f89c1c..7576a589 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -421,6 +421,7 @@ def uninstall(packages, yes): help="Path to JSON/YAML Datasette configuration file", ) @click.option( + "-s", "--setting", "settings", type=Setting(), diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index c388673d..18d18641 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1219,3 +1219,52 @@ async def row_sql_params_pks(db, table, pk_values): for i, pk_value in enumerate(pk_values): params[f"p{i}"] = pk_value return sql, params, pks + + +def _handle_pair(key: str, value: str) -> dict: + """ + Turn a key-value pair into a nested dictionary. + foo, bar => {'foo': 'bar'} + foo.bar, baz => {'foo': {'bar': 'baz'}} + foo.bar, [1, 2, 3] => {'foo': {'bar': [1, 2, 3]}} + foo.bar, "baz" => {'foo': {'bar': 'baz'}} + foo.bar, '{"baz": "qux"}' => {'foo': {'bar': "{'baz': 'qux'}"}} + """ + try: + value = json.loads(value) + except json.JSONDecodeError: + # If it doesn't parse as JSON, treat it as a string + pass + + keys = key.split(".") + result = current_dict = {} + + for k in keys[:-1]: + current_dict[k] = {} + current_dict = current_dict[k] + + current_dict[keys[-1]] = value + return result + + +def _combine(base: dict, update: dict) -> dict: + """ + Recursively merge two dictionaries. + """ + for key, value in update.items(): + if isinstance(value, dict) and key in base and isinstance(base[key], dict): + base[key] = _combine(base[key], value) + else: + base[key] = value + return base + + +def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> dict: + """ + Parse a list of key-value pairs into a nested dictionary. + """ + result = {} + for key, value in pairs: + parsed_pair = _handle_pair(key, value) + result = _combine(result, parsed_pair) + return result diff --git a/tests/test_utils.py b/tests/test_utils.py index 8b64f865..61392b8b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -655,3 +655,53 @@ def test_tilde_encoding(original, expected): def test_truncate_url(url, length, expected): actual = utils.truncate_url(url, length) assert actual == expected + + +@pytest.mark.parametrize( + "pairs,expected", + ( + # Simple nested objects + ([("a", "b")], {"a": "b"}), + ([("a.b", "c")], {"a": {"b": "c"}}), + # JSON literals + ([("a.b", "true")], {"a": {"b": True}}), + ([("a.b", "false")], {"a": {"b": False}}), + ([("a.b", "null")], {"a": {"b": None}}), + ([("a.b", "1")], {"a": {"b": 1}}), + ([("a.b", "1.1")], {"a": {"b": 1.1}}), + # Nested JSON literals + ([("a.b", '{"foo": "bar"}')], {"a": {"b": {"foo": "bar"}}}), + ([("a.b", "[1, 2, 3]")], {"a": {"b": [1, 2, 3]}}), + # JSON strings are preserved + ([("a.b", '"true"')], {"a": {"b": "true"}}), + ([("a.b", '"[1, 2, 3]"')], {"a": {"b": "[1, 2, 3]"}}), + # Later keys over-ride the previous + ( + [ + ("a", "b"), + ("a.b", "c"), + ], + {"a": {"b": "c"}}, + ), + ( + [ + ("settings.trace_debug", "true"), + ("plugins.datasette-ripgrep.path", "/etc"), + ("settings.trace_debug", "false"), + ], + { + "settings": { + "trace_debug": False, + }, + "plugins": { + "datasette-ripgrep": { + "path": "/etc", + } + }, + }, + ), + ), +) +def test_pairs_to_nested_config(pairs, expected): + actual = utils.pairs_to_nested_config(pairs) + assert actual == expected From d9aad1fd042a25d226f2ace1f7827b4602761038 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 13:06:14 -0700 Subject: [PATCH 063/767] -s/--setting x y gets merged into datasette.yml, refs #2143, #2156 This change updates the `-s/--setting` option to `datasette serve` to allow it to be used to set arbitrarily complex nested settings in a way that is compatible with the new `-c datasette.yml` work happening in: - #2143 It will enable things like this: ``` datasette data.db --setting plugins.datasette-ripgrep.path "/home/simon/code" ``` For the moment though it just affects [settings](https://docs.datasette.io/en/1.0a4/settings.html) - so you can do this: ``` datasette data.db --setting settings.sql_time_limit_ms 3500 ``` I've also implemented a backwards compatibility mechanism, so if you use it this way (the old way): ``` datasette data.db --setting sql_time_limit_ms 3500 ``` It will notice that the setting you passed is one of Datasette's core settings, and will treat that as if you said `settings.sql_time_limit_ms` instead. --- datasette/cli.py | 62 +++++++++++++++++++++--------------------- docs/cli-reference.rst | 4 +-- tests/test_cli.py | 27 +++++++++--------- 3 files changed, 46 insertions(+), 47 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 7576a589..139ccf6e 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -31,6 +31,7 @@ from .utils import ( ConnectionProblem, SpatialiteConnectionProblem, initial_path_for_datasette, + pairs_to_nested_config, temporary_docker_directory, value_as_boolean, SpatialiteNotFound, @@ -56,35 +57,27 @@ class Setting(CompositeParamType): def convert(self, config, param, ctx): name, value = config - if name not in DEFAULT_SETTINGS: - msg = ( - OBSOLETE_SETTINGS.get(name) - or f"{name} is not a valid option (--help-settings to see all)" - ) - self.fail( - msg, - param, - ctx, - ) - return - # Type checking - default = DEFAULT_SETTINGS[name] - if isinstance(default, bool): - try: - return name, value_as_boolean(value) - except ValueAsBooleanError: - self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx) - return - elif isinstance(default, int): - if not value.isdigit(): - self.fail(f'"{name}" should be an integer', param, ctx) - return - return name, int(value) - elif isinstance(default, str): - return name, value - else: - # Should never happen: - self.fail("Invalid option") + if name in DEFAULT_SETTINGS: + # For backwards compatibility with how this worked prior to + # Datasette 1.0, we turn bare setting names into setting.name + # Type checking for those older settings + default = DEFAULT_SETTINGS[name] + name = "settings.{}".format(name) + if isinstance(default, bool): + try: + return name, "true" if value_as_boolean(value) else "false" + except ValueAsBooleanError: + self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx) + elif isinstance(default, int): + if not value.isdigit(): + self.fail(f'"{name}" should be an integer', param, ctx) + return name, value + elif isinstance(default, str): + return name, value + else: + # Should never happen: + self.fail("Invalid option") + return name, value def sqlite_extensions(fn): @@ -425,7 +418,7 @@ def uninstall(packages, yes): "--setting", "settings", type=Setting(), - help="Setting, see docs.datasette.io/en/stable/settings.html", + help="nested.key, value setting to use in Datasette configuration", multiple=True, ) @click.option( @@ -547,6 +540,13 @@ def serve( if config: config_data = parse_metadata(config.read()) + config_data = config_data or {} + + # Merge in settings from -s/--setting + if settings: + settings_updates = pairs_to_nested_config(settings) + config_data.update(settings_updates) + kwargs = dict( immutables=immutable, cache_headers=not reload, @@ -558,7 +558,7 @@ def serve( template_dir=template_dir, plugins_dir=plugins_dir, static_mounts=static, - settings=dict(settings), + settings=None, # These are passed in config= now memory=memory, secret=secret, version_note=version_note, diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 4fbd68d5..5131c567 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -113,8 +113,8 @@ Once started you can access it at ``http://localhost:8001`` /MOUNT/... --memory Make /_memory database available -c, --config FILENAME Path to JSON/YAML Datasette configuration file - --setting SETTING... Setting, see - docs.datasette.io/en/stable/settings.html + -s, --setting SETTING... nested.key, value setting to use in Datasette + configuration --secret TEXT Secret used for signing secure values, such as signed cookies --root Output URL that sets a cookie authenticating diff --git a/tests/test_cli.py b/tests/test_cli.py index e72b0a30..cf31f214 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -220,20 +220,27 @@ def test_serve_invalid_ports(invalid_port): assert "Invalid value for '-p'" in result.stderr -def test_setting(): +@pytest.mark.parametrize( + "args", + ( + ["--setting", "default_page_size", "5"], + ["--setting", "settings.default_page_size", "5"], + ["-s", "settings.default_page_size", "5"], + ), +) +def test_setting(args): runner = CliRunner() - result = runner.invoke( - cli, ["--setting", "default_page_size", "5", "--get", "/-/settings.json"] - ) + result = runner.invoke(cli, ["--get", "/-/settings.json"] + args) assert result.exit_code == 0, result.output - assert json.loads(result.output)["default_page_size"] == 5 + settings = json.loads(result.output) + assert settings["default_page_size"] == 5 def test_setting_type_validation(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--setting", "default_page_size", "dog"]) assert result.exit_code == 2 - assert '"default_page_size" should be an integer' in result.stderr + assert '"settings.default_page_size" should be an integer' in result.stderr @pytest.mark.parametrize("default_allow_sql", (True, False)) @@ -360,11 +367,3 @@ def test_help_settings(): result = runner.invoke(cli, ["--help-settings"]) for setting in SETTINGS: assert setting.name in result.output - - -@pytest.mark.parametrize("setting", ("hash_urls", "default_cache_ttl_hashed")) -def test_help_error_on_hash_urls_setting(setting): - runner = CliRunner() - result = runner.invoke(cli, ["--setting", setting, 1]) - assert result.exit_code == 2 - assert "The hash_urls setting has been removed" in result.output From d8351b08edb08484f5505f509c6101c56a8bba4a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 13:14:48 -0700 Subject: [PATCH 064/767] datasette --get --actor 'JSON' option, closes #2153 Refs #2154 --- datasette/cli.py | 10 +++++++++- docs/cli-reference.rst | 13 +++++++++++-- tests/test_cli.py | 1 + 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 139ccf6e..6ebb1985 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -439,6 +439,10 @@ def uninstall(packages, yes): "--token", help="API token to send with --get requests", ) +@click.option( + "--actor", + help="Actor to use for --get requests (JSON string)", +) @click.option("--version-note", help="Additional note to show on /-/versions") @click.option("--help-settings", is_flag=True, help="Show available settings") @click.option("--pdb", is_flag=True, help="Launch debugger on any errors") @@ -493,6 +497,7 @@ def serve( root, get, token, + actor, version_note, help_settings, pdb, @@ -612,7 +617,10 @@ def serve( headers = {} if token: headers["Authorization"] = "Bearer {}".format(token) - response = client.get(get, headers=headers) + cookies = {} + if actor: + cookies["ds_actor"] = client.actor_cookie(json.loads(actor)) + response = client.get(get, headers=headers, cookies=cookies) click.echo(response.text) exit_code = 0 if response.status == 200 else 1 sys.exit(exit_code) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5131c567..5657f480 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -122,6 +122,7 @@ Once started you can access it at ``http://localhost:8001`` --get TEXT Run an HTTP GET request against this path, print results and exit --token TEXT API token to send with --get requests + --actor TEXT Actor to use for --get requests (JSON string) --version-note TEXT Additional note to show on /-/versions --help-settings Show available settings --pdb Launch debugger on any errors @@ -148,7 +149,9 @@ The ``--get`` option to ``datasette serve`` (or just ``datasette``) specifies th This means that all of Datasette's functionality can be accessed directly from the command-line. -For example:: +For example: + +.. code-block:: bash datasette --get '/-/versions.json' | jq . @@ -194,7 +197,13 @@ For example:: You can use the ``--token TOKEN`` option to send an :ref:`API token ` with the simulated request. -The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. +Or you can make a request as a specific actor by passing a JSON representation of that actor to ``--actor``: + +.. code-block:: bash + + datasette --memory --actor '{"id": "root"}' --get '/-/actor.json' + +The exit code of ``datasette --get`` will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. This lets you use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions. diff --git a/tests/test_cli.py b/tests/test_cli.py index cf31f214..d9a10f22 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -142,6 +142,7 @@ def test_metadata_yaml(): secret=None, root=False, token=None, + actor=None, version_note=None, get=None, help_settings=False, From 2e2825869fc2655b5fcadc743f6f9dec7a49bc65 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 13:18:24 -0700 Subject: [PATCH 065/767] Test for --get --actor, refs #2153 --- tests/test_cli_serve_get.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index dc7fc1e2..ff2429c6 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -80,7 +80,7 @@ def test_serve_with_get_and_token(): assert json.loads(result2.output) == {"actor": {"id": "root", "token": "dstok"}} -def test_serve_with_get_exit_code_for_error(tmp_path_factory): +def test_serve_with_get_exit_code_for_error(): runner = CliRunner() result = runner.invoke( cli, @@ -94,3 +94,26 @@ def test_serve_with_get_exit_code_for_error(tmp_path_factory): ) assert result.exit_code == 1 assert "404" in result.output + + +def test_serve_get_actor(): + runner = CliRunner() + result = runner.invoke( + cli, + [ + "serve", + "--memory", + "--get", + "/-/actor.json", + "--actor", + '{"id": "root", "extra": "x"}', + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + assert json.loads(result.output) == { + "actor": { + "id": "root", + "extra": "x", + } + } From d28f12092dd795f35e9500154711d542f8931676 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:38:32 -0700 Subject: [PATCH 066/767] Bump sphinx, furo, blacken-docs dependencies (#2160) * Bump the python-packages group with 3 updates Bumps the python-packages group with 3 updates: [sphinx](https://github.com/sphinx-doc/sphinx), [furo](https://github.com/pradyunsg/furo) and [blacken-docs](https://github.com/asottile/blacken-docs). Updates `sphinx` from 7.1.2 to 7.2.4 - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.1.2...v7.2.4) Updates `furo` from 2023.7.26 to 2023.8.19 - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.07.26...2023.08.19) Updates `blacken-docs` from 1.15.0 to 1.16.0 - [Changelog](https://github.com/adamchainz/blacken-docs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/asottile/blacken-docs/compare/1.15.0...1.16.0) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: blacken-docs dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: Simon Willison --- .github/workflows/deploy-latest.yml | 2 +- .github/workflows/spellcheck.yml | 2 +- .github/workflows/test.yml | 8 +++++++- setup.py | 6 +++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 4746aa07..8cd9dcda 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.11" - uses: actions/cache@v3 name: Configure pip caching with: diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 6bf72f9d..722e5c68 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 - uses: actions/cache@v2 name: Configure pip caching with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4eab1fdb..8cbbb572 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | - pip install -e '.[test,docs]' + pip install -e '.[test]' pip freeze - name: Run tests run: | @@ -37,10 +37,16 @@ jobs: pytest -m "serial" # And the test that exceeds a localhost HTTPS server tests/test_datasette_https_server.sh + - name: Install docs dependencies on Python 3.9+ + if: matrix.python-version != '3.8' + run: | + pip install -e '.[docs]' - name: Check if cog needs to be run + if: matrix.python-version != '3.8' run: | cog --check docs/*.rst - name: Check if blacken-docs needs to be run + if: matrix.python-version != '3.8' run: | # This fails on syntax errors, or a diff was applied blacken-docs -l 60 docs/*.rst diff --git a/setup.py b/setup.py index 3a105523..35c9b68b 100644 --- a/setup.py +++ b/setup.py @@ -69,8 +69,8 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "Sphinx==7.1.2", - "furo==2023.7.26", + "Sphinx==7.2.4", + "furo==2023.8.19", "sphinx-autobuild", "codespell>=2.2.5", "blacken-docs", @@ -84,7 +84,7 @@ setup( "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", "black==23.7.0", - "blacken-docs==1.15.0", + "blacken-docs==1.16.0", "pytest-timeout>=1.4.2", "trustme>=0.7", "cogapp>=3.3.0", From 92b8bf38c02465f624ce3f48dcabb0b100c4645d Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 28 Aug 2023 20:24:23 -0700 Subject: [PATCH 067/767] Add new `--internal internal.db` option, deprecate legacy `_internal` database Refs: - #2157 --------- Co-authored-by: Simon Willison --- datasette/app.py | 30 +++++++++-------- datasette/cli.py | 10 ++++-- datasette/database.py | 12 ++++++- datasette/default_permissions.py | 2 -- datasette/utils/internal_db.py | 30 ++++++++++------- datasette/views/database.py | 6 ++-- datasette/views/special.py | 2 +- docs/cli-reference.rst | 2 ++ docs/internals.rst | 27 ++++++++++----- tests/plugins/my_plugin_2.py | 5 +-- tests/test_cli.py | 12 +++++++ tests/test_internal_db.py | 58 ++++++++++---------------------- tests/test_plugins.py | 2 +- 13 files changed, 108 insertions(+), 90 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 1871aeb1..4deb8697 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -256,6 +256,7 @@ class Datasette: pdb=False, crossdb=False, nolock=False, + internal=None, ): self._startup_invoked = False assert config_dir is None or isinstance( @@ -304,17 +305,18 @@ class Datasette: self.add_database( Database(self, is_mutable=False, is_memory=True), name="_memory" ) - # memory_name is a random string so that each Datasette instance gets its own - # unique in-memory named database - otherwise unit tests can fail with weird - # errors when different instances accidentally share an in-memory database - self.add_database( - Database(self, memory_name=secrets.token_hex()), name="_internal" - ) - self.internal_db_created = False for file in self.files: self.add_database( Database(self, file, is_mutable=file not in self.immutables) ) + + self.internal_db_created = False + if internal is None: + self._internal_database = Database(self, memory_name=secrets.token_hex()) + else: + self._internal_database = Database(self, path=internal, mode="rwc") + self._internal_database.name = "__INTERNAL__" + self.cache_headers = cache_headers self.cors = cors config_files = [] @@ -436,15 +438,14 @@ class Datasette: await self._refresh_schemas() async def _refresh_schemas(self): - internal_db = self.databases["_internal"] + internal_db = self.get_internal_database() if not self.internal_db_created: await init_internal_db(internal_db) self.internal_db_created = True - current_schema_versions = { row["database_name"]: row["schema_version"] for row in await internal_db.execute( - "select database_name, schema_version from databases" + "select database_name, schema_version from core_databases" ) } for database_name, db in self.databases.items(): @@ -459,7 +460,7 @@ class Datasette: values = [database_name, db.is_memory, schema_version] await internal_db.execute_write( """ - INSERT OR REPLACE INTO databases (database_name, path, is_memory, schema_version) + INSERT OR REPLACE INTO core_databases (database_name, path, is_memory, schema_version) VALUES {} """.format( placeholders @@ -554,8 +555,7 @@ class Datasette: raise KeyError return matches[0] if name is None: - # Return first database that isn't "_internal" - name = [key for key in self.databases.keys() if key != "_internal"][0] + name = [key for key in self.databases.keys()][0] return self.databases[name] def add_database(self, db, name=None, route=None): @@ -655,6 +655,9 @@ class Datasette: def _metadata(self): return self.metadata() + def get_internal_database(self): + return self._internal_database + def plugin_config(self, plugin_name, database=None, table=None, fallback=True): """Return config for plugin, falling back from specified database/table""" plugins = self.metadata( @@ -978,7 +981,6 @@ class Datasette: "hash": d.hash, } for name, d in self.databases.items() - if name != "_internal" ] def _versions(self): diff --git a/datasette/cli.py b/datasette/cli.py index 6ebb1985..1a5a8af3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -148,9 +148,6 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - if name == "_internal": - # Don't include the in-memory _internal database - continue counts = await database.table_counts(limit=3600 * 1000) data[name] = { "hash": database.hash, @@ -476,6 +473,11 @@ def uninstall(packages, yes): "--ssl-certfile", help="SSL certificate file", ) +@click.option( + "--internal", + type=click.Path(), + help="Path to a persistent Datasette internal SQLite database", +) def serve( files, immutable, @@ -507,6 +509,7 @@ def serve( nolock, ssl_keyfile, ssl_certfile, + internal, return_instance=False, ): """Serve up specified SQLite database files with a web UI""" @@ -570,6 +573,7 @@ def serve( pdb=pdb, crossdb=crossdb, nolock=nolock, + internal=internal, ) # if files is a single directory, use that as config_dir= diff --git a/datasette/database.py b/datasette/database.py index af39ac9e..cb01301e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -29,7 +29,13 @@ AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) class Database: def __init__( - self, ds, path=None, is_mutable=True, is_memory=False, memory_name=None + self, + ds, + path=None, + is_mutable=True, + is_memory=False, + memory_name=None, + mode=None, ): self.name = None self.route = None @@ -50,6 +56,7 @@ class Database: self._write_connection = None # This is used to track all file connections so they can be closed self._all_file_connections = [] + self.mode = mode @property def cached_table_counts(self): @@ -90,6 +97,7 @@ class Database: return conn if self.is_memory: return sqlite3.connect(":memory:", uri=True) + # mode=ro or immutable=1? if self.is_mutable: qs = "?mode=ro" @@ -100,6 +108,8 @@ class Database: assert not (write and not self.is_mutable) if write: qs = "" + if self.mode is not None: + qs = f"?mode={self.mode}" conn = sqlite3.connect( f"file:{self.path}{qs}", uri=True, check_same_thread=False ) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 63a66c3c..f0b086e9 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -146,8 +146,6 @@ async def _resolve_metadata_view_permissions(datasette, actor, action, resource) if allow is not None: return actor_matches_allow(actor, allow) elif action == "view-database": - if resource == "_internal" and (actor is None or actor.get("id") != "root"): - return False database_allow = datasette.metadata("allow", database=resource) if database_allow is None: return None diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index e4b49e80..215695ca 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -5,13 +5,13 @@ from datasette.utils import table_column_details async def init_internal_db(db): create_tables_sql = textwrap.dedent( """ - CREATE TABLE IF NOT EXISTS databases ( + CREATE TABLE IF NOT EXISTS core_databases ( database_name TEXT PRIMARY KEY, path TEXT, is_memory INTEGER, schema_version INTEGER ); - CREATE TABLE IF NOT EXISTS tables ( + CREATE TABLE IF NOT EXISTS core_tables ( database_name TEXT, table_name TEXT, rootpage INTEGER, @@ -19,7 +19,7 @@ async def init_internal_db(db): PRIMARY KEY (database_name, table_name), FOREIGN KEY (database_name) REFERENCES databases(database_name) ); - CREATE TABLE IF NOT EXISTS columns ( + CREATE TABLE IF NOT EXISTS core_columns ( database_name TEXT, table_name TEXT, cid INTEGER, @@ -33,7 +33,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS indexes ( + CREATE TABLE IF NOT EXISTS core_indexes ( database_name TEXT, table_name TEXT, seq INTEGER, @@ -45,7 +45,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS foreign_keys ( + CREATE TABLE IF NOT EXISTS core_foreign_keys ( database_name TEXT, table_name TEXT, id INTEGER, @@ -69,12 +69,16 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - conn.execute("DELETE FROM tables WHERE database_name = ?", [database_name]) - conn.execute("DELETE FROM columns WHERE database_name = ?", [database_name]) + conn.execute("DELETE FROM core_tables WHERE database_name = ?", [database_name]) conn.execute( - "DELETE FROM foreign_keys WHERE database_name = ?", [database_name] + "DELETE FROM core_columns WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM core_foreign_keys WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM core_indexes WHERE database_name = ?", [database_name] ) - conn.execute("DELETE FROM indexes WHERE database_name = ?", [database_name]) await internal_db.execute_write_fn(delete_everything) @@ -133,14 +137,14 @@ async def populate_schema_tables(internal_db, db): await internal_db.execute_write_many( """ - INSERT INTO tables (database_name, table_name, rootpage, sql) + INSERT INTO core_tables (database_name, table_name, rootpage, sql) values (?, ?, ?, ?) """, tables_to_insert, ) await internal_db.execute_write_many( """ - INSERT INTO columns ( + INSERT INTO core_columns ( database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden ) VALUES ( :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden @@ -150,7 +154,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO foreign_keys ( + INSERT INTO core_foreign_keys ( database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match ) VALUES ( :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match @@ -160,7 +164,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO indexes ( + INSERT INTO core_indexes ( database_name, table_name, seq, name, "unique", origin, partial ) VALUES ( :database_name, :table_name, :seq, :name, :unique, :origin, :partial diff --git a/datasette/views/database.py b/datasette/views/database.py index d9abc38a..4647bedc 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -950,9 +950,9 @@ class TableCreateView(BaseView): async def _table_columns(datasette, database_name): - internal = datasette.get_database("_internal") - result = await internal.execute( - "select table_name, name from columns where database_name = ?", + internal_db = datasette.get_internal_database() + result = await internal_db.execute( + "select table_name, name from core_columns where database_name = ?", [database_name], ) table_columns = {} diff --git a/datasette/views/special.py b/datasette/views/special.py index c45a3eca..c1b84f8f 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -238,7 +238,7 @@ class CreateTokenView(BaseView): # Build list of databases and tables the user has permission to view database_with_tables = [] for database in self.ds.databases.values(): - if database.name in ("_internal", "_memory"): + if database.name == "_memory": continue if not await self.ds.permission_allowed( request.actor, "view-database", database.name diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5657f480..8e333447 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -134,6 +134,8 @@ Once started you can access it at ``http://localhost:8001`` mode --ssl-keyfile TEXT SSL key file --ssl-certfile TEXT SSL certificate file + --internal PATH Path to a persistent Datasette internal SQLite + database --help Show this message and exit. diff --git a/docs/internals.rst b/docs/internals.rst index 4b82e11c..fe9a2fa7 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -271,7 +271,7 @@ Property exposing a ``collections.OrderedDict`` of databases currently connected The dictionary keys are the name of the database that is used in the URL - e.g. ``/fixtures`` would have a key of ``"fixtures"``. The values are :ref:`internals_database` instances. -All databases are listed, irrespective of user permissions. This means that the ``_internal`` database will always be listed here. +All databases are listed, irrespective of user permissions. .. _datasette_permissions: @@ -479,6 +479,13 @@ The following example creates a token that can access ``view-instance`` and ``vi Returns the specified database object. Raises a ``KeyError`` if the database does not exist. Call this method without an argument to return the first connected database. +.. _get_internal_database: + +.get_internal_database() +------------------------ + +Returns a database object for reading and writing to the private :ref:`internal database `. + .. _datasette_add_database: .add_database(db, name=None, route=None) @@ -1127,19 +1134,21 @@ You can selectively disable CSRF protection using the :ref:`plugin_hook_skip_csr .. _internals_internal: -The _internal database -====================== +Datasette's internal database +============================= -.. warning:: - This API should be considered unstable - the structure of these tables may change prior to the release of Datasette 1.0. +Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a `--internal` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. -Datasette maintains an in-memory SQLite database with details of the the databases, tables and columns for all of the attached databases. +The internal database is not exposed in the Datasette application by default, which means private data can safely be stored without worry of accidentally leaking information through the default Datasette interface and API. However, other plugins do have full read and write access to the internal database. -By default all actors are denied access to the ``view-database`` permission for the ``_internal`` database, so the database is not visible to anyone unless they :ref:`sign in as root `. +Plugins can access this database by calling ``internal_db = datasette.get_internal_database()`` and then executing queries using the :ref:`Database API `. -Plugins can access this database by calling ``db = datasette.get_database("_internal")`` and then executing queries using the :ref:`Database API `. +Plugin authors are asked to practice good etiquette when using the internal database, as all plugins use the same database to store data. For example: -You can explore an example of this database by `signing in as root `__ to the ``latest.datasette.io`` demo instance and then navigating to `latest.datasette.io/_internal `__. +1. Use a unique prefix when creating tables, indices, and triggera in the internal database. If your plugin is called `datasette-xyz`, then prefix names with `datasette_xyz_*`. +2. Avoid long-running write statements that may stall or block other plugins that are trying to write at the same time. +3. Use temporary tables or shared in-memory attached databases when possible. +4. Avoid implementing features that could expose private data stored in the internal database by other plugins. .. _internals_utils: diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index d588342c..bb82b8c1 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -120,7 +120,7 @@ def permission_allowed(datasette, actor, action): assert ( 2 == ( - await datasette.get_database("_internal").execute("select 1 + 1") + await datasette.get_internal_database().execute("select 1 + 1") ).first()[0] ) if action == "this_is_allowed_async": @@ -142,7 +142,8 @@ def startup(datasette): async def inner(): # Run against _internal so tests that use the ds_client fixture # (which has no databases yet on startup) do not fail: - result = await datasette.get_database("_internal").execute("select 1 + 1") + internal_db = datasette.get_internal_database() + result = await internal_db.execute("select 1 + 1") datasette._startup_hook_calculation = result.first()[0] return inner diff --git a/tests/test_cli.py b/tests/test_cli.py index d9a10f22..e85bcef1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -154,6 +154,7 @@ def test_metadata_yaml(): ssl_keyfile=None, ssl_certfile=None, return_instance=True, + internal=None, ) client = _TestClient(ds) response = client.get("/-/metadata.json") @@ -368,3 +369,14 @@ def test_help_settings(): result = runner.invoke(cli, ["--help-settings"]) for setting in SETTINGS: assert setting.name in result.output + + +def test_internal_db(tmpdir): + runner = CliRunner() + internal_path = tmpdir / "internal.db" + assert not internal_path.exists() + result = runner.invoke( + cli, ["--memory", "--internal", str(internal_path), "--get", "/"] + ) + assert result.exit_code == 0 + assert internal_path.exists() diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index a666dd72..5276dc99 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -1,55 +1,35 @@ import pytest -@pytest.mark.asyncio -async def test_internal_only_available_to_root(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - assert (await ds_client.get("/_internal")).status_code == 403 - assert ( - await ds_client.get("/_internal", cookies={"ds_actor": cookie}) - ).status_code == 200 +# ensure refresh_schemas() gets called before interacting with internal_db +async def ensure_internal(ds_client): + await ds_client.get("/fixtures.json?sql=select+1") + return ds_client.ds.get_internal_database() @pytest.mark.asyncio async def test_internal_databases(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - databases = ( - await ds_client.get( - "/_internal/databases.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() - assert len(databases) == 2 - internal, fixtures = databases - assert internal["database_name"] == "_internal" - assert internal["is_memory"] == 1 - assert internal["path"] is None - assert isinstance(internal["schema_version"], int) - assert fixtures["database_name"] == "fixtures" + internal_db = await ensure_internal(ds_client) + databases = await internal_db.execute("select * from core_databases") + assert len(databases) == 1 + assert databases.rows[0]["database_name"] == "fixtures" @pytest.mark.asyncio async def test_internal_tables(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - tables = ( - await ds_client.get( - "/_internal/tables.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() + internal_db = await ensure_internal(ds_client) + tables = await internal_db.execute("select * from core_tables") assert len(tables) > 5 - table = tables[0] + table = tables.rows[0] assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"} @pytest.mark.asyncio async def test_internal_indexes(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - indexes = ( - await ds_client.get( - "/_internal/indexes.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() + internal_db = await ensure_internal(ds_client) + indexes = await internal_db.execute("select * from core_indexes") assert len(indexes) > 5 - index = indexes[0] + index = indexes.rows[0] assert set(index.keys()) == { "partial", "name", @@ -63,14 +43,10 @@ async def test_internal_indexes(ds_client): @pytest.mark.asyncio async def test_internal_foreign_keys(ds_client): - cookie = ds_client.actor_cookie({"id": "root"}) - foreign_keys = ( - await ds_client.get( - "/_internal/foreign_keys.json?_shape=array", cookies={"ds_actor": cookie} - ) - ).json() + internal_db = await ensure_internal(ds_client) + foreign_keys = await internal_db.execute("select * from core_foreign_keys") assert len(foreign_keys) > 5 - foreign_key = foreign_keys[0] + foreign_key = foreign_keys.rows[0] assert set(foreign_key.keys()) == { "table", "seq", diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 28fe720f..9761fa53 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -329,7 +329,7 @@ def test_hook_extra_body_script(app_client, path, expected_extra_body_script): @pytest.mark.asyncio async def test_hook_asgi_wrapper(ds_client): response = await ds_client.get("/fixtures") - assert "_internal, fixtures" == response.headers["x-databases"] + assert "fixtures" == response.headers["x-databases"] def test_hook_extra_template_vars(restore_working_directory): From a1f3d75a527b222cf1df51c41e1c424b38428a99 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 28 Aug 2023 20:46:12 -0700 Subject: [PATCH 068/767] Need to stick to Python 3.9 for gcloud --- .github/workflows/deploy-latest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 8cd9dcda..0dfa5a60 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -17,8 +17,9 @@ jobs: uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 + # gcloud commmand breaks on higher Python versions, so stick with 3.9: with: - python-version: "3.11" + python-version: "3.9" - uses: actions/cache@v3 name: Configure pip caching with: From 50da908213a0fc405ecd7a40090dfea7a2e7395c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Aug 2023 09:32:34 -0700 Subject: [PATCH 069/767] Cascade for restricted token view-table/view-database/view-instance operations (#2154) Closes #2102 * Permission is now a dataclass, not a namedtuple - refs https://github.com/simonw/datasette/pull/2154/#discussion_r1308087800 * datasette.get_permission() method --- datasette/app.py | 14 ++ datasette/default_permissions.py | 241 +++++++++++++++++++++++------- datasette/permissions.py | 20 ++- datasette/views/special.py | 12 +- docs/internals.rst | 12 +- docs/plugin_hooks.rst | 14 +- tests/test_internals_datasette.py | 14 ++ tests/test_permissions.py | 194 +++++++++++++++++++++++- 8 files changed, 449 insertions(+), 72 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 4deb8697..d95ec2bf 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -431,6 +431,20 @@ class Datasette: self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self) + def get_permission(self, name_or_abbr: str) -> "Permission": + """ + Returns a Permission object for the given name or abbreviation. Raises KeyError if not found. + """ + if name_or_abbr in self.permissions: + return self.permissions[name_or_abbr] + # Try abbreviation + for permission in self.permissions.values(): + if permission.abbr == name_or_abbr: + return permission + raise KeyError( + "No permission found with name or abbreviation {}".format(name_or_abbr) + ) + async def refresh_schemas(self): if self._refresh_schemas_lock.locked(): return diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index f0b086e9..960429fc 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -2,6 +2,7 @@ from datasette import hookimpl, Permission from datasette.utils import actor_matches_allow import itsdangerous import time +from typing import Union, Tuple @hookimpl @@ -9,32 +10,112 @@ def register_permissions(): return ( # name, abbr, description, takes_database, takes_resource, default Permission( - "view-instance", "vi", "View Datasette instance", False, False, True - ), - Permission("view-database", "vd", "View database", True, False, True), - Permission( - "view-database-download", "vdd", "Download database file", True, False, True - ), - Permission("view-table", "vt", "View table", True, True, True), - Permission("view-query", "vq", "View named query results", True, True, True), - Permission( - "execute-sql", "es", "Execute read-only SQL queries", True, False, True + name="view-instance", + abbr="vi", + description="View Datasette instance", + takes_database=False, + takes_resource=False, + default=True, ), Permission( - "permissions-debug", - "pd", - "Access permission debug tool", - False, - False, - False, + name="view-database", + abbr="vd", + description="View database", + takes_database=True, + takes_resource=False, + default=True, + implies_can_view=True, + ), + Permission( + name="view-database-download", + abbr="vdd", + description="Download database file", + takes_database=True, + takes_resource=False, + default=True, + ), + Permission( + name="view-table", + abbr="vt", + description="View table", + takes_database=True, + takes_resource=True, + default=True, + implies_can_view=True, + ), + Permission( + name="view-query", + abbr="vq", + description="View named query results", + takes_database=True, + takes_resource=True, + default=True, + implies_can_view=True, + ), + Permission( + name="execute-sql", + abbr="es", + description="Execute read-only SQL queries", + takes_database=True, + takes_resource=False, + default=True, + ), + Permission( + name="permissions-debug", + abbr="pd", + description="Access permission debug tool", + takes_database=False, + takes_resource=False, + default=False, + ), + Permission( + name="debug-menu", + abbr="dm", + description="View debug menu items", + takes_database=False, + takes_resource=False, + default=False, + ), + Permission( + name="insert-row", + abbr="ir", + description="Insert rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="delete-row", + abbr="dr", + description="Delete rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="update-row", + abbr="ur", + description="Update rows", + takes_database=True, + takes_resource=True, + default=False, + ), + Permission( + name="create-table", + abbr="ct", + description="Create tables", + takes_database=True, + takes_resource=False, + default=False, + ), + Permission( + name="drop-table", + abbr="dt", + description="Drop tables", + takes_database=True, + takes_resource=True, + default=False, ), - Permission("debug-menu", "dm", "View debug menu items", False, False, False), - # Write API permissions - Permission("insert-row", "ir", "Insert rows", True, True, False), - Permission("delete-row", "dr", "Delete rows", True, True, False), - Permission("update-row", "ur", "Update rows", True, True, False), - Permission("create-table", "ct", "Create tables", True, False, False), - Permission("drop-table", "dt", "Drop tables", True, True, False), ) @@ -176,6 +257,80 @@ async def _resolve_metadata_view_permissions(datasette, actor, action, resource) return actor_matches_allow(actor, database_allow_sql) +def restrictions_allow_action( + datasette: "Datasette", + restrictions: dict, + action: str, + resource: Union[str, Tuple[str, str]], +): + "Do these restrictions allow the requested action against the requested resource?" + if action == "view-instance": + # Special case for view-instance: it's allowed if the restrictions include any + # permissions that have the implies_can_view=True flag set + all_rules = restrictions.get("a") or [] + for database_rules in (restrictions.get("d") or {}).values(): + all_rules += database_rules + for database_resource_rules in (restrictions.get("r") or {}).values(): + for resource_rules in database_resource_rules.values(): + all_rules += resource_rules + permissions = [datasette.get_permission(action) for action in all_rules] + if any(p for p in permissions if p.implies_can_view): + return True + + if action == "view-database": + # Special case for view-database: it's allowed if the restrictions include any + # permissions that have the implies_can_view=True flag set AND takes_database + all_rules = restrictions.get("a") or [] + database_rules = list((restrictions.get("d") or {}).get(resource) or []) + all_rules += database_rules + resource_rules = ((restrictions.get("r") or {}).get(resource) or {}).values() + for resource_rules in (restrictions.get("r") or {}).values(): + for table_rules in resource_rules.values(): + all_rules += table_rules + permissions = [datasette.get_permission(action) for action in all_rules] + if any(p for p in permissions if p.implies_can_view and p.takes_database): + return True + + # Does this action have an abbreviation? + to_check = {action} + permission = datasette.permissions.get(action) + if permission and permission.abbr: + to_check.add(permission.abbr) + + # If restrictions is defined then we use those to further restrict the actor + # Crucially, we only use this to say NO (return False) - we never + # use it to return YES (True) because that might over-ride other + # restrictions placed on this actor + all_allowed = restrictions.get("a") + if all_allowed is not None: + assert isinstance(all_allowed, list) + if to_check.intersection(all_allowed): + return True + # How about for the current database? + if resource: + if isinstance(resource, str): + database_name = resource + else: + database_name = resource[0] + database_allowed = restrictions.get("d", {}).get(database_name) + if database_allowed is not None: + assert isinstance(database_allowed, list) + if to_check.intersection(database_allowed): + return True + # Or the current table? That's any time the resource is (database, table) + if resource is not None and not isinstance(resource, str) and len(resource) == 2: + database, table = resource + table_allowed = restrictions.get("r", {}).get(database, {}).get(table) + # TODO: What should this do for canned queries? + if table_allowed is not None: + assert isinstance(table_allowed, list) + if to_check.intersection(table_allowed): + return True + + # This action is not specifically allowed, so reject it + return False + + @hookimpl(specname="permission_allowed") def permission_allowed_actor_restrictions(datasette, actor, action, resource): if actor is None: @@ -184,40 +339,12 @@ def permission_allowed_actor_restrictions(datasette, actor, action, resource): # No restrictions, so we have no opinion return None _r = actor.get("_r") - - # Does this action have an abbreviation? - to_check = {action} - permission = datasette.permissions.get(action) - if permission and permission.abbr: - to_check.add(permission.abbr) - - # If _r is defined then we use those to further restrict the actor - # Crucially, we only use this to say NO (return False) - we never - # use it to return YES (True) because that might over-ride other - # restrictions placed on this actor - all_allowed = _r.get("a") - if all_allowed is not None: - assert isinstance(all_allowed, list) - if to_check.intersection(all_allowed): - return None - # How about for the current database? - if isinstance(resource, str): - database_allowed = _r.get("d", {}).get(resource) - if database_allowed is not None: - assert isinstance(database_allowed, list) - if to_check.intersection(database_allowed): - return None - # Or the current table? That's any time the resource is (database, table) - if resource is not None and not isinstance(resource, str) and len(resource) == 2: - database, table = resource - table_allowed = _r.get("r", {}).get(database, {}).get(table) - # TODO: What should this do for canned queries? - if table_allowed is not None: - assert isinstance(table_allowed, list) - if to_check.intersection(table_allowed): - return None - # This action is not specifically allowed, so reject it - return False + if restrictions_allow_action(datasette, _r, action, resource): + # Return None because we do not have an opinion here + return None + else: + # Block this permission check + return False @hookimpl diff --git a/datasette/permissions.py b/datasette/permissions.py index 1cd3474d..152f1721 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -1,6 +1,16 @@ -import collections +from dataclasses import dataclass, fields +from typing import Optional -Permission = collections.namedtuple( - "Permission", - ("name", "abbr", "description", "takes_database", "takes_resource", "default"), -) + +@dataclass +class Permission: + name: str + abbr: Optional[str] + description: Optional[str] + takes_database: bool + takes_resource: bool + default: bool + # This is deliberately undocumented: it's considered an internal + # implementation detail for view-table/view-database and should + # not be used by plugins as it may change in the future. + implies_can_view: bool = False diff --git a/datasette/views/special.py b/datasette/views/special.py index c1b84f8f..849750bf 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -122,7 +122,17 @@ class PermissionsDebugView(BaseView): # list() avoids error if check is performed during template render: { "permission_checks": list(reversed(self.ds._permission_checks)), - "permissions": list(self.ds.permissions.values()), + "permissions": [ + ( + p.name, + p.abbr, + p.description, + p.takes_database, + p.takes_resource, + p.default, + ) + for p in self.ds.permissions.values() + ], }, ) diff --git a/docs/internals.rst b/docs/internals.rst index fe9a2fa7..474e3328 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -280,7 +280,7 @@ All databases are listed, irrespective of user permissions. Property exposing a dictionary of permissions that have been registered using the :ref:`plugin_register_permissions` plugin hook. -The dictionary keys are the permission names - e.g. ``view-instance`` - and the values are ``Permission()`` named tuples describing the permission. Here is a :ref:`description of that tuple `. +The dictionary keys are the permission names - e.g. ``view-instance`` - and the values are ``Permission()`` objects describing the permission. Here is a :ref:`description of that object `. .. _datasette_plugin_config: @@ -469,6 +469,16 @@ The following example creates a token that can access ``view-instance`` and ``vi }, ) +.. _datasette_get_permission: + +.get_permission(name_or_abbr) +----------------------------- + +``name_or_abbr`` - string + The name or abbreviation of the permission to look up, e.g. ``view-table`` or ``vt``. + +Returns a :ref:`Permission object ` representing the permission, or raises a ``KeyError`` if one is not found. + .. _datasette_get_database: .get_database(name) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 497508ae..f8bb203d 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -794,24 +794,24 @@ If your plugin needs to register additional permissions unique to that plugin - ) ] -The fields of the ``Permission`` named tuple are as follows: +The fields of the ``Permission`` class are as follows: -``name`` +``name`` - string The name of the permission, e.g. ``upload-csvs``. This should be unique across all plugins that the user might have installed, so choose carefully. -``abbr`` +``abbr`` - string or None An abbreviation of the permission, e.g. ``uc``. This is optional - you can set it to ``None`` if you do not want to pick an abbreviation. Since this needs to be unique across all installed plugins it's best not to specify an abbreviation at all. If an abbreviation is provided it will be used when creating restricted signed API tokens. -``description`` +``description`` - string or None A human-readable description of what the permission lets you do. Should make sense as the second part of a sentence that starts "A user with this permission can ...". -``takes_database`` +``takes_database`` - boolean ``True`` if this permission can be granted on a per-database basis, ``False`` if it is only valid at the overall Datasette instance level. -``takes_resource`` +``takes_resource`` - boolean ``True`` if this permission can be granted on a per-resource basis. A resource is a database table, SQL view or :ref:`canned query `. -``default`` +``default`` - boolean The default value for this permission if it is not explicitly granted to a user. ``True`` means the permission is granted by default, ``False`` means it is not. This should only be ``True`` if you want anonymous users to be able to take this action. diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index d59ff729..c11e840c 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -159,3 +159,17 @@ def test_datasette_error_if_string_not_list(tmpdir): db_path = str(tmpdir / "data.db") with pytest.raises(ValueError): ds = Datasette(db_path) + + +@pytest.mark.asyncio +async def test_get_permission(ds_client): + ds = ds_client.ds + for name_or_abbr in ("vi", "view-instance", "vt", "view-table"): + permission = ds.get_permission(name_or_abbr) + if "-" in name_or_abbr: + assert permission.name == name_or_abbr + else: + assert permission.abbr == name_or_abbr + # And test KeyError + with pytest.raises(KeyError): + ds.get_permission("missing-permission") diff --git a/tests/test_permissions.py b/tests/test_permissions.py index f940d486..cad0525f 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,6 +1,7 @@ import collections from datasette.app import Datasette from datasette.cli import cli +from datasette.default_permissions import restrictions_allow_action from .fixtures import app_client, assert_permissions_checked, make_app_client from click.testing import CliRunner from bs4 import BeautifulSoup as Soup @@ -35,6 +36,8 @@ async def perms_ds(): one = ds.add_memory_database("perms_ds_one") two = ds.add_memory_database("perms_ds_two") await one.execute_write("create table if not exists t1 (id integer primary key)") + await one.execute_write("insert or ignore into t1 (id) values (1)") + await one.execute_write("create view if not exists v1 as select * from t1") await one.execute_write("create table if not exists t2 (id integer primary key)") await two.execute_write("create table if not exists t1 (id integer primary key)") return ds @@ -585,7 +588,6 @@ DEF = "USE_DEFAULT" ({"id": "t", "_r": {"a": ["vd"]}}, "view-database", "one", None, DEF), ({"id": "t", "_r": {"a": ["vt"]}}, "view-table", "one", "t1", DEF), # But not if it's the wrong permission - ({"id": "t", "_r": {"a": ["vd"]}}, "view-instance", None, None, False), ({"id": "t", "_r": {"a": ["vi"]}}, "view-database", "one", None, False), ({"id": "t", "_r": {"a": ["vd"]}}, "view-table", "one", "t1", False), # Works at the "d" for database level: @@ -629,11 +631,14 @@ DEF = "USE_DEFAULT" "t1", DEF, ), + # view-instance is granted if you have view-database + ({"id": "t", "_r": {"a": ["vd"]}}, "view-instance", None, None, DEF), ), ) async def test_actor_restricted_permissions( perms_ds, actor, permission, resource_1, resource_2, expected_result ): + perms_ds.pdb = True cookies = {"ds_actor": perms_ds.sign({"a": {"id": "root"}}, "actor")} csrftoken = (await perms_ds.client.get("/-/permissions", cookies=cookies)).cookies[ "ds_csrftoken" @@ -1018,3 +1023,190 @@ async def test_api_explorer_visibility( assert response.status_code == 403 finally: perms_ds._metadata_local = prev_metadata + + +@pytest.mark.asyncio +async def test_view_table_token_can_access_table(perms_ds): + actor = { + "id": "restricted-token", + "token": "dstok", + # Restricted to just view-table on perms_ds_two/t1 + "_r": {"r": {"perms_ds_two": {"t1": ["vt"]}}}, + } + cookies = {"ds_actor": perms_ds.client.actor_cookie(actor)} + response = await perms_ds.client.get("/perms_ds_two/t1.json", cookies=cookies) + assert response.status_code == 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "restrictions,verb,path,body,expected_status", + ( + # No restrictions + (None, "get", "/.json", None, 200), + (None, "get", "/perms_ds_one.json", None, 200), + (None, "get", "/perms_ds_one/t1.json", None, 200), + (None, "get", "/perms_ds_one/t1/1.json", None, 200), + (None, "get", "/perms_ds_one/v1.json", None, 200), + # Restricted to just view-instance + ({"a": ["vi"]}, "get", "/.json", None, 200), + ({"a": ["vi"]}, "get", "/perms_ds_one.json", None, 403), + ({"a": ["vi"]}, "get", "/perms_ds_one/t1.json", None, 403), + ({"a": ["vi"]}, "get", "/perms_ds_one/t1/1.json", None, 403), + ({"a": ["vi"]}, "get", "/perms_ds_one/v1.json", None, 403), + # Restricted to just view-database + ({"a": ["vd"]}, "get", "/.json", None, 200), # Can see instance too + ({"a": ["vd"]}, "get", "/perms_ds_one.json", None, 200), + ({"a": ["vd"]}, "get", "/perms_ds_one/t1.json", None, 403), + ({"a": ["vd"]}, "get", "/perms_ds_one/t1/1.json", None, 403), + ({"a": ["vd"]}, "get", "/perms_ds_one/v1.json", None, 403), + # Restricted to just view-table for specific database + ( + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/.json", + None, + 200, + ), # Can see instance + ( + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_one.json", + None, + 200, + ), # and this database + ( + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_two.json", + None, + 403, + ), # But not this one + ( + # Can see the table + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_one/t1.json", + None, + 200, + ), + ( + # And the view + {"d": {"perms_ds_one": ["vt"]}}, + "get", + "/perms_ds_one/v1.json", + None, + 200, + ), + # view-table access to a specific table + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/.json", + None, + 200, + ), + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one.json", + None, + 200, + ), + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one/t1.json", + None, + 200, + ), + # But cannot see the other table + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one/t2.json", + None, + 403, + ), + # Or the view + ( + {"r": {"perms_ds_one": {"t1": ["vt"]}}}, + "get", + "/perms_ds_one/v1.json", + None, + 403, + ), + ), +) +async def test_actor_restrictions( + perms_ds, restrictions, verb, path, body, expected_status +): + actor = {"id": "user"} + if restrictions: + actor["_r"] = restrictions + method = getattr(perms_ds.client, verb) + kwargs = {"cookies": {"ds_actor": perms_ds.client.actor_cookie(actor)}} + if body: + kwargs["json"] = body + perms_ds._permission_checks.clear() + response = await method(path, **kwargs) + assert response.status_code == expected_status, json.dumps( + { + "verb": verb, + "path": path, + "body": body, + "restrictions": restrictions, + "expected_status": expected_status, + "response_status": response.status_code, + "checks": [ + { + "action": check["action"], + "resource": check["resource"], + "result": check["result"], + } + for check in perms_ds._permission_checks + ], + }, + indent=2, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "restrictions,action,resource,expected", + ( + ({"a": ["view-instance"]}, "view-instance", None, True), + # view-table and view-database implies view-instance + ({"a": ["view-table"]}, "view-instance", None, True), + ({"a": ["view-database"]}, "view-instance", None, True), + # update-row does not imply view-instance + ({"a": ["update-row"]}, "view-instance", None, False), + # view-table on a resource implies view-instance + ({"r": {"db1": {"t1": ["view-table"]}}}, "view-instance", None, True), + # update-row on a resource does not imply view-instance + ({"r": {"db1": {"t1": ["update-row"]}}}, "view-instance", None, False), + # view-database on a resource implies view-instance + ({"d": {"db1": ["view-database"]}}, "view-instance", None, True), + # Having view-table on "a" allows access to any specific table + ({"a": ["view-table"]}, "view-table", ("dbname", "tablename"), True), + # Ditto for on the database + ( + {"d": {"dbname": ["view-table"]}}, + "view-table", + ("dbname", "tablename"), + True, + ), + # But not if it's allowed on a different database + ( + {"d": {"dbname": ["view-table"]}}, + "view-table", + ("dbname2", "tablename"), + False, + ), + ), +) +async def test_restrictions_allow_action(restrictions, action, resource, expected): + ds = Datasette() + await ds.invoke_startup() + actual = restrictions_allow_action(ds, restrictions, action, resource) + assert actual == expected From bb12229794655abaa21a9aa691d1f85d34b6c45a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Aug 2023 10:01:28 -0700 Subject: [PATCH 070/767] Rename core_ to catalog_, closes #2163 --- datasette/app.py | 4 ++-- datasette/utils/internal_db.py | 28 +++++++++++++++------------- datasette/views/database.py | 2 +- docs/internals.rst | 2 ++ tests/test_internal_db.py | 8 ++++---- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index d95ec2bf..0227f627 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -459,7 +459,7 @@ class Datasette: current_schema_versions = { row["database_name"]: row["schema_version"] for row in await internal_db.execute( - "select database_name, schema_version from core_databases" + "select database_name, schema_version from catalog_databases" ) } for database_name, db in self.databases.items(): @@ -474,7 +474,7 @@ class Datasette: values = [database_name, db.is_memory, schema_version] await internal_db.execute_write( """ - INSERT OR REPLACE INTO core_databases (database_name, path, is_memory, schema_version) + INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version) VALUES {} """.format( placeholders diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 215695ca..2e5ac53b 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -5,13 +5,13 @@ from datasette.utils import table_column_details async def init_internal_db(db): create_tables_sql = textwrap.dedent( """ - CREATE TABLE IF NOT EXISTS core_databases ( + CREATE TABLE IF NOT EXISTS catalog_databases ( database_name TEXT PRIMARY KEY, path TEXT, is_memory INTEGER, schema_version INTEGER ); - CREATE TABLE IF NOT EXISTS core_tables ( + CREATE TABLE IF NOT EXISTS catalog_tables ( database_name TEXT, table_name TEXT, rootpage INTEGER, @@ -19,7 +19,7 @@ async def init_internal_db(db): PRIMARY KEY (database_name, table_name), FOREIGN KEY (database_name) REFERENCES databases(database_name) ); - CREATE TABLE IF NOT EXISTS core_columns ( + CREATE TABLE IF NOT EXISTS catalog_columns ( database_name TEXT, table_name TEXT, cid INTEGER, @@ -33,7 +33,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS core_indexes ( + CREATE TABLE IF NOT EXISTS catalog_indexes ( database_name TEXT, table_name TEXT, seq INTEGER, @@ -45,7 +45,7 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES tables(database_name, table_name) ); - CREATE TABLE IF NOT EXISTS core_foreign_keys ( + CREATE TABLE IF NOT EXISTS catalog_foreign_keys ( database_name TEXT, table_name TEXT, id INTEGER, @@ -69,15 +69,17 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - conn.execute("DELETE FROM core_tables WHERE database_name = ?", [database_name]) conn.execute( - "DELETE FROM core_columns WHERE database_name = ?", [database_name] + "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] ) conn.execute( - "DELETE FROM core_foreign_keys WHERE database_name = ?", [database_name] + "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] ) conn.execute( - "DELETE FROM core_indexes WHERE database_name = ?", [database_name] + "DELETE FROM catalog_foreign_keys WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] ) await internal_db.execute_write_fn(delete_everything) @@ -137,14 +139,14 @@ async def populate_schema_tables(internal_db, db): await internal_db.execute_write_many( """ - INSERT INTO core_tables (database_name, table_name, rootpage, sql) + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) values (?, ?, ?, ?) """, tables_to_insert, ) await internal_db.execute_write_many( """ - INSERT INTO core_columns ( + INSERT INTO catalog_columns ( database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden ) VALUES ( :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden @@ -154,7 +156,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO core_foreign_keys ( + INSERT INTO catalog_foreign_keys ( database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match ) VALUES ( :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match @@ -164,7 +166,7 @@ async def populate_schema_tables(internal_db, db): ) await internal_db.execute_write_many( """ - INSERT INTO core_indexes ( + INSERT INTO catalog_indexes ( database_name, table_name, seq, name, "unique", origin, partial ) VALUES ( :database_name, :table_name, :seq, :name, :unique, :origin, :partial diff --git a/datasette/views/database.py b/datasette/views/database.py index 4647bedc..9ba5ce94 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -952,7 +952,7 @@ class TableCreateView(BaseView): async def _table_columns(datasette, database_name): internal_db = datasette.get_internal_database() result = await internal_db.execute( - "select table_name, name from core_columns where database_name = ?", + "select table_name, name from catalog_columns where database_name = ?", [database_name], ) table_columns = {} diff --git a/docs/internals.rst b/docs/internals.rst index 474e3328..743f5972 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1149,6 +1149,8 @@ Datasette's internal database Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a `--internal` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. +Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases. + The internal database is not exposed in the Datasette application by default, which means private data can safely be stored without worry of accidentally leaking information through the default Datasette interface and API. However, other plugins do have full read and write access to the internal database. Plugins can access this database by calling ``internal_db = datasette.get_internal_database()`` and then executing queries using the :ref:`Database API `. diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index 5276dc99..b41cabb4 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -10,7 +10,7 @@ async def ensure_internal(ds_client): @pytest.mark.asyncio async def test_internal_databases(ds_client): internal_db = await ensure_internal(ds_client) - databases = await internal_db.execute("select * from core_databases") + databases = await internal_db.execute("select * from catalog_databases") assert len(databases) == 1 assert databases.rows[0]["database_name"] == "fixtures" @@ -18,7 +18,7 @@ async def test_internal_databases(ds_client): @pytest.mark.asyncio async def test_internal_tables(ds_client): internal_db = await ensure_internal(ds_client) - tables = await internal_db.execute("select * from core_tables") + tables = await internal_db.execute("select * from catalog_tables") assert len(tables) > 5 table = tables.rows[0] assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"} @@ -27,7 +27,7 @@ async def test_internal_tables(ds_client): @pytest.mark.asyncio async def test_internal_indexes(ds_client): internal_db = await ensure_internal(ds_client) - indexes = await internal_db.execute("select * from core_indexes") + indexes = await internal_db.execute("select * from catalog_indexes") assert len(indexes) > 5 index = indexes.rows[0] assert set(index.keys()) == { @@ -44,7 +44,7 @@ async def test_internal_indexes(ds_client): @pytest.mark.asyncio async def test_internal_foreign_keys(ds_client): internal_db = await ensure_internal(ds_client) - foreign_keys = await internal_db.execute("select * from core_foreign_keys") + foreign_keys = await internal_db.execute("select * from catalog_foreign_keys") assert len(foreign_keys) > 5 foreign_key = foreign_keys.rows[0] assert set(foreign_key.keys()) == { From 30b28c8367a9c6870386ea10a202705b40862457 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 29 Aug 2023 10:17:54 -0700 Subject: [PATCH 071/767] Release 1.0a5 Refs #2093, #2102, #2153, #2156, #2157 --- datasette/version.py | 2 +- docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 1d003352..b99c212b 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a4" +__version__ = "1.0a5" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 937610bd..019d6c68 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v1_0_a5: + +1.0a5 (2023-08-29) +------------------ + +- When restrictions are applied to :ref:`API tokens `, those restrictions now behave slightly differently: applying the ``view-table`` restriction will imply the ability to ``view-database`` for the database containing that table, and both ``view-table`` and ``view-database`` will imply ``view-instance``. Previously you needed to create a token with restrictions that explicitly listed ``view-instance`` and ``view-database`` and ``view-table`` in order to view a table without getting a permission denied error. (:issue:`2102`) +- New ``datasette.yaml`` (or ``.json``) configuration file, which can be specified using ``datasette -c path-to-file``. The goal here to consolidate settings, plugin configuration, permissions, canned queries, and other Datasette configuration into a single single file, separate from ``metadata.yaml``. The legacy ``settings.json`` config file used for :ref:`config_dir` has been removed, and ``datasette.yaml`` has a ``"settings"`` section where the same settings key/value pairs can be included. In the next future alpha release, more configuration such as plugins/permissions/canned queries will be moved to the ``datasette.yaml`` file. See :issue:`2093` for more details. Thanks, Alex Garcia. +- The ``-s/--setting`` option can now take dotted paths to nested settings. These will then be used to set or over-ride the same options as are present in the new configuration file. (:issue:`2156`) +- New ``--actor '{"id": "json-goes-here"}'`` option for use with ``datasette --get`` to treat the simulated request as being made by a specific actor, see :ref:`cli_datasette_get`. (:issue:`2153`) +- The Datasette ``_internal`` database has had some changes. It no longer shows up in the ``datasette.databases`` list by default, and is now instead available to plugins using the ``datasette.get_internal_database()``. Plugins are invited to use this as a private database to store configuration and settings and secrets that should not be made visible through the default Datasette interface. Users can pass the new ``--internal internal.db`` option to persist that internal database to disk. Thanks, Alex Garcia. (:issue:`2157`). + .. _v1_0_a4: 1.0a4 (2023-08-21) From 6bfe104d47b888c70bfb7781f8f48ff11452b2b5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Aug 2023 15:12:24 -0700 Subject: [PATCH 072/767] DATASETTE_LOAD_PLUGINS environment variable for loading specific plugins Closes #2164 * Load only specified plugins for DATASETTE_LOAD_PLUGINS=datasette-one,datasette-two * Load no plugins if DATASETTE_LOAD_PLUGINS='' * Automated tests in a Bash script for DATASETTE_LOAD_PLUGINS --- .github/workflows/test.yml | 4 +++ datasette/plugins.py | 23 +++++++++++- docs/plugins.rst | 54 ++++++++++++++++++++++++++++ tests/test-datasette-load-plugins.sh | 29 +++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100755 tests/test-datasette-load-plugins.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8cbbb572..2784db86 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,3 +50,7 @@ jobs: run: | # This fails on syntax errors, or a diff was applied blacken-docs -l 60 docs/*.rst + - name: Test DATASETTE_LOAD_PLUGINS + run: | + pip install datasette-init datasette-json-html + tests/test-datasette-load-plugins.sh diff --git a/datasette/plugins.py b/datasette/plugins.py index fef0c8e9..6ec08a81 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,4 +1,5 @@ import importlib +import os import pluggy import pkg_resources import sys @@ -22,10 +23,30 @@ DEFAULT_PLUGINS = ( pm = pluggy.PluginManager("datasette") pm.add_hookspecs(hookspecs) -if not hasattr(sys, "_called_from_test"): +DATASETTE_LOAD_PLUGINS = os.environ.get("DATASETTE_LOAD_PLUGINS", None) + +if not hasattr(sys, "_called_from_test") and DATASETTE_LOAD_PLUGINS is None: # Only load plugins if not running tests pm.load_setuptools_entrypoints("datasette") +# Load any plugins specified in DATASETTE_LOAD_PLUGINS") +if DATASETTE_LOAD_PLUGINS is not None: + for package_name in [ + name for name in DATASETTE_LOAD_PLUGINS.split(",") if name.strip() + ]: + try: + distribution = pkg_resources.get_distribution(package_name) + entry_map = distribution.get_entry_map() + if "datasette" in entry_map: + for plugin_name, entry_point in entry_map["datasette"].items(): + mod = entry_point.load() + pm.register(mod, name=entry_point.name) + # Ensure name can be found in plugin_to_distinfo later: + pm._plugin_distinfo.append((mod, distribution)) + except pkg_resources.DistributionNotFound: + sys.stderr.write("Plugin {} could not be found\n".format(package_name)) + + # Load default plugins for plugin in DEFAULT_PLUGINS: mod = importlib.import_module(plugin) diff --git a/docs/plugins.rst b/docs/plugins.rst index 19bfdd0c..11db40af 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -81,6 +81,60 @@ You can use the name of a package on PyPI or any of the other valid arguments to datasette publish cloudrun mydb.db \ --install=https://url-to-my-package.zip + +.. _plugins_datasette_load_plugins: + +Controlling which plugins are loaded +------------------------------------ + +Datasette defaults to loading every plugin that is installed in the same virtual environment as Datasette itself. + +You can set the ``DATASETTE_LOAD_PLUGINS`` environment variable to a comma-separated list of plugin names to load a controlled subset of plugins instead. + +For example, to load just the ``datasette-vega`` and ``datasette-cluster-map`` plugins, set ``DATASETTE_LOAD_PLUGINS`` to ``datasette-vega,datasette-cluster-map``: + +.. code-block:: bash + + export DATASETTE_LOAD_PLUGINS='datasette-vega,datasette-cluster-map' + datasette mydb.db + +Or: + +.. code-block:: bash + + DATASETTE_LOAD_PLUGINS='datasette-vega,datasette-cluster-map' \ + datasette mydb.db + +To disable the loading of all additional plugins, set ``DATASETTE_LOAD_PLUGINS`` to an empty string: + +.. code-block:: bash + + export DATASETTE_LOAD_PLUGINS='' + datasette mydb.db + +A quick way to test this setting is to use it with the ``datasette plugins`` command: + +.. code-block:: bash + + DATASETTE_LOAD_PLUGINS='datasette-vega' datasette plugins + +This should output the following: + +.. code-block:: json + + [ + { + "name": "datasette-vega", + "static": true, + "templates": false, + "version": "0.6.2", + "hooks": [ + "extra_css_urls", + "extra_js_urls" + ] + } + ] + .. _plugins_installed: Seeing what plugins are installed diff --git a/tests/test-datasette-load-plugins.sh b/tests/test-datasette-load-plugins.sh new file mode 100755 index 00000000..e26d8377 --- /dev/null +++ b/tests/test-datasette-load-plugins.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# This should only run in environemnts where both +# datasette-init and datasette-json-html are installed + +PLUGINS=$(datasette plugins) +echo "$PLUGINS" | jq 'any(.[]; .name == "datasette-json-html")' | \ + grep -q true || ( \ + echo "Test failed: datasette-json-html not found" && \ + exit 1 \ + ) +# With the DATASETTE_LOAD_PLUGINS we should not see that +PLUGINS2=$(DATASETTE_LOAD_PLUGINS=datasette-init datasette plugins) +echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-json-html")' | \ + grep -q false || ( \ + echo "Test failed: datasette-json-html should not have been loaded" && \ + exit 1 \ + ) +echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-init")' | \ + grep -q true || ( \ + echo "Test failed: datasette-init should have been loaded" && \ + exit 1 \ + ) +# With DATASETTE_LOAD_PLUGINS='' we should see no plugins +PLUGINS3=$(DATASETTE_LOAD_PLUGINS='' datasette plugins) +echo "$PLUGINS3"| \ + grep -q '\[\]' || ( \ + echo "Test failed: datasette plugins should have returned []" && \ + exit 1 \ + ) From 2caa53a52a37e53f83e3a854fc721c7e26c5e9ff Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Aug 2023 16:19:24 -0700 Subject: [PATCH 073/767] ReST fix --- docs/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index 743f5972..d136ad5a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1147,7 +1147,7 @@ You can selectively disable CSRF protection using the :ref:`plugin_hook_skip_csr Datasette's internal database ============================= -Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a `--internal` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. +Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a ``--internal`` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances. Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases. From 4c3ef033110407f3b3dbce501659d523724985e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 30 Aug 2023 16:19:59 -0700 Subject: [PATCH 074/767] Another ReST fix --- docs/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index d136ad5a..540e7058 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1157,7 +1157,7 @@ Plugins can access this database by calling ``internal_db = datasette.get_intern Plugin authors are asked to practice good etiquette when using the internal database, as all plugins use the same database to store data. For example: -1. Use a unique prefix when creating tables, indices, and triggera in the internal database. If your plugin is called `datasette-xyz`, then prefix names with `datasette_xyz_*`. +1. Use a unique prefix when creating tables, indices, and triggera in the internal database. If your plugin is called ``datasette-xyz``, then prefix names with ``datasette_xyz_*``. 2. Avoid long-running write statements that may stall or block other plugins that are trying to write at the same time. 3. Use temporary tables or shared in-memory attached databases when possible. 4. Avoid implementing features that could expose private data stored in the internal database by other plugins. From 9cead33fb9c8704996181f1ab67c7376dee97f15 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Aug 2023 10:46:07 -0700 Subject: [PATCH 075/767] OperationalError: database table is locked fix See also: - https://til.simonwillison.net/datasette/remember-to-commit --- docs/internals.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index 540e7058..6b7d3df8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1012,6 +1012,7 @@ For example: def delete_and_return_count(conn): conn.execute("delete from some_table where id > 5") + conn.commit() return conn.execute( "select count(*) from some_table" ).fetchone()[0] @@ -1028,6 +1029,8 @@ The value returned from ``await database.execute_write_fn(...)`` will be the ret If your function raises an exception that exception will be propagated up to the ``await`` line. +If you see ``OperationalError: database table is locked`` errors you should check that you remembered to explicitly call ``conn.commit()`` in your write function. + If you specify ``block=False`` the method becomes fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. Any exceptions in your code will be silently swallowed. .. _database_close: From 98ffad9aed15a300e61fb712fa12f177844739b3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Aug 2023 15:46:18 -0700 Subject: [PATCH 076/767] execute-sql now implies can view instance/database, closes #2169 --- datasette/default_permissions.py | 1 + tests/test_permissions.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 960429fc..5a99d0d8 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -59,6 +59,7 @@ def register_permissions(): takes_database=True, takes_resource=False, default=True, + implies_can_view=True, ), Permission( name="permissions-debug", diff --git a/tests/test_permissions.py b/tests/test_permissions.py index cad0525f..b3987cff 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1183,6 +1183,10 @@ async def test_actor_restrictions( ({"a": ["update-row"]}, "view-instance", None, False), # view-table on a resource implies view-instance ({"r": {"db1": {"t1": ["view-table"]}}}, "view-instance", None, True), + # execute-sql on a database implies view-instance, view-database + ({"d": {"db1": ["es"]}}, "view-instance", None, True), + ({"d": {"db1": ["es"]}}, "view-database", "db1", True), + ({"d": {"db1": ["es"]}}, "view-database", "db2", False), # update-row on a resource does not imply view-instance ({"r": {"db1": {"t1": ["update-row"]}}}, "view-instance", None, False), # view-database on a resource implies view-instance From fd083e37ec53e7e625111168d324a572344a3b19 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 31 Aug 2023 16:06:30 -0700 Subject: [PATCH 077/767] Docs for plugins that define more plugin hooks, closes #1765 --- docs/writing_plugins.rst | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index a84789b5..a4c96011 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -325,3 +325,60 @@ This object is exposed in templates as the ``urls`` variable, which can be used Back to the Homepage See :ref:`internals_datasette_urls` for full details on this object. + +.. _writing_plugins_extra_hooks: + +Plugins that define new plugin hooks +------------------------------------ + +Plugins can define new plugin hooks that other plugins can use to further extend their functionality. + +`datasette-graphql `__ is one example of a plugin that does this. It defines a new hook called ``graphql_extra_fields``, `described here `__, which other plugins can use to define additional fields that should be included in the GraphQL schema. + +To define additional hooks, add a file to the plugin called ``datasette_your_plugin/hookspecs.py`` with content that looks like this: + +.. code-block:: python + + from pluggy import HookspecMarker + + hookspec = HookspecMarker("datasette") + + @hookspec + def name_of_your_hook_goes_here(datasette): + "Description of your hook." + +You should define your own hook name and arguments here, following the documentation for `Pluggy specifications `__. Make sure to pick a name that is unlikely to clash with hooks provided by any other plugins. + +Then, to register your plugin hooks, add the following code to your ``datasette_your_plugin/__init__.py`` file: + +.. code-block:: python + + from datasette.plugins import pm + from . import hookspecs + + pm.add_hookspecs(hookspecs) + +This will register your plugin hooks as part of the ``datasette`` plugin hook namespace. + +Within your plugin code you can trigger the hook using this pattern: + +.. code-block:: python + + from datasette.plugins import pm + + for plugin_return_value in pm.hook.name_of_your_hook_goes_here( + datasette=datasette + ): + # Do something with plugin_return_value + +Other plugins will then be able to register their own implementations of your hook using this syntax: + +.. code-block:: python + + from datasette import hookimpl + + @hookimpl + def name_of_your_hook_goes_here(datasette): + return "Response from this plugin hook" + +These plugin implementations can accept 0 or more of the named arguments that you defined in your hook specification. From 31d5c4ec05e27165283f0f0004c32227d8b78df8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Sep 2023 19:43:01 -0700 Subject: [PATCH 078/767] Contraction - Google and Microsoft styleguides like it I was trying out https://github.com/errata-ai/vale --- docs/authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 814d2e67..1a444d0c 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -4,7 +4,7 @@ Authentication and permissions ================================ -Datasette does not require authentication by default. Any visitor to a Datasette instance can explore the full data and execute read-only SQL queries. +Datasette doesn't require authentication by default. Any visitor to a Datasette instance can explore the full data and execute read-only SQL queries. Datasette's plugin system can be used to add many different styles of authentication, such as user accounts, single sign-on or API keys. From 05707aa16b5c6c39fbe48b3176b85a8ffe493938 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Sep 2023 19:50:09 -0700 Subject: [PATCH 079/767] click-default-group>=1.2.3 (#2173) * click-default-group>=1.2.3 Now available as a wheel: - https://github.com/click-contrib/click-default-group/issues/21 * Fix for blacken-docs --- docs/writing_plugins.rst | 7 ++++++- setup.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index a4c96011..d0dd8f36 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -343,6 +343,7 @@ To define additional hooks, add a file to the plugin called ``datasette_your_plu hookspec = HookspecMarker("datasette") + @hookspec def name_of_your_hook_goes_here(datasette): "Description of your hook." @@ -366,10 +367,13 @@ Within your plugin code you can trigger the hook using this pattern: from datasette.plugins import pm - for plugin_return_value in pm.hook.name_of_your_hook_goes_here( + for ( + plugin_return_value + ) in pm.hook.name_of_your_hook_goes_here( datasette=datasette ): # Do something with plugin_return_value + pass Other plugins will then be able to register their own implementations of your hook using this syntax: @@ -377,6 +381,7 @@ Other plugins will then be able to register their own implementations of your ho from datasette import hookimpl + @hookimpl def name_of_your_hook_goes_here(datasette): return "Response from this plugin hook" diff --git a/setup.py b/setup.py index 35c9b68b..7d8c1ebc 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( install_requires=[ "asgiref>=3.2.10", "click>=7.1.1", - "click-default-group-wheel>=1.2.2", + "click-default-group>=1.2.3", "Jinja2>=2.10.3", "hupper>=1.9", "httpx>=0.20", From e86eaaa4f371512689e973c18879298dab51f80a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 6 Sep 2023 09:16:27 -0700 Subject: [PATCH 080/767] Test against Python 3.12 preview (#2175) https://dev.to/hugovk/help-test-python-312-beta-1508/ --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2784db86..656b0b1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,13 +10,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - uses: actions/cache@v3 name: Configure pip caching with: From e4abae3fd7a828625d00c35c316852ffbaa5ef2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 09:34:31 -0700 Subject: [PATCH 081/767] Bump Sphinx (#2166) Bumps the python-packages group with 1 update: [sphinx](https://github.com/sphinx-doc/sphinx). - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.4...v7.2.5) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7d8c1ebc..c718086b 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "Sphinx==7.2.4", + "Sphinx==7.2.5", "furo==2023.8.19", "sphinx-autobuild", "codespell>=2.2.5", From fbcb103c0cb6668018ace539a01a6a1f156e8d6a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 07:47:24 -0700 Subject: [PATCH 082/767] Added example code to database_actions hook documentation --- docs/plugin_hooks.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f8bb203d..84a045b0 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1439,7 +1439,32 @@ database_actions(datasette, actor, database, request) This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. -Example: `datasette-graphql `_ +This example adds a new database action for creating a table, if the user has the ``edit-schema`` permission: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def database_actions(datasette, actor, database): + async def inner(): + if not await datasette.permission_allowed( + actor, "edit-schema", resource=database, default=False + ): + return [] + return [ + { + "href": datasette.urls.path( + "/-/edit-schema/{}/-/create".format(database) + ), + "label": "Create a table", + } + ] + + return inner + +Example: `datasette-graphql `_, `datasette-edit-schema `_ .. _plugin_hook_skip_csrf: From 2200abfa17f72b1cb741a36b44dc40a04b8ea001 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 15:49:50 -0700 Subject: [PATCH 083/767] Fix for flaky test_hidden_sqlite_stat1_table, closes #2179 --- tests/test_api.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index f96f571e..93ca43eb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1017,7 +1017,10 @@ async def test_hidden_sqlite_stat1_table(): await db.execute_write("analyze") data = (await ds.client.get("/db.json?_show_hidden=1")).json() tables = [(t["name"], t["hidden"]) for t in data["tables"]] - assert tables == [("normal", False), ("sqlite_stat1", True)] + assert tables in ( + [("normal", False), ("sqlite_stat1", True)], + [("normal", False), ("sqlite_stat1", True), ("sqlite_stat4", True)], + ) @pytest.mark.asyncio From dbfad6d2201bc65a0c73e699a10c479c1e199e11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 15:51:09 -0700 Subject: [PATCH 084/767] Foreign key label expanding respects table permissions, closes #2178 --- datasette/app.py | 9 ++++++- datasette/facets.py | 2 +- datasette/views/table.py | 2 +- tests/test_table_html.py | 53 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 0227f627..618c0ecc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -935,7 +935,7 @@ class Datasette: log_sql_errors=log_sql_errors, ) - async def expand_foreign_keys(self, database, table, column, values): + async def expand_foreign_keys(self, actor, database, table, column, values): """Returns dict mapping (column, value) -> label""" labeled_fks = {} db = self.databases[database] @@ -949,6 +949,13 @@ class Datasette: ][0] except IndexError: return {} + # Ensure user has permission to view the referenced table + if not await self.permission_allowed( + actor=actor, + action="view-table", + resource=(database, fk["other_table"]), + ): + return {} label_column = await db.label_column_for_table(fk["other_table"]) if not label_column: return {(fk["column"], value): str(value) for value in values} diff --git a/datasette/facets.py b/datasette/facets.py index 7fb0c68b..b23615fe 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -253,7 +253,7 @@ class ColumnFacet(Facet): # Attempt to expand foreign keys into labels values = [row["value"] for row in facet_rows] expanded = await self.ds.expand_foreign_keys( - self.database, self.table, column, values + self.request.actor, self.database, self.table, column, values ) else: expanded = {} diff --git a/datasette/views/table.py b/datasette/views/table.py index 6df8b915..50ba2b78 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1144,7 +1144,7 @@ async def table_view_data( # Expand them expanded_labels.update( await datasette.expand_foreign_keys( - database_name, table_name, column, values + request.actor, database_name, table_name, column, values ) ) if expanded_labels: diff --git a/tests/test_table_html.py b/tests/test_table_html.py index c4c7878c..e66eb6f0 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1204,3 +1204,56 @@ async def test_format_of_binary_links(size, title, length_bytes): sql_response = await ds.client.get("/{}".format(db_name), params={"sql": sql}) assert sql_response.status_code == 200 assert expected in sql_response.text + + +@pytest.mark.asyncio +async def test_foreign_key_labels_obey_permissions(): + ds = Datasette( + metadata={ + "databases": { + "foreign_key_labels": { + "tables": { + # Table a is only visible to root + "a": {"allow": {"id": "root"}}, + } + } + } + } + ) + db = ds.add_memory_database("foreign_key_labels") + await db.execute_write("create table a(id integer primary key, name text)") + await db.execute_write("insert into a (id, name) values (1, 'hello')") + await db.execute_write( + "create table b(id integer primary key, name text, a_id integer references a(id))" + ) + await db.execute_write("insert into b (id, name, a_id) values (1, 'world', 1)") + # Anonymous user can see table b but not table a + blah = await ds.client.get("/foreign_key_labels.json") + anon_a = await ds.client.get("/foreign_key_labels/a.json?_labels=on") + assert anon_a.status_code == 403 + anon_b = await ds.client.get("/foreign_key_labels/b.json?_labels=on") + assert anon_b.status_code == 200 + # root user can see both + cookies = {"ds_actor": ds.sign({"a": {"id": "root"}}, "actor")} + root_a = await ds.client.get( + "/foreign_key_labels/a.json?_labels=on", cookies=cookies + ) + assert root_a.status_code == 200 + root_b = await ds.client.get( + "/foreign_key_labels/b.json?_labels=on", cookies=cookies + ) + assert root_b.status_code == 200 + # Labels should have been expanded for root + assert root_b.json() == { + "ok": True, + "next": None, + "rows": [{"id": 1, "name": "world", "a_id": {"value": 1, "label": "hello"}}], + "truncated": False, + } + # But not for anon + assert anon_b.json() == { + "ok": True, + "next": None, + "rows": [{"id": 1, "name": "world", "a_id": 1}], + "truncated": False, + } From ab040470e2b191c0de48b213193da71e48cd66ed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 15:57:27 -0700 Subject: [PATCH 085/767] Applied blacken-docs --- docs/plugin_hooks.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 84a045b0..04fb24ce 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1450,13 +1450,18 @@ This example adds a new database action for creating a table, if the user has th def database_actions(datasette, actor, database): async def inner(): if not await datasette.permission_allowed( - actor, "edit-schema", resource=database, default=False + actor, + "edit-schema", + resource=database, + default=False, ): return [] return [ { "href": datasette.urls.path( - "/-/edit-schema/{}/-/create".format(database) + "/-/edit-schema/{}/-/create".format( + database + ) ), "label": "Create a table", } From c26370485a4fd4bf130da051be9163d92c57f24f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 16:28:30 -0700 Subject: [PATCH 086/767] Label expand permission check respects cascade, closes #2178 --- datasette/app.py | 22 ++++++++++------- tests/test_table_html.py | 52 +++++++++++++++++++++++++++++++++------- 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 618c0ecc..ea9739f0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -950,13 +950,19 @@ class Datasette: except IndexError: return {} # Ensure user has permission to view the referenced table - if not await self.permission_allowed( - actor=actor, - action="view-table", - resource=(database, fk["other_table"]), - ): + other_table = fk["other_table"] + other_column = fk["other_column"] + visible, _ = await self.check_visibility( + actor, + permissions=[ + ("view-table", (database, other_table)), + ("view-database", database), + "view-instance", + ], + ) + if not visible: return {} - label_column = await db.label_column_for_table(fk["other_table"]) + label_column = await db.label_column_for_table(other_table) if not label_column: return {(fk["column"], value): str(value) for value in values} labeled_fks = {} @@ -965,9 +971,9 @@ class Datasette: from {other_table} where {other_column} in ({placeholders}) """.format( - other_column=escape_sqlite(fk["other_column"]), + other_column=escape_sqlite(other_column), label_column=escape_sqlite(label_column), - other_table=escape_sqlite(fk["other_table"]), + other_table=escape_sqlite(other_table), placeholders=", ".join(["?"] * len(set(values))), ) try: diff --git a/tests/test_table_html.py b/tests/test_table_html.py index e66eb6f0..6707665d 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1207,9 +1207,11 @@ async def test_format_of_binary_links(size, title, length_bytes): @pytest.mark.asyncio -async def test_foreign_key_labels_obey_permissions(): - ds = Datasette( - metadata={ +@pytest.mark.parametrize( + "metadata", + ( + # Blocked at table level + { "databases": { "foreign_key_labels": { "tables": { @@ -1218,15 +1220,47 @@ async def test_foreign_key_labels_obey_permissions(): } } } - } - ) + }, + # Blocked at database level + { + "databases": { + "foreign_key_labels": { + # Only root can view this database + "allow": {"id": "root"}, + "tables": { + # But table b is visible to everyone + "b": {"allow": True}, + }, + } + } + }, + # Blocked at the instance level + { + "allow": {"id": "root"}, + "databases": { + "foreign_key_labels": { + "tables": { + # Table b is visible to everyone + "b": {"allow": True}, + } + } + }, + }, + ), +) +async def test_foreign_key_labels_obey_permissions(metadata): + ds = Datasette(metadata=metadata) db = ds.add_memory_database("foreign_key_labels") - await db.execute_write("create table a(id integer primary key, name text)") - await db.execute_write("insert into a (id, name) values (1, 'hello')") await db.execute_write( - "create table b(id integer primary key, name text, a_id integer references a(id))" + "create table if not exists a(id integer primary key, name text)" + ) + await db.execute_write("insert or replace into a (id, name) values (1, 'hello')") + await db.execute_write( + "create table if not exists b(id integer primary key, name text, a_id integer references a(id))" + ) + await db.execute_write( + "insert or replace into b (id, name, a_id) values (1, 'world', 1)" ) - await db.execute_write("insert into b (id, name, a_id) values (1, 'world', 1)") # Anonymous user can see table b but not table a blah = await ds.client.get("/foreign_key_labels.json") anon_a = await ds.client.get("/foreign_key_labels/a.json?_labels=on") From b645174271aa08e8ca83b27ff83ce078ecd15da2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 21:23:59 -0700 Subject: [PATCH 087/767] actors_from_ids plugin hook and datasette.actors_from_ids() method (#2181) * Prototype of actors_from_ids plugin hook, refs #2180 * datasette-remote-actors example plugin, refs #2180 --- datasette/app.py | 10 +++++++ datasette/hookspecs.py | 5 ++++ docs/internals.rst | 21 ++++++++++++++ docs/plugin_hooks.rst | 57 ++++++++++++++++++++++++++++++++++++++ tests/test_plugins.py | 62 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 155 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index ea9739f0..fdec2c86 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -819,6 +819,16 @@ class Datasette: ) return crumbs + async def actors_from_ids( + self, actor_ids: Iterable[Union[str, int]] + ) -> Dict[Union[id, str], Dict]: + result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids) + if result is None: + # Do the default thing + return {actor_id: {"id": actor_id} for actor_id in actor_ids} + result = await await_me_maybe(result) + return result + async def permission_allowed( self, actor, action, resource=None, default=DEFAULT_NOT_SET ): diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 801073fc..9069927b 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -94,6 +94,11 @@ def actor_from_request(datasette, request): """Return an actor dictionary based on the incoming request""" +@hookspec(firstresult=True) +def actors_from_ids(datasette, actor_ids): + """Returns a dictionary mapping those IDs to actor dictionaries""" + + @hookspec def filters_from_request(request, database, table, datasette): """ diff --git a/docs/internals.rst b/docs/internals.rst index 6b7d3df8..13f1d4a1 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -322,6 +322,27 @@ await .render_template(template, context=None, request=None) Renders a `Jinja template `__ using Datasette's preconfigured instance of Jinja and returns the resulting string. The template will have access to Datasette's default template functions and any functions that have been made available by other plugins. +.. _datasette_actors_from_ids: + +await .actors_from_ids(actor_ids) +--------------------------------- + +``actor_ids`` - list of strings or integers + A list of actor IDs to look up. + +Returns a dictionary, where the keys are the IDs passed to it and the values are the corresponding actor dictionaries. + +This method is mainly designed to be used with plugins. See the :ref:`plugin_hook_actors_from_ids` documentation for details. + +If no plugins that implement that hook are installed, the default return value looks like this: + +.. code-block:: json + + { + "1": {"id": "1"}, + "2": {"id": "2"} + } + .. _datasette_permission_allowed: await .permission_allowed(actor, action, resource=None, default=...) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 04fb24ce..e966919b 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1071,6 +1071,63 @@ Instead of returning a dictionary, this function can return an awaitable functio Examples: `datasette-auth-tokens `_, `datasette-auth-passwords `_ +.. _plugin_hook_actors_from_ids: + +actors_from_ids(datasette, actor_ids) +------------------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor_ids`` - list of strings or integers + The actor IDs to look up. + +The hook must return a dictionary that maps the incoming actor IDs to their full dictionary representation. + +Some plugins that implement social features may store the ID of the :ref:`actor ` that performed an action - added a comment, bookmarked a table or similar - and then need a way to resolve those IDs into display-friendly actor dictionaries later on. + +Unlike other plugin hooks, this only uses the first implementation of the hook to return a result. You can expect users to only have a single plugin installed that implements this hook. + +If no plugin is installed, Datasette defaults to returning actors that are just ``{"id": actor_id}``. + +The hook can return a dictionary or an awaitable function that then returns a dictionary. + +This example implementation returns actors from a database table: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def actors_from_ids(datasette, actor_ids): + db = datasette.get_database("actors") + + async def inner(): + sql = "select id, name from actors where id in ({})".format( + ", ".join("?" for _ in actor_ids) + ) + actors = {} + for row in (await db.execute(sql, actor_ids)).rows: + actor = dict(row) + actors[actor["id"]] = actor + return actors + + return inner + +The returned dictionary from this example looks like this: + +.. code-block:: json + + { + "1": {"id": "1", "name": "Tony"}, + "2": {"id": "2", "name": "Tina"}, + } + +These IDs could be integers or strings, depending on how the actors used by the Datasette instance are configured. + +Example: `datasette-remote-actors `_ + .. _plugin_hook_filters_from_request: filters_from_request(request, database, table, datasette) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 9761fa53..625ae635 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1215,3 +1215,65 @@ async def test_hook_register_permissions_allows_identical_duplicates(): await ds.invoke_startup() # Check that ds.permissions has only one of each assert len([p for p in ds.permissions.values() if p.abbr == "abbr1"]) == 1 + + +@pytest.mark.asyncio +async def test_hook_actors_from_ids(): + # Without the hook should return default {"id": id} list + ds = Datasette() + await ds.invoke_startup() + db = ds.add_memory_database("actors_from_ids") + await db.execute_write( + "create table actors (id text primary key, name text, age int)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('3', 'Cate Blanchett', 52)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('5', 'Rooney Mara', 36)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('7', 'Sarah Paulson', 46)" + ) + await db.execute_write( + "insert into actors (id, name, age) values ('9', 'Helena Bonham Carter', 55)" + ) + table_names = await db.table_names() + assert table_names == ["actors"] + actors1 = await ds.actors_from_ids(["3", "5", "7"]) + assert actors1 == { + "3": {"id": "3"}, + "5": {"id": "5"}, + "7": {"id": "7"}, + } + + class ActorsFromIdsPlugin: + __name__ = "ActorsFromIdsPlugin" + + @hookimpl + def actors_from_ids(self, datasette, actor_ids): + db = datasette.get_database("actors_from_ids") + + async def inner(): + sql = "select id, name from actors where id in ({})".format( + ", ".join("?" for _ in actor_ids) + ) + actors = {} + result = await db.execute(sql, actor_ids) + for row in result.rows: + actor = dict(row) + actors[actor["id"]] = actor + return actors + + return inner + + try: + pm.register(ActorsFromIdsPlugin(), name="ActorsFromIdsPlugin") + actors2 = await ds.actors_from_ids(["3", "5", "7"]) + assert actors2 == { + "3": {"id": "3", "name": "Cate Blanchett"}, + "5": {"id": "5", "name": "Rooney Mara"}, + "7": {"id": "7", "name": "Sarah Paulson"}, + } + finally: + pm.unregister(name="ReturnNothingPlugin") From a4c96d01b27ce7cd06662a024da3547132a7c412 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Sep 2023 21:44:08 -0700 Subject: [PATCH 088/767] Release 1.0a6 Refs #1765, #2164, #2169, #2175, #2178, #2181 --- datasette/version.py | 2 +- docs/changelog.rst | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index b99c212b..4b65999d 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a5" +__version__ = "1.0a6" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 019d6c68..81554f83 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,18 @@ Changelog ========= +.. _v1_0_a6: + +1.0a6 (2023-09-07) +------------------ + +- New plugin hook: :ref:`plugin_hook_actors_from_ids` and an internal method to accompany it, :ref:`datasette_actors_from_ids`. This mechanism is intended to be used by plugins that may need to display the actor who was responsible for something managed by that plugin: they can now resolve the recorded IDs of actors into the full actor objects. (:issue:`2181`) +- ``DATASETTE_LOAD_PLUGINS`` environment variable for :ref:`controlling which plugins ` are loaded by Datasette. (:issue:`2164`) +- Datasette now checks if the user has permission to view a table linked to by a foreign key before turning that foreign key into a clickable link. (:issue:`2178`) +- The ``execute-sql`` permission now implies that the actor can also view the database and instance. (:issue:`2169`) +- Documentation describing a pattern for building plugins that themselves :ref:`define further hooks ` for other plugins. (:issue:`1765`) +- Datasette is now tested against the Python 3.12 preview. (`#2175 `__) + .. _v1_0_a5: 1.0a5 (2023-08-29) From b2ec8717c3619260a1b535eea20e618bf95aa30b Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Wed, 13 Sep 2023 14:06:25 -0700 Subject: [PATCH 089/767] Plugin configuration now lives in datasette.yaml/json * Checkpoint, moving top-level plugin config to datasette.json * Support database-level and table-level plugin configuration in datasette.yaml Refs #2093 --- datasette/app.py | 48 +++++++++++++++----- docs/configuration.rst | 97 ++++++++++++++++++++++++++++++++++++++-- docs/index.rst | 1 + docs/internals.rst | 2 +- docs/plugin_hooks.rst | 2 +- docs/writing_plugins.rst | 7 +-- tests/conftest.py | 3 +- tests/fixtures.py | 50 ++++++++++++++------- tests/test_cli.py | 38 ++++++++++++++++ tests/test_plugins.py | 23 +++------- 10 files changed, 217 insertions(+), 54 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index fdec2c86..53486007 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -368,7 +368,7 @@ class Datasette: for key in config_settings: if key not in DEFAULT_SETTINGS: raise StartupError("Invalid setting '{}' in datasette.json".format(key)) - + self.config = config # CLI settings should overwrite datasette.json settings self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions @@ -674,15 +674,43 @@ class Datasette: def plugin_config(self, plugin_name, database=None, table=None, fallback=True): """Return config for plugin, falling back from specified database/table""" - plugins = self.metadata( - "plugins", database=database, table=table, fallback=fallback - ) - if plugins is None: - return None - plugin_config = plugins.get(plugin_name) - # Resolve any $file and $env keys - plugin_config = resolve_env_secrets(plugin_config, os.environ) - return plugin_config + if database is None and table is None: + config = self._plugin_config_top(plugin_name) + else: + config = self._plugin_config_nested(plugin_name, database, table, fallback) + + return resolve_env_secrets(config, os.environ) + + def _plugin_config_top(self, plugin_name): + """Returns any top-level plugin configuration for the specified plugin.""" + return ((self.config or {}).get("plugins") or {}).get(plugin_name) + + def _plugin_config_nested(self, plugin_name, database, table=None, fallback=True): + """Returns any database or table-level plugin configuration for the specified plugin.""" + db_config = ((self.config or {}).get("databases") or {}).get(database) + + # if there's no db-level configuration, then return early, falling back to top-level if needed + if not db_config: + return self._plugin_config_top(plugin_name) if fallback else None + + db_plugin_config = (db_config.get("plugins") or {}).get(plugin_name) + + if table: + table_plugin_config = ( + ((db_config.get("tables") or {}).get(table) or {}).get("plugins") or {} + ).get(plugin_name) + + # fallback to db_config or top-level config, in that order, if needed + if table_plugin_config is None and fallback: + return db_plugin_config or self._plugin_config_top(plugin_name) + + return table_plugin_config + + # fallback to top-level if needed + if db_plugin_config is None and fallback: + self._plugin_config_top(plugin_name) + + return db_plugin_config def app_css_hash(self): if not hasattr(self, "_app_css_hash"): diff --git a/docs/configuration.rst b/docs/configuration.rst index ed9975ac..214e9044 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,10 +1,101 @@ .. _configuration: Configuration -======== +============= -Datasette offers many way to configure your Datasette instances: server settings, plugin configuration, authentication, and more. +Datasette offers several ways to configure your Datasette instances: server settings, plugin configuration, authentication, and more. -To facilitate this, You can provide a `datasette.yaml` configuration file to datasette with the ``--config``/ ``-c`` flag: +To facilitate this, You can provide a ``datasette.yaml`` configuration file to datasette with the ``--config``/ ``-c`` flag: + +.. code-block:: bash datasette mydatabase.db --config datasette.yaml + +.. _configuration_reference: + +``datasette.yaml`` reference +---------------------------- + +Here's a full example of all the valid configuration options that can exist inside ``datasette.yaml``. + +.. tab:: YAML + + .. code-block:: yaml + + # Datasette settings block + settings: + default_page_size: 50 + sql_time_limit_ms: 3500 + max_returned_rows: 2000 + + # top-level plugin configuration + plugins: + datasette-my-plugin: + key: valueA + + # Database and table-level configuration + databases: + your_db_name: + # plugin configuration for the your_db_name database + plugins: + datasette-my-plugin: + key: valueA + tables: + your_table_name: + # plugin configuration for the your_table_name table + # inside your_db_name database + plugins: + datasette-my-plugin: + key: valueB + +.. _configuration_reference_settings: +Settings configuration +~~~~~~~~~~~~~~~~~~~~~~ + +:ref:`settings` can be configured in ``datasette.yaml`` with the ``settings`` key. + +.. tab:: YAML + + .. code-block:: yaml + + # inside datasette.yaml + settings: + default_allow_sql: off + default_page_size: 50 + + +.. _configuration_reference_plugins: +Plugin configuration +~~~~~~~~~~~~~~~~~~~~ + +Configuration for plugins can be defined inside ``datasette.yaml``. For top-level plugin configuration, use the ``plugins`` key. + +.. tab:: YAML + + .. code-block:: yaml + + # inside datasette.yaml + plugins: + datasette-my-plugin: + key: my_value + +For database level or table level plugin configuration, nest it under the appropriate place under ``databases``. + +.. tab:: YAML + + .. code-block:: yaml + + # inside datasette.yaml + databases: + my_database: + # plugin configuration for the my_database database + plugins: + datasette-my-plugin: + key: my_value + my_other_database: + tables: + my_table: + # plugin configuration for the my_table table inside the my_other_database database + plugins: + datasette-my-plugin: + key: my_value diff --git a/docs/index.rst b/docs/index.rst index f5c1f232..cfa3443c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,6 +39,7 @@ Contents getting_started installation + configuration ecosystem cli-reference pages diff --git a/docs/internals.rst b/docs/internals.rst index 13f1d4a1..7fc7948c 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -296,7 +296,7 @@ The dictionary keys are the permission names - e.g. ``view-instance`` - and the ``table`` - None or string The table the user is interacting with. -This method lets you read plugin configuration values that were set in ``metadata.json``. See :ref:`writing_plugins_configuration` for full details of how this method should be used. +This method lets you read plugin configuration values that were set in ``datasette.yaml``. See :ref:`writing_plugins_configuration` for full details of how this method should be used. The return value will be the value from the configuration file - usually a dictionary. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index e966919b..1816d48c 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -909,7 +909,7 @@ Potential use-cases: * Run some initialization code for the plugin * Create database tables that a plugin needs on startup -* Validate the metadata configuration for a plugin on startup, and raise an error if it is invalid +* Validate the configuration for a plugin on startup, and raise an error if it is invalid .. note:: diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index d0dd8f36..c028b4ff 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -184,7 +184,7 @@ This will return the ``{"latitude_column": "lat", "longitude_column": "lng"}`` i If there is no configuration for that plugin, the method will return ``None``. -If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option like so: +If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option inside ``datasette.yaml`` like so: .. [[[cog from metadata_doc import metadata_example @@ -234,11 +234,10 @@ If it cannot find the requested configuration at the table layer, it will fall b In this case, the above code would return that configuration for ANY table within the ``sf-trees`` database. -The plugin configuration could also be set at the top level of ``metadata.yaml``: +The plugin configuration could also be set at the top level of ``datasette.yaml``: .. [[[cog metadata_example(cog, { - "title": "This is the top-level title in metadata.json", "plugins": { "datasette-cluster-map": { "latitude_column": "xlat", @@ -252,7 +251,6 @@ The plugin configuration could also be set at the top level of ``metadata.yaml`` .. code-block:: yaml - title: This is the top-level title in metadata.json plugins: datasette-cluster-map: latitude_column: xlat @@ -264,7 +262,6 @@ The plugin configuration could also be set at the top level of ``metadata.yaml`` .. code-block:: json { - "title": "This is the top-level title in metadata.json", "plugins": { "datasette-cluster-map": { "latitude_column": "xlat", diff --git a/tests/conftest.py b/tests/conftest.py index fb7f768e..31336aea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,7 @@ def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): @pytest_asyncio.fixture async def ds_client(): from datasette.app import Datasette - from .fixtures import METADATA, PLUGINS_DIR + from .fixtures import CONFIG, METADATA, PLUGINS_DIR global _ds_client if _ds_client is not None: @@ -49,6 +49,7 @@ async def ds_client(): ds = Datasette( metadata=METADATA, + config=CONFIG, plugins_dir=PLUGINS_DIR, settings={ "default_page_size": 50, diff --git a/tests/fixtures.py b/tests/fixtures.py index a6700239..9cf6b605 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -114,6 +114,7 @@ def make_app_client( inspect_data=None, static_mounts=None, template_dir=None, + config=None, metadata=None, crossdb=False, ): @@ -158,6 +159,7 @@ def make_app_client( memory=memory, cors=cors, metadata=metadata or METADATA, + config=config or CONFIG, plugins_dir=PLUGINS_DIR, settings=settings, inspect_data=inspect_data, @@ -296,6 +298,33 @@ def generate_sortable_rows(num): } +CONFIG = { + "plugins": { + "name-of-plugin": {"depth": "root"}, + "env-plugin": {"foo": {"$env": "FOO_ENV"}}, + "env-plugin-list": [{"in_a_list": {"$env": "FOO_ENV"}}], + "file-plugin": {"foo": {"$file": TEMP_PLUGIN_SECRET_FILE}}, + }, + "databases": { + "fixtures": { + "plugins": {"name-of-plugin": {"depth": "database"}}, + "tables": { + "simple_primary_key": { + "plugins": { + "name-of-plugin": { + "depth": "table", + "special": "this-is-simple_primary_key", + } + }, + }, + "sortable": { + "plugins": {"name-of-plugin": {"depth": "table"}}, + }, + }, + } + }, +} + METADATA = { "title": "Datasette Fixtures", "description_html": 'An example SQLite database demonstrating Datasette. Sign in as root user', @@ -306,26 +335,13 @@ METADATA = { "about": "About Datasette", "about_url": "https://github.com/simonw/datasette", "extra_css_urls": ["/static/extra-css-urls.css"], - "plugins": { - "name-of-plugin": {"depth": "root"}, - "env-plugin": {"foo": {"$env": "FOO_ENV"}}, - "env-plugin-list": [{"in_a_list": {"$env": "FOO_ENV"}}], - "file-plugin": {"foo": {"$file": TEMP_PLUGIN_SECRET_FILE}}, - }, "databases": { "fixtures": { "description": "Test tables description", - "plugins": {"name-of-plugin": {"depth": "database"}}, "tables": { "simple_primary_key": { "description_html": "Simple primary key", "title": "This HTML is escaped", - "plugins": { - "name-of-plugin": { - "depth": "table", - "special": "this-is-simple_primary_key", - } - }, }, "sortable": { "sortable_columns": [ @@ -334,7 +350,6 @@ METADATA = { "sortable_with_nulls_2", "text", ], - "plugins": {"name-of-plugin": {"depth": "table"}}, }, "no_primary_key": {"sortable_columns": [], "hidden": True}, "units": {"units": {"distance": "m", "frequency": "Hz"}}, @@ -768,6 +783,7 @@ def assert_permissions_checked(datasette, actions): type=click.Path(file_okay=True, dir_okay=False), ) @click.argument("metadata", required=False) +@click.argument("config", required=False) @click.argument( "plugins_path", type=click.Path(file_okay=False, dir_okay=True), required=False ) @@ -782,7 +798,7 @@ def assert_permissions_checked(datasette, actions): type=click.Path(file_okay=True, dir_okay=False), help="Write out second test DB to this file", ) -def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename): +def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename): """Write out the fixtures database used by Datasette's test suite""" if metadata and not metadata.endswith(".json"): raise click.ClickException("Metadata should end with .json") @@ -805,6 +821,10 @@ def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename): with open(metadata, "w") as fp: fp.write(json.dumps(METADATA, indent=4)) print(f"- metadata written to {metadata}") + if config: + with open(config, "w") as fp: + fp.write(json.dumps(CONFIG, indent=4)) + print(f"- config written to {config}") if plugins_path: path = pathlib.Path(plugins_path) if not path.exists(): diff --git a/tests/test_cli.py b/tests/test_cli.py index e85bcef1..213db416 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -238,6 +238,44 @@ def test_setting(args): assert settings["default_page_size"] == 5 +def test_plugin_s_overwrite(): + runner = CliRunner() + plugins_dir = str(pathlib.Path(__file__).parent / "plugins") + + result = runner.invoke( + cli, + [ + "--plugins-dir", + plugins_dir, + "--get", + "/_memory.json?sql=select+prepare_connection_args()", + ], + ) + assert result.exit_code == 0, result.output + assert ( + json.loads(result.output).get("rows")[0].get("prepare_connection_args()") + == 'database=_memory, datasette.plugin_config("name-of-plugin")=None' + ) + + result = runner.invoke( + cli, + [ + "--plugins-dir", + plugins_dir, + "--get", + "/_memory.json?sql=select+prepare_connection_args()", + "-s", + "plugins.name-of-plugin", + "OVERRIDE", + ], + ) + assert result.exit_code == 0, result.output + assert ( + json.loads(result.output).get("rows")[0].get("prepare_connection_args()") + == 'database=_memory, datasette.plugin_config("name-of-plugin")=OVERRIDE' + ) + + def test_setting_type_validation(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--setting", "default_page_size", "dog"]) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 625ae635..37530991 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -234,9 +234,6 @@ async def test_plugin_config(ds_client): async def test_plugin_config_env(ds_client): os.environ["FOO_ENV"] = "FROM_ENVIRONMENT" assert {"foo": "FROM_ENVIRONMENT"} == ds_client.ds.plugin_config("env-plugin") - # Ensure secrets aren't visible in /-/metadata.json - metadata = await ds_client.get("/-/metadata.json") - assert {"foo": {"$env": "FOO_ENV"}} == metadata.json()["plugins"]["env-plugin"] del os.environ["FOO_ENV"] @@ -246,11 +243,6 @@ async def test_plugin_config_env_from_list(ds_client): assert [{"in_a_list": "FROM_ENVIRONMENT"}] == ds_client.ds.plugin_config( "env-plugin-list" ) - # Ensure secrets aren't visible in /-/metadata.json - metadata = await ds_client.get("/-/metadata.json") - assert [{"in_a_list": {"$env": "FOO_ENV"}}] == metadata.json()["plugins"][ - "env-plugin-list" - ] del os.environ["FOO_ENV"] @@ -259,11 +251,6 @@ async def test_plugin_config_file(ds_client): with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp: fp.write("FROM_FILE") assert {"foo": "FROM_FILE"} == ds_client.ds.plugin_config("file-plugin") - # Ensure secrets aren't visible in /-/metadata.json - metadata = await ds_client.get("/-/metadata.json") - assert {"foo": {"$file": TEMP_PLUGIN_SECRET_FILE}} == metadata.json()["plugins"][ - "file-plugin" - ] os.remove(TEMP_PLUGIN_SECRET_FILE) @@ -722,7 +709,7 @@ async def test_hook_register_routes(ds_client, path, body): @pytest.mark.parametrize("configured_path", ("path1", "path2")) def test_hook_register_routes_with_datasette(configured_path): with make_app_client( - metadata={ + config={ "plugins": { "register-route-demo": { "path": configured_path, @@ -741,7 +728,7 @@ def test_hook_register_routes_with_datasette(configured_path): def test_hook_register_routes_override(): "Plugins can over-ride default paths such as /db/table" with make_app_client( - metadata={ + config={ "plugins": { "register-route-demo": { "path": "blah", @@ -1099,7 +1086,7 @@ async def test_hook_filters_from_request(ds_client): @pytest.mark.parametrize("extra_metadata", (False, True)) async def test_hook_register_permissions(extra_metadata): ds = Datasette( - metadata={ + config={ "plugins": { "datasette-register-permissions": { "permissions": [ @@ -1151,7 +1138,7 @@ async def test_hook_register_permissions_no_duplicates(duplicate): if duplicate == "abbr": abbr2 = "abbr1" ds = Datasette( - metadata={ + config={ "plugins": { "datasette-register-permissions": { "permissions": [ @@ -1186,7 +1173,7 @@ async def test_hook_register_permissions_no_duplicates(duplicate): @pytest.mark.asyncio async def test_hook_register_permissions_allows_identical_duplicates(): ds = Datasette( - metadata={ + config={ "plugins": { "datasette-register-permissions": { "permissions": [ From 16f0b6d8222d06682a31b904d0a402c391ae1c1c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 13 Sep 2023 14:15:32 -0700 Subject: [PATCH 090/767] JSON/YAML tabs on configuration docs page --- docs/configuration.rst | 171 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 214e9044..4a7258b9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -18,6 +18,40 @@ To facilitate this, You can provide a ``datasette.yaml`` configuration file to d Here's a full example of all the valid configuration options that can exist inside ``datasette.yaml``. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # Datasette settings block + settings: + default_page_size: 50 + sql_time_limit_ms: 3500 + max_returned_rows: 2000 + + # top-level plugin configuration + plugins: + datasette-my-plugin: + key: valueA + + # Database and table-level configuration + databases: + your_db_name: + # plugin configuration for the your_db_name database + plugins: + datasette-my-plugin: + key: valueA + tables: + your_table_name: + # plugin configuration for the your_table_name table + # inside your_db_name database + plugins: + datasette-my-plugin: + key: valueB + """) + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -48,12 +82,61 @@ Here's a full example of all the valid configuration options that can exist insi datasette-my-plugin: key: valueB +.. tab:: JSON + + .. code-block:: json + + { + "settings": { + "default_page_size": 50, + "sql_time_limit_ms": 3500, + "max_returned_rows": 2000 + }, + "plugins": { + "datasette-my-plugin": { + "key": "valueA" + } + }, + "databases": { + "your_db_name": { + "plugins": { + "datasette-my-plugin": { + "key": "valueA" + } + }, + "tables": { + "your_table_name": { + "plugins": { + "datasette-my-plugin": { + "key": "valueB" + } + } + } + } + } + } + } +.. [[[end]]] + .. _configuration_reference_settings: Settings configuration ~~~~~~~~~~~~~~~~~~~~~~ :ref:`settings` can be configured in ``datasette.yaml`` with the ``settings`` key. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # inside datasette.yaml + settings: + default_allow_sql: off + default_page_size: 50 + """).strip() + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -63,6 +146,17 @@ Settings configuration default_allow_sql: off default_page_size: 50 +.. tab:: JSON + + .. code-block:: json + + { + "settings": { + "default_allow_sql": "off", + "default_page_size": 50 + } + } +.. [[[end]]] .. _configuration_reference_plugins: Plugin configuration @@ -70,6 +164,19 @@ Plugin configuration Configuration for plugins can be defined inside ``datasette.yaml``. For top-level plugin configuration, use the ``plugins`` key. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # inside datasette.yaml + plugins: + datasette-my-plugin: + key: my_value + """).strip() + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -79,8 +186,44 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve datasette-my-plugin: key: my_value +.. tab:: JSON + + .. code-block:: json + + { + "plugins": { + "datasette-my-plugin": { + "key": "my_value" + } + } + } +.. [[[end]]] + For database level or table level plugin configuration, nest it under the appropriate place under ``databases``. +.. [[[cog + from metadata_doc import metadata_example + import textwrap + metadata_example(cog, yaml=textwrap.dedent( + """ + # inside datasette.yaml + databases: + my_database: + # plugin configuration for the my_database database + plugins: + datasette-my-plugin: + key: my_value + my_other_database: + tables: + my_table: + # plugin configuration for the my_table table inside the my_other_database database + plugins: + datasette-my-plugin: + key: my_value + """).strip() + ) +.. ]]] + .. tab:: YAML .. code-block:: yaml @@ -99,3 +242,31 @@ For database level or table level plugin configuration, nest it under the approp plugins: datasette-my-plugin: key: my_value + +.. tab:: JSON + + .. code-block:: json + + { + "databases": { + "my_database": { + "plugins": { + "datasette-my-plugin": { + "key": "my_value" + } + } + }, + "my_other_database": { + "tables": { + "my_table": { + "plugins": { + "datasette-my-plugin": { + "key": "my_value" + } + } + } + } + } + } + } +.. [[[end]]] \ No newline at end of file From 852f5014853943fa27f43ddaa2d442545b3259fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 Sep 2023 09:35:18 -0700 Subject: [PATCH 091/767] Switch from pkg_resources to importlib.metadata in app.py, refs #2057 --- datasette/app.py | 6 +++--- tests/test_plugins.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 53486007..c0e80700 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -8,11 +8,11 @@ import functools import glob import hashlib import httpx +import importlib.metadata import inspect from itsdangerous import BadSignature import json import os -import pkg_resources import re import secrets import sys @@ -1118,9 +1118,9 @@ class Datasette: if using_pysqlite3: for package in ("pysqlite3", "pysqlite3-binary"): try: - info["pysqlite3"] = pkg_resources.get_distribution(package).version + info["pysqlite3"] = importlib.metadata.version(package) break - except pkg_resources.DistributionNotFound: + except importlib.metadata.PackageNotFoundError: pass return info diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 37530991..3bc117f3 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1264,3 +1264,25 @@ async def test_hook_actors_from_ids(): } finally: pm.unregister(name="ReturnNothingPlugin") + + +@pytest.mark.asyncio +async def test_plugin_is_installed(): + datasette = Datasette(memory=True) + + class DummyPlugin: + __name__ = "DummyPlugin" + + @hookimpl + def actors_from_ids(self, datasette, actor_ids): + return {} + + try: + pm.register(DummyPlugin(), name="DummyPlugin") + response = await datasette.client.get("/-/plugins.json") + assert response.status_code == 200 + installed_plugins = {p["name"] for p in response.json()} + assert "DummyPlugin" in installed_plugins + + finally: + pm.unregister(name="DummyPlugin") From f56e043747bde4faa1d78588636df6c0dadebc65 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 18 Sep 2023 10:39:11 -0700 Subject: [PATCH 092/767] test_facet_against_in_memory_database, refs #2189 This is meant to illustrate a crashing bug but it does not trigger it. --- tests/test_facets.py | 47 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_facets.py b/tests/test_facets.py index 48cc0ff2..a68347f0 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -643,3 +643,50 @@ async def test_conflicting_facet_names_json(ds_client): "created_2", "tags_2", } + + +@pytest.mark.asyncio +async def test_facet_against_in_memory_database(): + ds = Datasette() + db = ds.add_memory_database("mem") + await db.execute_write("create table t (id integer primary key, name text)") + to_insert = [["one"] for _ in range(800)] + [["two"] for _ in range(300)] + await db.execute_write_many("insert into t (name) values (?)", to_insert) + response1 = await ds.client.get("/mem/t.json") + assert response1.status_code == 200 + response2 = await ds.client.get("/mem/t.json?_facet=name&_size=0") + assert response2.status_code == 200 + assert response2.json() == { + "ok": True, + "next": None, + "facet_results": { + "results": { + "name": { + "name": "name", + "type": "column", + "hideable": True, + "toggle_url": "/mem/t.json?_size=0", + "results": [ + { + "value": "one", + "label": "one", + "count": 800, + "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=one", + "selected": False, + }, + { + "value": "two", + "label": "two", + "count": 300, + "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=two", + "selected": False, + }, + ], + "truncated": False, + } + }, + "timed_out": [], + }, + "rows": [], + "truncated": False, + } From 6ed7908580fa2ba9297c3225d85c56f8b08b9937 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 18 Sep 2023 10:44:13 -0700 Subject: [PATCH 093/767] Simplified test for #2189 This now executes two facets, in the hope that parallel facet execution would illustrate the bug - but it did not illustrate the bug. --- tests/test_facets.py | 51 +++++++++++--------------------------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/tests/test_facets.py b/tests/test_facets.py index a68347f0..85c8f85b 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -649,44 +649,17 @@ async def test_conflicting_facet_names_json(ds_client): async def test_facet_against_in_memory_database(): ds = Datasette() db = ds.add_memory_database("mem") - await db.execute_write("create table t (id integer primary key, name text)") - to_insert = [["one"] for _ in range(800)] + [["two"] for _ in range(300)] - await db.execute_write_many("insert into t (name) values (?)", to_insert) - response1 = await ds.client.get("/mem/t.json") + await db.execute_write( + "create table t (id integer primary key, name text, name2 text)" + ) + to_insert = [{"name": "one", "name2": "1"} for _ in range(800)] + [ + {"name": "two", "name2": "2"} for _ in range(300) + ] + print(to_insert) + await db.execute_write_many( + "insert into t (name, name2) values (:name, :name2)", to_insert + ) + response1 = await ds.client.get("/mem/t") assert response1.status_code == 200 - response2 = await ds.client.get("/mem/t.json?_facet=name&_size=0") + response2 = await ds.client.get("/mem/t?_facet=name&_facet=name2") assert response2.status_code == 200 - assert response2.json() == { - "ok": True, - "next": None, - "facet_results": { - "results": { - "name": { - "name": "name", - "type": "column", - "hideable": True, - "toggle_url": "/mem/t.json?_size=0", - "results": [ - { - "value": "one", - "label": "one", - "count": 800, - "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=one", - "selected": False, - }, - { - "value": "two", - "label": "two", - "count": 300, - "toggle_url": "http://localhost/mem/t.json?_facet=name&_size=0&name=two", - "selected": False, - }, - ], - "truncated": False, - } - }, - "timed_out": [], - }, - "rows": [], - "truncated": False, - } From b0e5d8afa308759f4ee9f3ecdf61101dffc4a037 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 Sep 2023 15:10:55 -0700 Subject: [PATCH 094/767] Stop using parallel SQL queries for tables Refs: - #2189 --- datasette/views/table.py | 16 ++++++---------- docs/internals.rst | 1 + 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 50ba2b78..4f4baeed 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -74,11 +74,10 @@ class Row: return json.dumps(d, default=repr, indent=2) -async def _gather_parallel(*args): - return await asyncio.gather(*args) - - -async def _gather_sequential(*args): +async def run_sequential(*args): + # This used to be swappable for asyncio.gather() to run things in + # parallel, but this lead to hard-to-debug locking issues with + # in-memory databases: https://github.com/simonw/datasette/issues/2189 results = [] for fn in args: results.append(await fn) @@ -1183,9 +1182,6 @@ async def table_view_data( ) rows = rows[:page_size] - # For performance profiling purposes, ?_noparallel=1 turns off asyncio.gather - gather = _gather_sequential if request.args.get("_noparallel") else _gather_parallel - # Resolve extras extras = _get_extras(request) if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")): @@ -1249,7 +1245,7 @@ async def table_view_data( if not nofacet: # Run them in parallel facet_awaitables = [facet.facet_results() for facet in facet_instances] - facet_awaitable_results = await gather(*facet_awaitables) + facet_awaitable_results = await run_sequential(*facet_awaitables) for ( instance_facet_results, instance_facets_timed_out, @@ -1282,7 +1278,7 @@ async def table_view_data( ): # Run them in parallel facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] - for suggest_result in await gather(*facet_suggest_awaitables): + for suggest_result in await run_sequential(*facet_suggest_awaitables): suggested_facets.extend(suggest_result) return suggested_facets diff --git a/docs/internals.rst b/docs/internals.rst index 7fc7948c..4e9a6747 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1317,6 +1317,7 @@ This example uses the :ref:`register_routes() ` plugin h (r"/parallel-queries$", parallel_queries), ] +Note that running parallel SQL queries in this way has `been known to cause problems in the past `__, so treat this example with caution. Adding ``?_trace=1`` will show that the trace covers both of those child tasks. From 6763572948ffd047a89a3bbf7c300e91f51ae98f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:11:24 -0700 Subject: [PATCH 095/767] Bump sphinx, furo, black Bumps the python-packages group with 3 updates: [sphinx](https://github.com/sphinx-doc/sphinx), [furo](https://github.com/pradyunsg/furo) and [black](https://github.com/psf/black). Updates `sphinx` from 7.2.5 to 7.2.6 - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.5...v7.2.6) Updates `furo` from 2023.8.19 to 2023.9.10 - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.08.19...2023.09.10) Updates `black` from 23.7.0 to 23.9.1 - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.7.0...23.9.1) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-packages - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index c718086b..415fd27c 100644 --- a/setup.py +++ b/setup.py @@ -69,8 +69,8 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "Sphinx==7.2.5", - "furo==2023.8.19", + "Sphinx==7.2.6", + "furo==2023.9.10", "sphinx-autobuild", "codespell>=2.2.5", "blacken-docs", @@ -83,7 +83,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==23.7.0", + "black==23.9.1", "blacken-docs==1.16.0", "pytest-timeout>=1.4.2", "trustme>=0.7", From 10bc80547330e826a749ce710da21ae29f7e6048 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:11:35 -0700 Subject: [PATCH 096/767] Finish removing pkg_resources, closes #2057 --- datasette/plugins.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index 6ec08a81..017f3b9d 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,7 +1,7 @@ -import importlib +import importlib.metadata +import importlib.resources import os import pluggy -import pkg_resources import sys from . import hookspecs @@ -35,15 +35,15 @@ if DATASETTE_LOAD_PLUGINS is not None: name for name in DATASETTE_LOAD_PLUGINS.split(",") if name.strip() ]: try: - distribution = pkg_resources.get_distribution(package_name) - entry_map = distribution.get_entry_map() - if "datasette" in entry_map: - for plugin_name, entry_point in entry_map["datasette"].items(): + distribution = importlib.metadata.distribution(package_name) + entry_points = distribution.entry_points + for entry_point in entry_points: + if entry_point.group == "datasette": mod = entry_point.load() pm.register(mod, name=entry_point.name) # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) - except pkg_resources.DistributionNotFound: + except importlib.metadata.PackageNotFoundError: sys.stderr.write("Plugin {} could not be found\n".format(package_name)) @@ -61,16 +61,16 @@ def get_plugins(): templates_path = None if plugin.__name__ not in DEFAULT_PLUGINS: try: - if pkg_resources.resource_isdir(plugin.__name__, "static"): - static_path = pkg_resources.resource_filename( - plugin.__name__, "static" + if (importlib.resources.files(plugin.__name__) / "static").is_dir(): + static_path = str( + importlib.resources.files(plugin.__name__) / "static" ) - if pkg_resources.resource_isdir(plugin.__name__, "templates"): - templates_path = pkg_resources.resource_filename( - plugin.__name__, "templates" + if (importlib.resources.files(plugin.__name__) / "templates").is_dir(): + templates_path = str( + importlib.resources.files(plugin.__name__) / "templates" ) - except (KeyError, ImportError): - # Caused by --plugins_dir= plugins - KeyError/ImportError thrown in Py3.5 + except (TypeError, ModuleNotFoundError): + # Caused by --plugins_dir= plugins pass plugin_info = { "name": plugin.__name__, @@ -81,6 +81,6 @@ def get_plugins(): distinfo = plugin_to_distinfo.get(plugin) if distinfo: plugin_info["version"] = distinfo.version - plugin_info["name"] = distinfo.project_name + plugin_info["name"] = distinfo.name plugins.append(plugin_info) return plugins From 947520c1fe940de79f5db856dd693330f1bbf547 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:31:32 -0700 Subject: [PATCH 097/767] Release notes for 0.64.4 on main --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 81554f83..52e1db3b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_64_4: + +0.64.4 (2023-09-21) +------------------- + +- Fix for a crashing bug caused by viewing the table page for a named in-memory database. (:issue:`2189`) + .. _v1_0_a6: 1.0a6 (2023-09-07) From b0d0a0e5de8bb5b9b6c253e8af451a532266bcf1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:42:15 -0700 Subject: [PATCH 098/767] importlib_resources for Python < 3.9, refs #2057 --- datasette/plugins.py | 15 ++++++++++----- setup.py | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index 017f3b9d..a93145cf 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,10 +1,15 @@ import importlib.metadata -import importlib.resources import os import pluggy import sys from . import hookspecs +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources + + DEFAULT_PLUGINS = ( "datasette.publish.heroku", "datasette.publish.cloudrun", @@ -61,13 +66,13 @@ def get_plugins(): templates_path = None if plugin.__name__ not in DEFAULT_PLUGINS: try: - if (importlib.resources.files(plugin.__name__) / "static").is_dir(): + if (importlib_resources.files(plugin.__name__) / "static").is_dir(): static_path = str( - importlib.resources.files(plugin.__name__) / "static" + importlib_resources.files(plugin.__name__) / "static" ) - if (importlib.resources.files(plugin.__name__) / "templates").is_dir(): + if (importlib_resources.files(plugin.__name__) / "templates").is_dir(): templates_path = str( - importlib.resources.files(plugin.__name__) / "templates" + importlib_resources.files(plugin.__name__) / "templates" ) except (TypeError, ModuleNotFoundError): # Caused by --plugins_dir= plugins diff --git a/setup.py b/setup.py index 415fd27c..a2728f6b 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ setup( "Jinja2>=2.10.3", "hupper>=1.9", "httpx>=0.20", + 'importlib_resources>=1.3.1; python_version < "3.9"', "pint>=0.9", "pluggy>=1.0", "uvicorn>=0.11", From 80a9cd9620fddf2695d12d8386a91e7c6b145ef2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 12:55:50 -0700 Subject: [PATCH 099/767] test-datasette-load-plugins now fails correctly, refs #2193 --- tests/test-datasette-load-plugins.sh | 41 +++++++++++++--------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/tests/test-datasette-load-plugins.sh b/tests/test-datasette-load-plugins.sh index e26d8377..03e08bb1 100755 --- a/tests/test-datasette-load-plugins.sh +++ b/tests/test-datasette-load-plugins.sh @@ -3,27 +3,24 @@ # datasette-init and datasette-json-html are installed PLUGINS=$(datasette plugins) -echo "$PLUGINS" | jq 'any(.[]; .name == "datasette-json-html")' | \ - grep -q true || ( \ - echo "Test failed: datasette-json-html not found" && \ - exit 1 \ - ) -# With the DATASETTE_LOAD_PLUGINS we should not see that +if ! echo "$PLUGINS" | jq 'any(.[]; .name == "datasette-json-html")' | grep -q true; then + echo "Test failed: datasette-json-html not found" + exit 1 +fi + PLUGINS2=$(DATASETTE_LOAD_PLUGINS=datasette-init datasette plugins) -echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-json-html")' | \ - grep -q false || ( \ - echo "Test failed: datasette-json-html should not have been loaded" && \ - exit 1 \ - ) -echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-init")' | \ - grep -q true || ( \ - echo "Test failed: datasette-init should have been loaded" && \ - exit 1 \ - ) -# With DATASETTE_LOAD_PLUGINS='' we should see no plugins +if ! echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-json-html")' | grep -q false; then + echo "Test failed: datasette-json-html should not have been loaded" + exit 1 +fi + +if ! echo "$PLUGINS2" | jq 'any(.[]; .name == "datasette-init")' | grep -q true; then + echo "Test failed: datasette-init should have been loaded" + exit 1 +fi + PLUGINS3=$(DATASETTE_LOAD_PLUGINS='' datasette plugins) -echo "$PLUGINS3"| \ - grep -q '\[\]' || ( \ - echo "Test failed: datasette plugins should have returned []" && \ - exit 1 \ - ) +if ! echo "$PLUGINS3" | grep -q '\[\]'; then + echo "Test failed: datasette plugins should have returned []" + exit 1 +fi From b7cf0200e21796a6ff653c6f94a4ee5fcfde0346 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 13:22:40 -0700 Subject: [PATCH 100/767] Swap order of config and metadata options, refs #2194 --- tests/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 9cf6b605..16aa234e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -782,8 +782,8 @@ def assert_permissions_checked(datasette, actions): default="fixtures.db", type=click.Path(file_okay=True, dir_okay=False), ) -@click.argument("metadata", required=False) @click.argument("config", required=False) +@click.argument("metadata", required=False) @click.argument( "plugins_path", type=click.Path(file_okay=False, dir_okay=True), required=False ) From 2da1a6acec915b81a16127008fd739c7d6075681 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 13:26:13 -0700 Subject: [PATCH 101/767] Use importlib_metadata for Python 3.8, refs #2057 --- datasette/plugins.py | 10 ++++++---- setup.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index a93145cf..f23f5cfb 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,4 +1,4 @@ -import importlib.metadata +import importlib import os import pluggy import sys @@ -6,8 +6,10 @@ from . import hookspecs if sys.version_info >= (3, 9): import importlib.resources as importlib_resources + import importlib.metadata as importlib_metadata else: import importlib_resources + import importlib_metadata DEFAULT_PLUGINS = ( @@ -40,7 +42,7 @@ if DATASETTE_LOAD_PLUGINS is not None: name for name in DATASETTE_LOAD_PLUGINS.split(",") if name.strip() ]: try: - distribution = importlib.metadata.distribution(package_name) + distribution = importlib_metadata.distribution(package_name) entry_points = distribution.entry_points for entry_point in entry_points: if entry_point.group == "datasette": @@ -48,7 +50,7 @@ if DATASETTE_LOAD_PLUGINS is not None: pm.register(mod, name=entry_point.name) # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) - except importlib.metadata.PackageNotFoundError: + except importlib_metadata.PackageNotFoundError: sys.stderr.write("Plugin {} could not be found\n".format(package_name)) @@ -86,6 +88,6 @@ def get_plugins(): distinfo = plugin_to_distinfo.get(plugin) if distinfo: plugin_info["version"] = distinfo.version - plugin_info["name"] = distinfo.name + plugin_info["name"] = distinfo.name or distinfo.project_name plugins.append(plugin_info) return plugins diff --git a/setup.py b/setup.py index a2728f6b..65a3b335 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ setup( "hupper>=1.9", "httpx>=0.20", 'importlib_resources>=1.3.1; python_version < "3.9"', + 'importlib_metadata>=4.6; python_version < "3.9"', "pint>=0.9", "pluggy>=1.0", "uvicorn>=0.11", From f130c7c0a88e50cea4121ea18d1f6db2431b6fab Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 14:09:57 -0700 Subject: [PATCH 102/767] Deploy with fixtures-metadata.json, refs #2194, #2195 --- .github/workflows/deploy-latest.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 0dfa5a60..e0405440 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -38,8 +38,14 @@ jobs: run: | pytest -n auto -m "not serial" pytest -m "serial" - - name: Build fixtures.db - run: python tests/fixtures.py fixtures.db fixtures.json plugins --extra-db-filename extra_database.db + - name: Build fixtures.db and other files needed to deploy the demo + run: |- + python tests/fixtures.py \ + fixtures.db \ + fixtures-config.json \ + fixtures-metadata.json \ + plugins \ + --extra-db-filename extra_database.db - name: Build docs.db if: ${{ github.ref == 'refs/heads/main' }} run: |- @@ -88,13 +94,13 @@ jobs: } return queries EOF - - name: Make some modifications to metadata.json - run: | - cat fixtures.json | \ - jq '.databases |= . + {"ephemeral": {"allow": {"id": "*"}}}' | \ - jq '.plugins |= . + {"datasette-ephemeral-tables": {"table_ttl": 900}}' \ - > metadata.json - cat metadata.json + # - name: Make some modifications to metadata.json + # run: | + # cat fixtures.json | \ + # jq '.databases |= . + {"ephemeral": {"allow": {"id": "*"}}}' | \ + # jq '.plugins |= . + {"datasette-ephemeral-tables": {"table_ttl": 900}}' \ + # > metadata.json + # cat metadata.json - name: Set up Cloud Run uses: google-github-actions/setup-gcloud@v0 with: @@ -112,7 +118,7 @@ jobs: # Replace 1.0 with one-dot-zero in SUFFIX export SUFFIX=${SUFFIX//1.0/one-dot-zero} datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \ - -m metadata.json \ + -m fixtures-metadata.json \ --plugins-dir=plugins \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ From e4f868801a6633400045f59584cfe650961c3fa6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 14:58:39 -0700 Subject: [PATCH 103/767] Use importlib_metadata for 3.9 as well, refs #2057 --- datasette/plugins.py | 4 +++- setup.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/datasette/plugins.py b/datasette/plugins.py index f23f5cfb..1ed3747f 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -6,9 +6,11 @@ from . import hookspecs if sys.version_info >= (3, 9): import importlib.resources as importlib_resources - import importlib.metadata as importlib_metadata else: import importlib_resources +if sys.version_info >= (3, 10): + import importlib.metadata as importlib_metadata +else: import importlib_metadata diff --git a/setup.py b/setup.py index 65a3b335..d09a9e3d 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( "hupper>=1.9", "httpx>=0.20", 'importlib_resources>=1.3.1; python_version < "3.9"', - 'importlib_metadata>=4.6; python_version < "3.9"', + 'importlib_metadata>=4.6; python_version < "3.10"', "pint>=0.9", "pluggy>=1.0", "uvicorn>=0.11", From 836b1587f08800658c63679d850f0149003c5311 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Sep 2023 15:06:19 -0700 Subject: [PATCH 104/767] Release notes for 1.0a7 Refs #2189 --- datasette/version.py | 2 +- docs/changelog.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 4b65999d..55e2cd42 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a6" +__version__ = "1.0a7" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 52e1db3b..9a5290c0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v1_0_a7: + +1.0a7 (2023-09-21) +------------------ + +- Fix for a crashing bug caused by viewing the table page for a named in-memory database. (:issue:`2189`) + .. _v0_64_4: 0.64.4 (2023-09-21) From d51e63d3bb3e32f80d1c0f04adff7c1dd5a7b0c0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Oct 2023 09:03:37 -0700 Subject: [PATCH 105/767] Release notes for 0.64.5, refs #2197 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9a5290c0..48bf9ef5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_64_5: + +0.64.5 (2023-10-08) +------------------- + +- Dropped dependency on ``click-default-group-wheel``, which could cause a dependency conflict. (:issue:`2197`) + .. _v1_0_a7: 1.0a7 (2023-09-21) From 85a41987c7753c3af92ba6b8b6007211eb46602f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Oct 2023 09:07:11 -0700 Subject: [PATCH 106/767] Fixed typo acepts -> accepts --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 1816d48c..eb6bf4ae 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -488,7 +488,7 @@ This will register ``render_demo`` to be called when paths with the extension `` ``render_demo`` is a Python function. It can be a regular function or an ``async def render_demo()`` awaitable function, depending on if it needs to make any asynchronous calls. -``can_render_demo`` is a Python function (or ``async def`` function) which acepts the same arguments as ``render_demo`` but just returns ``True`` or ``False``. It lets Datasette know if the current SQL query can be represented by the plugin - and hence influnce if a link to this output format is displayed in the user interface. If you omit the ``"can_render"`` key from the dictionary every query will be treated as being supported by the plugin. +``can_render_demo`` is a Python function (or ``async def`` function) which accepts the same arguments as ``render_demo`` but just returns ``True`` or ``False``. It lets Datasette know if the current SQL query can be represented by the plugin - and hence influnce if a link to this output format is displayed in the user interface. If you omit the ``"can_render"`` key from the dictionary every query will be treated as being supported by the plugin. When a request is received, the ``"render"`` callback function is called with zero or more of the following arguments. Datasette will inspect your callback function and pass arguments that match its function signature. From 4e1188f60f8b4f90c32a372f3f70a26a3ebb88ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 8 Oct 2023 09:09:45 -0700 Subject: [PATCH 107/767] Upgrade spellcheck.yml workflow --- .github/workflows/spellcheck.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 722e5c68..0ce9e10c 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,18 +9,13 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 with: - python-version: 3.11 - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: '3.11' + cache: 'pip' + cache-dependency-path: '**/setup.py' - name: Install dependencies run: | pip install -e '.[docs]' From 35deaabcb105903790d18710a26e77545f6852ce Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 12 Oct 2023 09:16:37 -0700 Subject: [PATCH 108/767] Move non-metadata configuration from metadata.yaml to datasette.yaml * Allow and permission blocks moved to datasette.yaml * Documentation updates, initial framework for configuration reference --- datasette/app.py | 6 +- datasette/default_permissions.py | 37 ++-- docs/authentication.rst | 338 ++++++++++++++---------------- docs/configuration.rst | 64 ++++-- docs/custom_templates.rst | 137 ++++++------ docs/facets.rst | 12 +- docs/full_text_search.rst | 4 +- docs/internals.rst | 2 +- docs/metadata.rst | 126 ++++++++--- docs/metadata_doc.py | 21 +- docs/plugins.rst | 30 +-- docs/settings.rst | 5 +- docs/sql_queries.rst | 56 ++--- docs/writing_plugins.rst | 8 +- tests/fixtures.py | 48 ++--- tests/test_canned_queries.py | 14 +- tests/test_html.py | 44 ++-- tests/test_internals_datasette.py | 6 +- tests/test_permissions.py | 138 ++++++------ tests/test_plugins.py | 4 +- tests/test_table_api.py | 2 +- tests/test_table_html.py | 8 +- 22 files changed, 606 insertions(+), 504 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index c0e80700..7dfc63c6 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -721,7 +721,9 @@ class Datasette: return self._app_css_hash async def get_canned_queries(self, database_name, actor): - queries = self.metadata("queries", database=database_name, fallback=False) or {} + queries = ( + ((self.config or {}).get("databases") or {}).get(database_name) or {} + ).get("queries") or {} for more_queries in pm.hook.canned_queries( datasette=self, database=database_name, @@ -1315,7 +1317,7 @@ class Datasette: ): hook = await await_me_maybe(hook) collected.extend(hook) - collected.extend(self.metadata(key) or []) + collected.extend((self.config or {}).get(key) or []) output = [] for url_or_dict in collected: if isinstance(url_or_dict, dict): diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index 5a99d0d8..d29dbe84 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -144,14 +144,14 @@ def permission_allowed_default(datasette, actor, action, resource): "view-query", "execute-sql", ): - result = await _resolve_metadata_view_permissions( + result = await _resolve_config_view_permissions( datasette, actor, action, resource ) if result is not None: return result # Check custom permissions: blocks - result = await _resolve_metadata_permissions_blocks( + result = await _resolve_config_permissions_blocks( datasette, actor, action, resource ) if result is not None: @@ -164,10 +164,10 @@ def permission_allowed_default(datasette, actor, action, resource): return inner -async def _resolve_metadata_permissions_blocks(datasette, actor, action, resource): +async def _resolve_config_permissions_blocks(datasette, actor, action, resource): # Check custom permissions: blocks - metadata = datasette.metadata() - root_block = (metadata.get("permissions", None) or {}).get(action) + config = datasette.config or {} + root_block = (config.get("permissions", None) or {}).get(action) if root_block: root_result = actor_matches_allow(actor, root_block) if root_result is not None: @@ -180,7 +180,7 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc else: database = resource[0] database_block = ( - (metadata.get("databases", {}).get(database, {}).get("permissions", None)) or {} + (config.get("databases", {}).get(database, {}).get("permissions", None)) or {} ).get(action) if database_block: database_result = actor_matches_allow(actor, database_block) @@ -192,7 +192,7 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc database, table_or_query = resource table_block = ( ( - metadata.get("databases", {}) + config.get("databases", {}) .get(database, {}) .get("tables", {}) .get(table_or_query, {}) @@ -207,7 +207,7 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc # Finally the canned queries query_block = ( ( - metadata.get("databases", {}) + config.get("databases", {}) .get(database, {}) .get("queries", {}) .get(table_or_query, {}) @@ -222,25 +222,30 @@ async def _resolve_metadata_permissions_blocks(datasette, actor, action, resourc return None -async def _resolve_metadata_view_permissions(datasette, actor, action, resource): +async def _resolve_config_view_permissions(datasette, actor, action, resource): + config = datasette.config or {} if action == "view-instance": - allow = datasette.metadata("allow") + allow = config.get("allow") if allow is not None: return actor_matches_allow(actor, allow) elif action == "view-database": - database_allow = datasette.metadata("allow", database=resource) + database_allow = ((config.get("databases") or {}).get(resource) or {}).get( + "allow" + ) if database_allow is None: return None return actor_matches_allow(actor, database_allow) elif action == "view-table": database, table = resource - tables = datasette.metadata("tables", database=database) or {} + tables = ((config.get("databases") or {}).get(database) or {}).get( + "tables" + ) or {} table_allow = (tables.get(table) or {}).get("allow") if table_allow is None: return None return actor_matches_allow(actor, table_allow) elif action == "view-query": - # Check if this query has a "allow" block in metadata + # Check if this query has a "allow" block in config database, query_name = resource query = await datasette.get_canned_query(database, query_name, actor) assert query is not None @@ -250,9 +255,11 @@ async def _resolve_metadata_view_permissions(datasette, actor, action, resource) return actor_matches_allow(actor, allow) elif action == "execute-sql": # Use allow_sql block from database block, or from top-level - database_allow_sql = datasette.metadata("allow_sql", database=resource) + database_allow_sql = ((config.get("databases") or {}).get(resource) or {}).get( + "allow_sql" + ) if database_allow_sql is None: - database_allow_sql = datasette.metadata("allow_sql") + database_allow_sql = config.get("allow_sql") if database_allow_sql is None: return None return actor_matches_allow(actor, database_allow_sql) diff --git a/docs/authentication.rst b/docs/authentication.rst index 1a444d0c..a301113a 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -67,7 +67,7 @@ An **action** is a string describing the action the actor would like to perform. A **resource** is the item the actor wishes to interact with - for example a specific database or table. Some actions, such as ``permissions-debug``, are not associated with a particular resource. -Datasette's built-in view permissions (``view-database``, ``view-table`` etc) default to *allow* - unless you :ref:`configure additional permission rules ` unauthenticated users will be allowed to access content. +Datasette's built-in view permissions (``view-database``, ``view-table`` etc) default to *allow* - unless you :ref:`configure additional permission rules ` unauthenticated users will be allowed to access content. Permissions with potentially harmful effects should default to *deny*. Plugin authors should account for this when designing new plugins - for example, the `datasette-upload-csvs `__ plugin defaults to deny so that installations don't accidentally allow unauthenticated users to create new tables by uploading a CSV file. @@ -76,7 +76,7 @@ Permissions with potentially harmful effects should default to *deny*. Plugin au Defining permissions with "allow" blocks ---------------------------------------- -The standard way to define permissions in Datasette is to use an ``"allow"`` block. This is a JSON document describing which actors are allowed to perform a permission. +The standard way to define permissions in Datasette is to use an ``"allow"`` block :ref:`in the datasette.yaml file `. This is a JSON document describing which actors are allowed to perform a permission. The most basic form of allow block is this (`allow demo `__, `deny demo `__): @@ -186,18 +186,18 @@ The /-/allow-debug tool The ``/-/allow-debug`` tool lets you try out different ``"action"`` blocks against different ``"actor"`` JSON objects. You can try that out here: https://latest.datasette.io/-/allow-debug -.. _authentication_permissions_metadata: +.. _authentication_permissions_config: -Access permissions in metadata -============================== +Access permissions in ``datasette.yaml`` +======================================== -There are two ways to configure permissions using ``metadata.json`` (or ``metadata.yaml``). +There are two ways to configure permissions using ``datasette.yaml`` (or ``datasette.json``). For simple visibility permissions you can use ``"allow"`` blocks in the root, database, table and query sections. For other permissions you can use a ``"permissions"`` block, described :ref:`in the next section `. -You can limit who is allowed to view different parts of your Datasette instance using ``"allow"`` keys in your :ref:`metadata` configuration. +You can limit who is allowed to view different parts of your Datasette instance using ``"allow"`` keys in your :ref:`configuration`. You can control the following: @@ -216,25 +216,25 @@ Access to an instance Here's how to restrict access to your entire Datasette instance to just the ``"id": "root"`` user: .. [[[cog - from metadata_doc import metadata_example - metadata_example(cog, { - "title": "My private Datasette instance", - "allow": { - "id": "root" - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + from metadata_doc import config_example + config_example(cog, """ title: My private Datasette instance allow: id: root + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + title: My private Datasette instance + allow: + id: root + + +.. tab:: datasette.json .. code-block:: json @@ -249,21 +249,22 @@ Here's how to restrict access to your entire Datasette instance to just the ``"i To deny access to all users, you can use ``"allow": false``: .. [[[cog - metadata_example(cog, { - "title": "My entirely inaccessible instance", - "allow": False - }) + config_example(cog, """ + title: My entirely inaccessible instance + allow: false + """) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml - title: My entirely inaccessible instance - allow: false + + title: My entirely inaccessible instance + allow: false -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -283,28 +284,26 @@ Access to specific databases To limit access to a specific ``private.db`` database to just authenticated users, use the ``"allow"`` block like this: .. [[[cog - metadata_example(cog, { - "databases": { - "private": { - "allow": { - "id": "*" - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: private: allow: - id: '*' + id: "*" + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + private: + allow: + id: "*" + + +.. tab:: datasette.json .. code-block:: json @@ -327,34 +326,30 @@ Access to specific tables and views To limit access to the ``users`` table in your ``bakery.db`` database: .. [[[cog - metadata_example(cog, { - "databases": { - "bakery": { - "tables": { - "users": { - "allow": { - "id": "*" - } - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: bakery: tables: users: allow: id: '*' + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + bakery: + tables: + users: + allow: + id: '*' + + +.. tab:: datasette.json .. code-block:: json @@ -385,32 +380,12 @@ This works for SQL views as well - you can list their names in the ``"tables"`` Access to specific canned queries --------------------------------- -:ref:`canned_queries` allow you to configure named SQL queries in your ``metadata.json`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. +:ref:`canned_queries` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. To limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user`: .. [[[cog - metadata_example(cog, { - "databases": { - "dogs": { - "queries": { - "add_name": { - "sql": "INSERT INTO names (name) VALUES (:name)", - "write": True, - "allow": { - "id": ["root"] - } - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: dogs: queries: @@ -420,9 +395,26 @@ To limit access to the ``add_name`` canned query in your ``dogs.db`` database to allow: id: - root + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + dogs: + queries: + add_name: + sql: INSERT INTO names (name) VALUES (:name) + write: true + allow: + id: + - root + + +.. tab:: datasette.json .. code-block:: json @@ -461,19 +453,20 @@ You can alternatively use an ``"allow_sql"`` block to control who is allowed to To prevent any user from executing arbitrary SQL queries, use this: .. [[[cog - metadata_example(cog, { - "allow_sql": False - }) + config_example(cog, """ + allow_sql: false + """) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml - allow_sql: false + + allow_sql: false -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -485,22 +478,22 @@ To prevent any user from executing arbitrary SQL queries, use this: To enable just the :ref:`root user` to execute SQL for all databases in your instance, use the following: .. [[[cog - metadata_example(cog, { - "allow_sql": { - "id": "root" - } - }) + config_example(cog, """ + allow_sql: + id: root + """) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml - allow_sql: - id: root + + allow_sql: + id: root -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -514,28 +507,26 @@ To enable just the :ref:`root user` to execute SQL for all To limit this ability for just one specific database, use this: .. [[[cog - metadata_example(cog, { - "databases": { - "mydatabase": { - "allow_sql": { - "id": "root" - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: mydatabase: allow_sql: id: root + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + mydatabase: + allow_sql: + id: root + + +.. tab:: datasette.json .. code-block:: json @@ -552,33 +543,32 @@ To limit this ability for just one specific database, use this: .. _authentication_permissions_other: -Other permissions in metadata -============================= +Other permissions in ``datasette.yaml`` +======================================= -For all other permissions, you can use one or more ``"permissions"`` blocks in your metadata. +For all other permissions, you can use one or more ``"permissions"`` blocks in your ``datasette.yaml`` configuration file. -To grant access to the :ref:`permissions debug tool ` to all signed in users you can grant ``permissions-debug`` to any actor with an ``id`` matching the wildcard ``*`` by adding this a the root of your metadata: +To grant access to the :ref:`permissions debug tool ` to all signed in users, you can grant ``permissions-debug`` to any actor with an ``id`` matching the wildcard ``*`` by adding this a the root of your configuration: .. [[[cog - metadata_example(cog, { - "permissions": { - "debug-menu": { - "id": "*" - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ permissions: debug-menu: id: '*' + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + permissions: + debug-menu: + id: '*' + + +.. tab:: datasette.json .. code-block:: json @@ -594,31 +584,28 @@ To grant access to the :ref:`permissions debug tool ` to a To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` database: .. [[[cog - metadata_example(cog, { - "databases": { - "docs": { - "permissions": { - "create-table": { - "id": "editor" - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: docs: permissions: create-table: id: editor + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + docs: + permissions: + create-table: + id: editor + + +.. tab:: datasette.json .. code-block:: json @@ -638,27 +625,7 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` And for ``insert-row`` against the ``reports`` table in that ``docs`` database: .. [[[cog - metadata_example(cog, { - "databases": { - "docs": { - "tables": { - "reports": { - "permissions": { - "insert-row": { - "id": "editor" - } - } - } - } - } - } - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ databases: docs: tables: @@ -666,9 +633,24 @@ And for ``insert-row`` against the ``reports`` table in that ``docs`` database: permissions: insert-row: id: editor + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + databases: + docs: + tables: + reports: + permissions: + insert-row: + id: editor + + +.. tab:: datasette.json .. code-block:: json diff --git a/docs/configuration.rst b/docs/configuration.rst index 4a7258b9..4e108602 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -13,15 +13,15 @@ To facilitate this, You can provide a ``datasette.yaml`` configuration file to d .. _configuration_reference: -``datasette.yaml`` reference +``datasette.yaml`` Reference ---------------------------- Here's a full example of all the valid configuration options that can exist inside ``datasette.yaml``. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # Datasette settings block settings: @@ -52,10 +52,11 @@ Here's a full example of all the valid configuration options that can exist insi ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml + # Datasette settings block settings: default_page_size: 50 @@ -82,7 +83,8 @@ Here's a full example of all the valid configuration options that can exist insi datasette-my-plugin: key: valueB -.. tab:: JSON + +.. tab:: datasette.json .. code-block:: json @@ -125,9 +127,9 @@ Settings configuration :ref:`settings` can be configured in ``datasette.yaml`` with the ``settings`` key. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # inside datasette.yaml settings: @@ -137,7 +139,7 @@ Settings configuration ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml @@ -146,7 +148,7 @@ Settings configuration default_allow_sql: off default_page_size: 50 -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -165,9 +167,9 @@ Plugin configuration Configuration for plugins can be defined inside ``datasette.yaml``. For top-level plugin configuration, use the ``plugins`` key. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # inside datasette.yaml plugins: @@ -177,7 +179,7 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml @@ -186,7 +188,7 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve datasette-my-plugin: key: my_value -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -202,9 +204,9 @@ Configuration for plugins can be defined inside ``datasette.yaml``. For top-leve For database level or table level plugin configuration, nest it under the appropriate place under ``databases``. .. [[[cog - from metadata_doc import metadata_example + from metadata_doc import config_example import textwrap - metadata_example(cog, yaml=textwrap.dedent( + config_example(cog, textwrap.dedent( """ # inside datasette.yaml databases: @@ -224,7 +226,7 @@ For database level or table level plugin configuration, nest it under the approp ) .. ]]] -.. tab:: YAML +.. tab:: datasette.yaml .. code-block:: yaml @@ -243,7 +245,7 @@ For database level or table level plugin configuration, nest it under the approp datasette-my-plugin: key: my_value -.. tab:: JSON +.. tab:: datasette.json .. code-block:: json @@ -269,4 +271,30 @@ For database level or table level plugin configuration, nest it under the approp } } } -.. [[[end]]] \ No newline at end of file +.. [[[end]]] + + +.. _configuration_reference_permissions: +Permissions Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO + + +.. _configuration_reference_authentication: +Authentication Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO + +.. _configuration_reference_canned_queries: +Canned Queries Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO + +.. _configuration_reference_css_js: +Extra CSS and JS Configuration +~~~~~~~~~~~~~~~~~~~~ + +TODO diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index c0f64cb5..d8e4ac96 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -10,35 +10,34 @@ Datasette provides a number of ways of customizing the way data is displayed. Custom CSS and JavaScript ------------------------- -When you launch Datasette, you can specify a custom metadata file like this:: +When you launch Datasette, you can specify a custom configuration file like this:: - datasette mydb.db --metadata metadata.yaml + datasette mydb.db --config datasette.yaml -Your ``metadata.yaml`` file can include links that look like this: +Your ``datasette.yaml`` file can include links that look like this: .. [[[cog - from metadata_doc import metadata_example - metadata_example(cog, { - "extra_css_urls": [ - "https://simonwillison.net/static/css/all.bf8cd891642c.css" - ], - "extra_js_urls": [ - "https://code.jquery.com/jquery-3.2.1.slim.min.js" - ] - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + from metadata_doc import config_example + config_example(cog, """ extra_css_urls: - https://simonwillison.net/static/css/all.bf8cd891642c.css extra_js_urls: - https://code.jquery.com/jquery-3.2.1.slim.min.js + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + extra_css_urls: + - https://simonwillison.net/static/css/all.bf8cd891642c.css + extra_js_urls: + - https://code.jquery.com/jquery-3.2.1.slim.min.js + + +.. tab:: datasette.json .. code-block:: json @@ -62,35 +61,30 @@ The extra CSS and JavaScript files will be linked in the ```` of every pag You can also specify a SRI (subresource integrity hash) for these assets: .. [[[cog - metadata_example(cog, { - "extra_css_urls": [ - { - "url": "https://simonwillison.net/static/css/all.bf8cd891642c.css", - "sri": "sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI" - } - ], - "extra_js_urls": [ - { - "url": "https://code.jquery.com/jquery-3.2.1.slim.min.js", - "sri": "sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g=" - } - ] - }) -.. ]]] - -.. tab:: YAML - - .. code-block:: yaml - + config_example(cog, """ extra_css_urls: - url: https://simonwillison.net/static/css/all.bf8cd891642c.css sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI extra_js_urls: - url: https://code.jquery.com/jquery-3.2.1.slim.min.js sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml -.. tab:: JSON + extra_css_urls: + - url: https://simonwillison.net/static/css/all.bf8cd891642c.css + sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI + extra_js_urls: + - url: https://code.jquery.com/jquery-3.2.1.slim.min.js + sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + + +.. tab:: datasette.json .. code-block:: json @@ -115,7 +109,7 @@ This will produce: .. code-block:: html + {% for url in extra_js_urls %} {% endfor %} diff --git a/demos/plugins/example_js_manager_plugins.py b/demos/plugins/example_js_manager_plugins.py new file mode 100644 index 00000000..7db45464 --- /dev/null +++ b/demos/plugins/example_js_manager_plugins.py @@ -0,0 +1,21 @@ +from datasette import hookimpl + +# Test command: +# datasette fixtures.db \ --plugins-dir=demos/plugins/ +# \ --static static:demos/plugins/static + +# Create a set with view names that qualify for this JS, since plugins won't do anything on other pages +# Same pattern as in Nteract data explorer +# https://github.com/hydrosquall/datasette-nteract-data-explorer/blob/main/datasette_nteract_data_explorer/__init__.py#L77 +PERMITTED_VIEWS = {"table", "query", "database"} + + +@hookimpl +def extra_js_urls(view_name): + print(view_name) + if view_name in PERMITTED_VIEWS: + return [ + { + "url": f"/static/table-example-plugins.js", + } + ] diff --git a/demos/plugins/static/table-example-plugins.js b/demos/plugins/static/table-example-plugins.js new file mode 100644 index 00000000..8c19d9a6 --- /dev/null +++ b/demos/plugins/static/table-example-plugins.js @@ -0,0 +1,100 @@ +/** + * Example usage of Datasette JS Manager API + */ + +document.addEventListener("datasette_init", function (evt) { + const { detail: manager } = evt; + // === Demo plugins: remove before merge=== + addPlugins(manager); +}); + +/** + * Examples for to test datasette JS api + */ +const addPlugins = (manager) => { + + manager.registerPlugin("column-name-plugin", { + version: 0.1, + makeColumnActions: (columnMeta) => { + const { column } = columnMeta; + + return [ + { + label: "Copy name to clipboard", + onClick: (evt) => copyToClipboard(column), + }, + { + label: "Log column metadata to console", + onClick: (evt) => console.log(column), + }, + ]; + }, + }); + + manager.registerPlugin("panel-plugin-graphs", { + version: 0.1, + makeAboveTablePanelConfigs: () => { + return [ + { + id: 'first-panel', + label: "First", + render: node => { + const description = document.createElement('p'); + description.innerText = 'Hello world'; + node.appendChild(description); + } + }, + { + id: 'second-panel', + label: "Second", + render: node => { + const iframe = document.createElement('iframe'); + iframe.src = "https://observablehq.com/embed/@d3/sortable-bar-chart?cell=viewof+order&cell=chart"; + iframe.width = 800; + iframe.height = 635; + iframe.frameborder = '0'; + node.appendChild(iframe); + } + }, + ]; + }, + }); + + manager.registerPlugin("panel-plugin-maps", { + version: 0.1, + makeAboveTablePanelConfigs: () => { + return [ + { + // ID only has to be unique within a plugin, manager namespaces for you + id: 'first-map-panel', + label: "Map plugin", + // datasette-vega, leafleft can provide a "render" function + render: node => node.innerHTML = "Here sits a map", + }, + { + id: 'second-panel', + label: "Image plugin", + render: node => { + const img = document.createElement('img'); + img.src = 'https://datasette.io/static/datasette-logo.svg' + node.appendChild(img); + }, + } + ]; + }, + }); + + // Future: dispatch message to some other part of the page with CustomEvent API + // Could use to drive filter/sort query builder actions without page refresh. +} + + + +async function copyToClipboard(str) { + try { + await navigator.clipboard.writeText(str); + } catch (err) { + /** Rejected - text failed to copy to the clipboard. Browsers didn't give permission */ + console.error('Failed to copy: ', err); + } +} From 067cc75dfa01612f9a47815b33804361e18bf5c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Dec 2023 09:49:04 -0800 Subject: [PATCH 114/767] Fixed broken example links in row page documentation --- docs/pages.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pages.rst b/docs/pages.rst index 0ae72351..2ce05428 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -70,10 +70,10 @@ Table cells with extremely long text contents are truncated on the table view ac Rows which are the targets of foreign key references from other tables will show a link to a filtered search for all records that reference that row. Here's an example from the Registers of Members Interests database: -`../people/uk.org.publicwhip%2Fperson%2F10001 `_ +`../people/uk~2Eorg~2Epublicwhip~2Fperson~2F10001 `_ Note that this URL includes the encoded primary key of the record. Here's that same page as JSON: -`../people/uk.org.publicwhip%2Fperson%2F10001.json `_ +`../people/uk~2Eorg~2Epublicwhip~2Fperson~2F10001.json `_ From 89c8ca0f3ff51fcbf5f710c529bc7a3552da0731 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Dec 2023 10:32:55 -0800 Subject: [PATCH 115/767] Fix for round_trip_load() YAML error, refs #2219 --- docs/metadata_doc.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py index 3dc5b5f8..a8f13414 100644 --- a/docs/metadata_doc.py +++ b/docs/metadata_doc.py @@ -1,7 +1,7 @@ import json import textwrap from yaml import safe_dump -from ruamel.yaml import round_trip_load +from ruamel.yaml import YAML def metadata_example(cog, data=None, yaml=None): @@ -11,8 +11,7 @@ def metadata_example(cog, data=None, yaml=None): if yaml: # dedent it first yaml = textwrap.dedent(yaml).strip() - # round_trip_load to preserve key order: - data = round_trip_load(yaml) + data = YAML().load(yaml) output_yaml = yaml else: output_yaml = safe_dump(data, sort_keys=False) @@ -27,8 +26,7 @@ def metadata_example(cog, data=None, yaml=None): def config_example(cog, input): if type(input) is str: - # round_trip_load to preserve key order: - data = round_trip_load(input) + data = YAML().load(input) output_yaml = input else: data = input From 4284c74bc133ab494bf4b6dcd4a20b97b05ebb83 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Dec 2023 10:51:03 -0800 Subject: [PATCH 116/767] db.execute_isolated_fn() method (#2220) Closes #2218 --- datasette/database.py | 61 ++++++++++++++++++++++++------ docs/internals.rst | 19 +++++++++- tests/test_internals_database.py | 65 ++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 12 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index cb01301e..f2c980d7 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -159,6 +159,26 @@ class Database: kwargs["count"] = count return results + async def execute_isolated_fn(self, fn): + # Open a new connection just for the duration of this function + # blocking the write queue to avoid any writes occurring during it + if self.ds.executor is None: + # non-threaded mode + isolated_connection = self.connect(write=True) + try: + result = fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass + return result + else: + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) + async def execute_write_fn(self, fn, block=True): if self.ds.executor is None: # non-threaded mode @@ -166,9 +186,10 @@ class Database: self._write_connection = self.connect(write=True) self.ds._prepare_connection(self._write_connection, self.name) return fn(self._write_connection) + else: + return await self._send_to_write_thread(fn, block) - # threaded mode - task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") + async def _send_to_write_thread(self, fn, block=True, isolated_connection=False): if self._write_queue is None: self._write_queue = queue.Queue() if self._write_thread is None: @@ -176,8 +197,9 @@ class Database: target=self._execute_writes, daemon=True ) self._write_thread.start() + task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") reply_queue = janus.Queue() - self._write_queue.put(WriteTask(fn, task_id, reply_queue)) + self._write_queue.put(WriteTask(fn, task_id, reply_queue, isolated_connection)) if block: result = await reply_queue.async_q.get() if isinstance(result, Exception): @@ -202,12 +224,28 @@ class Database: if conn_exception is not None: result = conn_exception else: - try: - result = task.fn(conn) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - result = e + if task.isolated_connection: + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + result = e + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass + else: + try: + result = task.fn(conn) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + result = e task.reply_queue.sync_q.put(result) async def execute_fn(self, fn): @@ -515,12 +553,13 @@ class Database: class WriteTask: - __slots__ = ("fn", "task_id", "reply_queue") + __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection") - def __init__(self, fn, task_id, reply_queue): + def __init__(self, fn, task_id, reply_queue, isolated_connection): self.fn = fn self.task_id = task_id self.reply_queue = reply_queue + self.isolated_connection = isolated_connection class QueryInterrupted(Exception): diff --git a/docs/internals.rst b/docs/internals.rst index 649ca35d..d269bc7d 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1017,7 +1017,7 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() ` but executes the provided function in an entirely isolated SQLite connection, which is opened, used and then closed again in a single call to this method. + +The :ref:`prepare_connection() ` plugin hook is not executed against this connection. + +This allows plugins to execute database operations that might conflict with how database connections are usually configured. For example, running a ``VACUUM`` operation while bypassing any restrictions placed by the `datasette-sqlite-authorizer `__ plugin. + +Plugins can also use this method to load potentially dangerous SQLite extensions, use them to perform an operation and then have them safely unloaded at the end of the call, without risk of exposing them to other connections. + +Functions run using ``execute_isolated_fn()`` share the same queue as ``execute_write_fn()``, which guarantees that no writes can be executed at the same time as the isolated function is executing. + +The return value of the function will be returned by this method. Any exceptions raised by the function will be raised out of the ``await`` line as well. + .. _database_close: db.close() diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 647ae7bd..e0511100 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -1,6 +1,7 @@ """ Tests for the datasette.database.Database class """ +from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues from datasette.utils.sqlite import sqlite3 from datasette.utils import Column @@ -519,6 +520,70 @@ async def test_execute_write_fn_connection_exception(tmpdir, app_client): app_client.ds.remove_database("immutable-db") +def table_exists(conn, name): + return bool( + conn.execute( + """ + with all_tables as ( + select name from sqlite_master where type = 'table' + union all + select name from temp.sqlite_master where type = 'table' + ) + select 1 from all_tables where name = ? + """, + (name,), + ).fetchall(), + ) + + +def table_exists_checker(name): + def inner(conn): + return table_exists(conn, name) + + return inner + + +@pytest.mark.asyncio +@pytest.mark.parametrize("disable_threads", (False, True)) +async def test_execute_isolated(db, disable_threads): + if disable_threads: + ds = Datasette(memory=True, settings={"num_sql_threads": 0}) + db = ds.add_database(Database(ds, memory_name="test_num_sql_threads_zero")) + + # Create temporary table in write + await db.execute_write( + "create temporary table created_by_write (id integer primary key)" + ) + # Should stay visible to write connection + assert await db.execute_write_fn(table_exists_checker("created_by_write")) + + def create_shared_table(conn): + conn.execute("create table shared (id integer primary key)") + # And a temporary table that should not continue to exist + conn.execute( + "create temporary table created_by_isolated (id integer primary key)" + ) + assert table_exists(conn, "created_by_isolated") + # Also confirm that created_by_write does not exist + return table_exists(conn, "created_by_write") + + # shared should not exist + assert not await db.execute_fn(table_exists_checker("shared")) + + # Create it using isolated + created_by_write_exists = await db.execute_isolated_fn(create_shared_table) + assert not created_by_write_exists + + # shared SHOULD exist now + assert await db.execute_fn(table_exists_checker("shared")) + + # created_by_isolated should not exist, even in write connection + assert not await db.execute_write_fn(table_exists_checker("created_by_isolated")) + + # ... and a second call to isolated should not see that connection either + assert not await db.execute_isolated_fn(table_exists_checker("created_by_isolated")) + + @pytest.mark.asyncio async def test_mtime_ns(db): assert isinstance(db.mtime_ns, int) From 978249beda1a3e7185f61000b0dd57018541c511 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Dec 2023 15:07:42 -0800 Subject: [PATCH 117/767] Removed rogue print("max_csv_mb") Found this while working on #2214 --- datasette/views/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index 0080b33c..db08557e 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -484,7 +484,6 @@ async def stream_csv(datasette, fetch_data, request, database): async def stream_fn(r): nonlocal data, trace - print("max_csv_mb", datasette.setting("max_csv_mb")) limited_writer = LimitedWriter(r, datasette.setting("max_csv_mb")) if trace: await limited_writer.write(preamble) From 872dae1e1a1511e2edfb9d7ddf6ea5096c11d5c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Dec 2023 15:08:11 -0800 Subject: [PATCH 118/767] Fix for CSV labels=on missing foreign key bug, closes #2214 --- datasette/views/base.py | 14 ++++++++------ tests/test_csv.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/datasette/views/base.py b/datasette/views/base.py index db08557e..e59fd683 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -553,16 +553,18 @@ async def stream_csv(datasette, fetch_data, request, database): if cell is None: new_row.extend(("", "")) else: - assert isinstance(cell, dict) - new_row.append(cell["value"]) - new_row.append(cell["label"]) + if not isinstance(cell, dict): + new_row.extend((cell, "")) + else: + new_row.append(cell["value"]) + new_row.append(cell["label"]) else: new_row.append(cell) await writer.writerow(new_row) - except Exception as e: - sys.stderr.write("Caught this error: {}\n".format(e)) + except Exception as ex: + sys.stderr.write("Caught this error: {}\n".format(ex)) sys.stderr.flush() - await r.write(str(e)) + await r.write(str(ex)) return await limited_writer.write(postamble) diff --git a/tests/test_csv.py b/tests/test_csv.py index ed83d685..9f772f89 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -1,3 +1,4 @@ +from datasette.app import Datasette from bs4 import BeautifulSoup as Soup import pytest from .fixtures import ( # noqa @@ -95,6 +96,40 @@ async def test_table_csv_with_nullable_labels(ds_client): assert response.text == EXPECTED_TABLE_WITH_NULLABLE_LABELS_CSV +@pytest.mark.asyncio +async def test_table_csv_with_invalid_labels(): + # https://github.com/simonw/datasette/issues/2214 + ds = Datasette() + await ds.invoke_startup() + db = ds.add_memory_database("db_2214") + await db.execute_write_script( + """ + create table t1 (id integer primary key, name text); + insert into t1 (id, name) values (1, 'one'); + insert into t1 (id, name) values (2, 'two'); + create table t2 (textid text primary key, name text); + insert into t2 (textid, name) values ('a', 'alpha'); + insert into t2 (textid, name) values ('b', 'beta'); + create table if not exists maintable ( + id integer primary key, + fk_integer integer references t1(id), + fk_text text references t2(textid) + ); + insert into maintable (id, fk_integer, fk_text) values (1, 1, 'a'); + insert into maintable (id, fk_integer, fk_text) values (2, 3, 'b'); -- invalid fk_integer + insert into maintable (id, fk_integer, fk_text) values (3, 2, 'c'); -- invalid fk_text + """ + ) + response = await ds.client.get("/db_2214/maintable.csv?_labels=1") + assert response.status_code == 200 + assert response.text == ( + "id,fk_integer,fk_integer_label,fk_text,fk_text_label\r\n" + "1,1,one,a,alpha\r\n" + "2,3,,b,beta\r\n" + "3,2,two,c,\r\n" + ) + + @pytest.mark.asyncio async def test_table_csv_blob_columns(ds_client): response = await ds_client.get("/fixtures/binary_data.csv") From 45b88f2056e0a4da204b50f5e17ba953fcb51865 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 22 Dec 2023 15:14:50 -0800 Subject: [PATCH 119/767] Release notes from 0.64.6, refs #2214 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f2f17a50..af3d2a0b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_64_6: + +0.64.6 (2023-12-22) +------------------- + +- Fixed a bug where CSV export with expanded labels could fail if a foreign key reference did not correctly resolve. (:issue:`2214`) + .. _v0_64_5: 0.64.5 (2023-10-08) From c7a4706bcc0d6736533b91437e54a8af9226a10a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jan 2024 14:33:23 -0800 Subject: [PATCH 120/767] jinja2_environment_from_request() plugin hook Closes #2225 --- datasette/app.py | 49 +++++++++++++++++++++-------------- datasette/handle_exception.py | 3 ++- datasette/hookspecs.py | 5 ++++ datasette/views/base.py | 3 ++- datasette/views/database.py | 6 +++-- datasette/views/table.py | 3 ++- docs/plugin_hooks.rst | 42 ++++++++++++++++++++++++++++++ tests/test_plugins.py | 42 +++++++++++++++++++++++++++++- 8 files changed, 128 insertions(+), 25 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f33865e4..482cebb4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -420,21 +420,31 @@ class Datasette: ), ] ) - self.jinja_env = Environment( + environment = Environment( loader=template_loader, autoescape=True, enable_async=True, # undefined=StrictUndefined, ) - self.jinja_env.filters["escape_css_string"] = escape_css_string - self.jinja_env.filters["quote_plus"] = urllib.parse.quote_plus - self.jinja_env.filters["escape_sqlite"] = escape_sqlite - self.jinja_env.filters["to_css_class"] = to_css_class + environment.filters["escape_css_string"] = escape_css_string + environment.filters["quote_plus"] = urllib.parse.quote_plus + self._jinja_env = environment + environment.filters["escape_sqlite"] = escape_sqlite + environment.filters["to_css_class"] = to_css_class self._register_renderers() self._permission_checks = collections.deque(maxlen=200) self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self) + def get_jinja_environment(self, request: Request = None) -> Environment: + environment = self._jinja_env + if request: + for environment in pm.hook.jinja2_environment_from_request( + datasette=self, request=request, env=environment + ): + pass + return environment + def get_permission(self, name_or_abbr: str) -> "Permission": """ Returns a Permission object for the given name or abbreviation. Raises KeyError if not found. @@ -514,7 +524,7 @@ class Datasette: abbrs[p.abbr] = p self.permissions[p.name] = p for hook in pm.hook.prepare_jinja2_environment( - env=self.jinja_env, datasette=self + env=self._jinja_env, datasette=self ): await await_me_maybe(hook) for hook in pm.hook.startup(datasette=self): @@ -1218,7 +1228,7 @@ class Datasette: else: if isinstance(templates, str): templates = [templates] - template = self.jinja_env.select_template(templates) + template = self.get_jinja_environment(request).select_template(templates) if dataclasses.is_dataclass(context): context = dataclasses.asdict(context) body_scripts = [] @@ -1568,16 +1578,6 @@ class DatasetteRouter: def __init__(self, datasette, routes): self.ds = datasette self.routes = routes or [] - # Build a list of pages/blah/{name}.html matching expressions - pattern_templates = [ - filepath - for filepath in self.ds.jinja_env.list_templates() - if "{" in filepath and filepath.startswith("pages/") - ] - self.page_routes = [ - (route_pattern_from_filepath(filepath[len("pages/") :]), filepath) - for filepath in pattern_templates - ] async def __call__(self, scope, receive, send): # Because we care about "foo/bar" v.s. "foo%2Fbar" we decode raw_path ourselves @@ -1677,13 +1677,24 @@ class DatasetteRouter: route_path = request.scope.get("route_path", request.scope["path"]) # Jinja requires template names to use "/" even on Windows template_name = "pages" + route_path + ".html" + # Build a list of pages/blah/{name}.html matching expressions + environment = self.ds.get_jinja_environment(request) + pattern_templates = [ + filepath + for filepath in environment.list_templates() + if "{" in filepath and filepath.startswith("pages/") + ] + page_routes = [ + (route_pattern_from_filepath(filepath[len("pages/") :]), filepath) + for filepath in pattern_templates + ] try: - template = self.ds.jinja_env.select_template([template_name]) + template = environment.select_template([template_name]) except TemplateNotFound: template = None if template is None: # Try for a pages/blah/{name}.html template match - for regex, wildcard_template in self.page_routes: + for regex, wildcard_template in page_routes: match = regex.match(route_path) if match is not None: context.update(match.groupdict()) diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index 8b7e83e3..bef6b4ee 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -57,7 +57,8 @@ def handle_exception(datasette, request, exception): if request.path.split("?")[0].endswith(".json"): return Response.json(info, status=status, headers=headers) else: - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) return Response.html( await template.render_async( dict( diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 9069927b..b6975dce 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -99,6 +99,11 @@ def actors_from_ids(datasette, actor_ids): """Returns a dictionary mapping those IDs to actor dictionaries""" +@hookspec +def jinja2_environment_from_request(datasette, request, env): + """Return a Jinja2 environment based on the incoming request""" + + @hookspec def filters_from_request(request, database, table, datasette): """ diff --git a/datasette/views/base.py b/datasette/views/base.py index e59fd683..bdc1e9cf 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -143,7 +143,8 @@ class BaseView: async def render(self, templates, request, context=None): context = context or {} - template = self.ds.jinja_env.select_template(templates) + environment = self.ds.get_jinja_environment(request) + template = environment.select_template(templates) template_context = { **context, **{ diff --git a/datasette/views/database.py b/datasette/views/database.py index 9ba5ce94..03e70379 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -143,7 +143,8 @@ class DatabaseView(View): datasette.urls.path(path_with_format(request=request, format="json")), ) templates = (f"database-{to_css_class(database)}.html", "database.html") - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) context = { **json_data, "database_color": db.color, @@ -594,7 +595,8 @@ class QueryView(View): f"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html", ) - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) alternate_url_json = datasette.absolute_url( request, datasette.urls.path(path_with_format(request=request, format="json")), diff --git a/datasette/views/table.py b/datasette/views/table.py index 4f4baeed..7ee5d6bf 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -806,7 +806,8 @@ async def table_view_traced(datasette, request): f"table-{to_css_class(resolved.db.name)}-{to_css_class(resolved.table)}.html", "table.html", ] - template = datasette.jinja_env.select_template(templates) + environment = datasette.get_jinja_environment(request) + template = environment.select_template(templates) alternate_url_json = datasette.absolute_url( request, datasette.urls.path(path_with_format(request=request, format="json")), diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index eb6bf4ae..f67d15d6 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1128,6 +1128,48 @@ These IDs could be integers or strings, depending on how the actors used by the Example: `datasette-remote-actors `_ +.. _plugin_hook_jinja2_environment_from_request: + +jinja2_environment_from_request(datasette, request, env) +-------------------------------------------------------- + +``datasette`` - :ref:`internals_datasette` + A Datasette instance. + +``request`` - :ref:`internals_request` or ``None`` + The current HTTP request, if one is available. + +``env`` - ``Environment`` + The Jinja2 environment that will be used to render the current page. + +This hook can be used to return a customized `Jinja environment `__ based on the incoming request. + +If you want to run a single Datasette instance that serves different content for different domains, you can do so like this: + +.. code-block:: python + + from datasette import hookimpl + from jinja2 import ChoiceLoader, FileSystemLoader + + + @hookimpl + def jinja2_environment_from_request(request, env): + if request and request.host == "www.niche-museums.com": + return env.overlay( + loader=ChoiceLoader( + [ + FileSystemLoader( + "/mnt/niche-museums/templates" + ), + env.loader, + ] + ), + enable_async=True, + ) + return env + +This uses the Jinja `overlay() method `__ to create a new environment identical to the default environment except for having a different template loader, which first looks in the ``/mnt/niche-museums/templates`` directory before falling back on the default loader. + .. _plugin_hook_filters_from_request: filters_from_request(request, database, table, datasette) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 82e2f7f1..bdd4ba49 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -16,6 +16,7 @@ from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils.sqlite import sqlite3 from datasette.utils import CustomRow, StartupError from jinja2.environment import Template +from jinja2 import ChoiceLoader, FileSystemLoader import base64 import importlib import json @@ -563,7 +564,8 @@ async def test_hook_register_output_renderer_can_render(ds_client): async def test_hook_prepare_jinja2_environment(ds_client): ds_client.ds._HELLO = "HI" await ds_client.ds.invoke_startup() - template = ds_client.ds.jinja_env.from_string( + environment = ds_client.ds.get_jinja_environment(None) + template = environment.from_string( "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}, {{ b|select_times_three }}", {"a": 3412341, "b": 5}, ) @@ -1294,3 +1296,41 @@ async def test_plugin_is_installed(): finally: pm.unregister(name="DummyPlugin") + + +@pytest.mark.asyncio +async def test_hook_jinja2_environment_from_request(tmpdir): + templates = pathlib.Path(tmpdir / "templates") + templates.mkdir() + (templates / "index.html").write_text("Hello museums!", "utf-8") + + class EnvironmentPlugin: + @hookimpl + def jinja2_environment_from_request(self, request, env): + if request and request.host == "www.niche-museums.com": + return env.overlay( + loader=ChoiceLoader( + [ + FileSystemLoader(str(templates)), + env.loader, + ] + ), + enable_async=True, + ) + return env + + datasette = Datasette(memory=True) + + try: + pm.register(EnvironmentPlugin(), name="EnvironmentPlugin") + response = await datasette.client.get("/") + assert response.status_code == 200 + assert "Hello museums!" not in response.text + # Try again with the hostname + response2 = await datasette.client.get( + "/", headers={"host": "www.niche-museums.com"} + ) + assert response2.status_code == 200 + assert "Hello museums!" in response2.text + finally: + pm.unregister(name="EnvironmentPlugin") From 1fc76fee6268c21003c0fe730cc8e93210ce6bb8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 5 Jan 2024 16:59:25 -0800 Subject: [PATCH 121/767] 1.0a8.dev1 version number Not going to release this to PyPI but I will build my own wheel of it --- datasette/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 55e2cd42..75d44727 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a7" +__version__ = "1.0a8.dev1" __version_info__ = tuple(__version__.split(".")) From 0b2c6a7ebd4fd540d9bdfb169c621452d280e608 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jan 2024 13:12:57 -0800 Subject: [PATCH 122/767] Fix for ?_extra=columns bug, closes #2230 Also refs #262 - started a test suite for extras. --- datasette/renderer.py | 2 +- tests/test_table_api.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/datasette/renderer.py b/datasette/renderer.py index 224031a7..a446e69d 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -68,7 +68,7 @@ def json_renderer(request, args, data, error, truncated=None): elif shape in ("objects", "object", "array"): columns = data.get("columns") rows = data.get("rows") - if rows and columns: + if rows and columns and not isinstance(rows[0], dict): data["rows"] = [dict(zip(columns, row)) for row in rows] if shape == "object": shape_error = None diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 5dbb8b8f..ae4fdb17 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1362,3 +1362,27 @@ async def test_col_nocol_errors(ds_client, path, expected_error): response = await ds_client.get(path) assert response.status_code == 400 assert response.json()["error"] == expected_error + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "extra,expected_json", + ( + ( + "columns", + { + "ok": True, + "next": None, + "columns": ["id", "content", "content2"], + "rows": [{"id": "1", "content": "hey", "content2": "world"}], + "truncated": False, + }, + ), + ), +) +async def test_table_extras(ds_client, extra, expected_json): + response = await ds_client.get( + "/fixtures/primary_key_multiple_columns.json?_extra=" + extra + ) + assert response.status_code == 200 + assert response.json() == expected_json From 2ff4d4a60a348c143f79d63c48c329ffd0c1f02f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jan 2024 13:13:53 -0800 Subject: [PATCH 123/767] Test for ?_extra=count, refs #262 --- tests/test_table_api.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_table_api.py b/tests/test_table_api.py index ae4fdb17..bde7a38e 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1378,6 +1378,16 @@ async def test_col_nocol_errors(ds_client, path, expected_error): "truncated": False, }, ), + ( + "count", + { + "ok": True, + "next": None, + "rows": [{"id": "1", "content": "hey", "content2": "world"}], + "truncated": False, + "count": 1, + }, + ), ), ) async def test_table_extras(ds_client, extra, expected_json): From 48148e66a846d585e08ec6ab4ae3da8e60d55ab5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 10:42:36 -0800 Subject: [PATCH 124/767] Link from actors_from_ids plugin hook docs to datasette.actors_from_ids() --- docs/plugin_hooks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f67d15d6..9115c3df 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1086,6 +1086,8 @@ The hook must return a dictionary that maps the incoming actor IDs to their full Some plugins that implement social features may store the ID of the :ref:`actor ` that performed an action - added a comment, bookmarked a table or similar - and then need a way to resolve those IDs into display-friendly actor dictionaries later on. +The :ref:`await datasette.actors_from_ids(actor_ids) ` internal method can be used to look up actors from their IDs. It will dispatch to the first plugin that implements this hook. + Unlike other plugin hooks, this only uses the first implementation of the hook to return a result. You can expect users to only have a single plugin installed that implements this hook. If no plugin is installed, Datasette defaults to returning actors that are just ``{"id": actor_id}``. From 7506a89be0d1c97632bed47635eb90f92815d6c7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 13:04:34 -0800 Subject: [PATCH 125/767] Docs on datasette.client for tests, closes #1830 Also covers ds.client.actor_cookie() helper --- docs/testing_plugins.rst | 28 +++++++++++++++++++++++++++ tests/test_docs.py | 41 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 6d2097ad..e10514c6 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -82,6 +82,34 @@ This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepar If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request. +.. _testing_datasette_client: + +Using datasette.client in tests +------------------------------- + +The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client `__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test. + +I simple test looks like this: + +.. literalinclude:: ../tests/test_docs.py + :language: python + :start-after: # -- start test_homepage -- + :end-before: # -- end test_homepage -- + +Or for a JSON API: + +.. literalinclude:: ../tests/test_docs.py + :language: python + :start-after: # -- start test_actor_is_null -- + :end-before: # -- end test_actor_is_null -- + +To make requests as an authenticated actor, create a signed ``ds_cookie`` using the ``datasette.client.actor_cookie()`` helper function and pass it in ``cookies=`` like this: + +.. literalinclude:: ../tests/test_docs.py + :language: python + :start-after: # -- start test_signed_cookie_actor -- + :end-before: # -- end test_signed_cookie_actor -- + .. _testing_plugins_pdb: Using pdb for errors thrown inside Datasette diff --git a/tests/test_docs.py b/tests/test_docs.py index e9b813fe..fdd44788 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,9 +1,8 @@ """ Tests to ensure certain things are documented. """ -from click.testing import CliRunner from datasette import app, utils -from datasette.cli import cli +from datasette.app import Datasette from datasette.filters import Filters from pathlib import Path import pytest @@ -102,3 +101,41 @@ def documented_fns(): @pytest.mark.parametrize("fn", utils.functions_marked_as_documented) def test_functions_marked_with_documented_are_documented(documented_fns, fn): assert fn.__name__ in documented_fns + + +# Tests for testing_plugins.rst documentation + + +# -- start test_homepage -- +@pytest.mark.asyncio +async def test_homepage(): + ds = Datasette(memory=True) + response = await ds.client.get("/") + html = response.text + assert "

" in html + + +# -- end test_homepage -- + + +# -- start test_actor_is_null -- +@pytest.mark.asyncio +async def test_actor_is_null(): + ds = Datasette(memory=True) + response = await ds.client.get("/-/actor.json") + assert response.json() == {"actor": None} + + +# -- end test_actor_is_null -- + + +# -- start test_signed_cookie_actor -- +@pytest.mark.asyncio +async def test_signed_cookie_actor(): + ds = Datasette(memory=True) + cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} + response = await ds.client.get("/-/actor.json", cookies=cookies) + assert response.json() == {"actor": {"id": "root"}} + + +# -- end test_signed_cookie_actor -- From 0f63cb83ed31753a9bd9ec5cc71de16906767337 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 13:08:52 -0800 Subject: [PATCH 126/767] Typo fix --- docs/testing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index e10514c6..33ac4b22 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -89,7 +89,7 @@ Using datasette.client in tests The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client `__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test. -I simple test looks like this: +A simple test looks like this: .. literalinclude:: ../tests/test_docs.py :language: python From a25bf6bea789c409580386f77b7440ff525d09b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jan 2024 14:10:40 -0800 Subject: [PATCH 127/767] fmt: off to fix problem with Black, closes #2231 --- tests/test_docs.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index fdd44788..17c01a0b 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -105,7 +105,7 @@ def test_functions_marked_with_documented_are_documented(documented_fns, fn): # Tests for testing_plugins.rst documentation - +# fmt: off # -- start test_homepage -- @pytest.mark.asyncio async def test_homepage(): @@ -113,8 +113,6 @@ async def test_homepage(): response = await ds.client.get("/") html = response.text assert "

" in html - - # -- end test_homepage -- @@ -124,8 +122,6 @@ async def test_actor_is_null(): ds = Datasette(memory=True) response = await ds.client.get("/-/actor.json") assert response.json() == {"actor": None} - - # -- end test_actor_is_null -- @@ -136,6 +132,4 @@ async def test_signed_cookie_actor(): cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} response = await ds.client.get("/-/actor.json", cookies=cookies) assert response.json() == {"actor": {"id": "root"}} - - # -- end test_signed_cookie_actor -- From 7a5adb592ae6674a2058639c66e85eb1b49448fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jan 2024 14:12:14 -0800 Subject: [PATCH 128/767] Docs on temporary plugins in fixtures, closes #2234 --- docs/testing_plugins.rst | 16 ++++++++++++++++ tests/test_docs_plugins.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tests/test_docs_plugins.py diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 33ac4b22..f1363fb4 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -313,3 +313,19 @@ When writing tests for plugins you may find it useful to register a test plugin assert response.status_code == 500 finally: pm.unregister(name="undo") + +To reuse the same temporary plugin in multiple tests, you can register it inside a fixture in your ``conftest.py`` file like this: + +.. literalinclude:: ../tests/test_docs_plugins.py + :language: python + :start-after: # -- start datasette_with_plugin_fixture -- + :end-before: # -- end datasette_with_plugin_fixture -- + +Note the ``yield`` statement here - this ensures that the ``finally:`` block that unregisters the plugin is executed only after the test function itself has completed. + +Then in a test: + +.. literalinclude:: ../tests/test_docs_plugins.py + :language: python + :start-after: # -- start datasette_with_plugin_test -- + :end-before: # -- end datasette_with_plugin_test -- diff --git a/tests/test_docs_plugins.py b/tests/test_docs_plugins.py new file mode 100644 index 00000000..92b4514c --- /dev/null +++ b/tests/test_docs_plugins.py @@ -0,0 +1,34 @@ +# fmt: off +# -- start datasette_with_plugin_fixture -- +from datasette import hookimpl +from datasette.app import Datasette +from datasette.plugins import pm +import pytest +import pytest_asyncio + + +@pytest_asyncio.fixture +async def datasette_with_plugin(): + class TestPlugin: + __name__ = "TestPlugin" + + @hookimpl + def register_routes(self): + return [ + (r"^/error$", lambda: 1 / 0), + ] + + pm.register(TestPlugin(), name="undo") + try: + yield Datasette() + finally: + pm.unregister(name="undo") +# -- end datasette_with_plugin_fixture -- + + +# -- start datasette_with_plugin_test -- +@pytest.mark.asyncio +async def test_error(datasette_with_plugin): + response = await datasette_with_plugin.client.get("/error") + assert response.status_code == 500 +# -- end datasette_with_plugin_test -- From c3caf36af7db79336a5c8e697b2374e90e34ff5d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 30 Jan 2024 19:54:03 -0800 Subject: [PATCH 129/767] Template slot family of plugin hooks - top_homepage() and others New plugin hooks: top_homepage top_database top_table top_row top_query top_canned_query New datasette.utils.make_slot_function() Closes #1191 --- datasette/hookspecs.py | 30 ++++++++ datasette/templates/database.html | 2 + datasette/templates/index.html | 2 + datasette/templates/query.html | 2 + datasette/templates/row.html | 2 + datasette/templates/table.html | 2 + datasette/utils/__init__.py | 17 +++++ datasette/views/database.py | 21 +++++- datasette/views/index.py | 9 ++- datasette/views/row.py | 12 ++- datasette/views/table.py | 8 ++ docs/plugin_hooks.rst | 119 ++++++++++++++++++++++++++++++ tests/test_docs.py | 4 +- tests/test_plugins.py | 101 +++++++++++++++++++++++++ 14 files changed, 324 insertions(+), 7 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index b6975dce..2f4c6027 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -158,3 +158,33 @@ def skip_csrf(datasette, scope): @hookspec def handle_exception(datasette, request, exception): """Handle an uncaught exception. Can return a Response or None.""" + + +@hookspec +def top_homepage(datasette, request): + """HTML to include at the top of the homepage""" + + +@hookspec +def top_database(datasette, request, database): + """HTML to include at the top of the database page""" + + +@hookspec +def top_table(datasette, request, database, table): + """HTML to include at the top of the table page""" + + +@hookspec +def top_row(datasette, request, database, table, row): + """HTML to include at the top of the row page""" + + +@hookspec +def top_query(datasette, request, database, sql): + """HTML to include at the top of the query results page""" + + +@hookspec +def top_canned_query(datasette, request, database, query_name): + """HTML to include at the top of the canned query page""" diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 3d4dae07..4b125a44 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -34,6 +34,8 @@

{% endif %}
+{{ top_database() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if allow_execute_sql %} diff --git a/datasette/templates/index.html b/datasette/templates/index.html index 06e09635..203abca8 100644 --- a/datasette/templates/index.html +++ b/datasette/templates/index.html @@ -7,6 +7,8 @@ {% block content %}

{{ metadata.title or "Datasette" }}{% if private %} 🔒{% endif %}

+{{ top_homepage() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% for database in databases %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index b8f06f84..1815e592 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -30,6 +30,8 @@

{{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

+{% if canned_query %}{{ top_canned_query() }}{% else %}{{ top_query() }}{% endif %} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 4d179a85..6d4b996e 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -22,6 +22,8 @@ {% block content %}

{{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}

+{{ top_row() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}

This data as {% for name, url in renderers.items() %}{{ name }}{{ ", " if not loop.last }}{% endfor %}

diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 88580e52..5aee6319 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -45,6 +45,8 @@
{% endif %}
+{{ top_table() }} + {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if metadata.get("columns") %} diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 0f449b89..8914c043 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1283,3 +1283,20 @@ def fail_if_plugins_in_metadata(metadata: dict, filename=None): f'Datasette no longer accepts plugin configuration in --metadata. Move your "plugins" configuration blocks to a separate file - we suggest calling that datasette.{suggested_extension} - and start Datasette with datasette -c datasette.{suggested_extension}. See https://docs.datasette.io/en/latest/configuration.html for more details.' ) return metadata + + +def make_slot_function(name, datasette, request, **kwargs): + from datasette.plugins import pm + + method = getattr(pm.hook, name, None) + assert method is not None, "No hook found for {}".format(name) + + async def inner(): + html_bits = [] + for hook in method(datasette=datasette, request=request, **kwargs): + html = await await_me_maybe(hook) + if html is not None: + html_bits.append(html) + return markupsafe.Markup("".join(html_bits)) + + return inner diff --git a/datasette/views/database.py b/datasette/views/database.py index 03e70379..caeb4e46 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,5 +1,4 @@ from dataclasses import dataclass, field -from typing import Callable from urllib.parse import parse_qsl, urlencode import asyncio import hashlib @@ -18,6 +17,7 @@ from datasette.utils import ( call_with_supported_arguments, derive_named_parameters, format_bytes, + make_slot_function, tilde_decode, to_css_class, validate_sql_select, @@ -161,6 +161,9 @@ class DatabaseView(View): f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates ], + "top_database": make_slot_function( + "top_database", datasette, request, database=database + ), } return Response.html( await datasette.render_template( @@ -246,6 +249,12 @@ class QueryContext: "help": "List of templates that were considered for rendering this page" } ) + top_query: callable = field( + metadata={"help": "Callable to render the top_query slot"} + ) + top_canned_query: callable = field( + metadata={"help": "Callable to render the top_canned_query slot"} + ) async def get_tables(datasette, request, db): @@ -727,6 +736,16 @@ class QueryView(View): f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates ], + top_query=make_slot_function( + "top_query", datasette, request, database=database, sql=sql + ), + top_canned_query=make_slot_function( + "top_canned_query", + datasette, + request, + database=database, + query_name=canned_query["name"] if canned_query else None, + ), ), request=request, view_name="database", diff --git a/datasette/views/index.py b/datasette/views/index.py index 95b29302..595cf234 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -1,10 +1,12 @@ -import hashlib import json -from datasette.utils import add_cors_headers, CustomJSONEncoder +from datasette.plugins import pm +from datasette.utils import add_cors_headers, make_slot_function, CustomJSONEncoder from datasette.utils.asgi import Response from datasette.version import __version__ +from markupsafe import Markup + from .base import BaseView @@ -142,5 +144,8 @@ class IndexView(BaseView): "private": not await self.ds.permission_allowed( None, "view-instance" ), + "top_homepage": make_slot_function( + "top_homepage", self.ds, request + ), }, ) diff --git a/datasette/views/row.py b/datasette/views/row.py index 8f07a662..ce877753 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -2,11 +2,9 @@ from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted from .base import DataView, BaseView, _error from datasette.utils import ( - tilde_decode, - urlsafe_components, + make_slot_function, to_css_class, escape_sqlite, - row_sql_params_pks, ) import json import sqlite_utils @@ -73,6 +71,14 @@ class RowView(DataView): .get(database, {}) .get("tables", {}) .get(table, {}), + "top_row": make_slot_function( + "top_row", + self.ds, + request, + database=resolved.db.name, + table=resolved.table, + row=rows[0], + ), } data = { diff --git a/datasette/views/table.py b/datasette/views/table.py index 7ee5d6bf..be7479f8 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -17,6 +17,7 @@ from datasette.utils import ( append_querystring, compound_keys_after_sql, format_bytes, + make_slot_function, tilde_encode, escape_sqlite, filters_should_redirect, @@ -842,6 +843,13 @@ async def table_view_traced(datasette, request): f"{'*' if template_name == template.name else ''}{template_name}" for template_name in templates ], + top_table=make_slot_function( + "top_table", + datasette, + request, + database=resolved.db.name, + table=resolved.table, + ), ), request=request, view_name="table", diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 9115c3df..ce648ba7 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1641,3 +1641,122 @@ This hook is responsible for returning a dictionary corresponding to Datasette : return metadata Example: `datasette-remote-metadata plugin `__ + +.. _plugin_hook_slots: + +Template slots +-------------- + +The following set of plugin hooks can be used to return extra HTML content that will be inserted into the corresponding page, directly below the ``

`` heading. + +Multiple plugins can contribute content here. The order in which it is displayed can be controlled using Pluggy's `call time order options `__. + +Each of these plugin hooks can return either a string or an awaitable function that returns a string. + +.. _plugin_hook_top_homepage: + +top_homepage(datasette, request) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +Returns HTML to be displayed at the top of the Datasette homepage. + +.. _plugin_hook_top_database: + +top_database(datasette, request, database) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +Returns HTML to be displayed at the top of the database page. + +.. _plugin_hook_top_table: + +top_table(datasette, request, database, table) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``table`` - string + The name of the table. + +Returns HTML to be displayed at the top of the table page. + +.. _plugin_hook_top_row: + +top_row(datasette, request, database, table, row) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``table`` - string + The name of the table. + +``row`` - ``sqlite.Row`` + The SQLite row object being displayed. + +Returns HTML to be displayed at the top of the row page. + +.. _plugin_hook_top_query: + +top_query(datasette, request, database, sql) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``sql`` - string + The SQL query. + +Returns HTML to be displayed at the top of the query results page. + +.. _plugin_hook_top_canned_query: + +top_canned_query(datasette, request, database, query_name) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``database`` - string + The name of the database. + +``query_name`` - string + The name of the canned query. + +Returns HTML to be displayed at the top of the canned query page. diff --git a/tests/test_docs.py b/tests/test_docs.py index 17c01a0b..0a803861 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -41,7 +41,9 @@ def plugin_hooks_content(): "plugin", [name for name in dir(app.pm.hook) if not name.startswith("_")] ) def test_plugin_hooks_are_documented(plugin, plugin_hooks_content): - headings = get_headings(plugin_hooks_content, "-") + headings = set() + headings.update(get_headings(plugin_hooks_content, "-")) + headings.update(get_headings(plugin_hooks_content, "~")) assert plugin in headings hook_caller = getattr(app.pm.hook, plugin) arg_names = [a for a in hook_caller.spec.argnames if a != "__multicall__"] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index bdd4ba49..784c460a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1334,3 +1334,104 @@ async def test_hook_jinja2_environment_from_request(tmpdir): assert "Hello museums!" in response2.text finally: pm.unregister(name="EnvironmentPlugin") + + +class SlotPlugin: + __name__ = "SlotPlugin" + + @hookimpl + def top_homepage(self, request): + return "Xtop_homepage:" + request.args["z"] + + @hookimpl + def top_database(self, request, database): + async def inner(): + return "Xtop_database:{}:{}".format(database, request.args["z"]) + + return inner + + @hookimpl + def top_table(self, request, database, table): + return "Xtop_table:{}:{}:{}".format(database, table, request.args["z"]) + + @hookimpl + def top_row(self, request, database, table, row): + return "Xtop_row:{}:{}:{}:{}".format( + database, table, row["name"], request.args["z"] + ) + + @hookimpl + def top_query(self, request, database, sql): + return "Xtop_query:{}:{}:{}".format(database, sql, request.args["z"]) + + @hookimpl + def top_canned_query(self, request, database, query_name): + return "Xtop_query:{}:{}:{}".format(database, query_name, request.args["z"]) + + +@pytest.mark.asyncio +async def test_hook_top_homepage(): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + datasette = Datasette(memory=True) + response = await datasette.client.get("/?z=foo") + assert response.status_code == 200 + assert "Xtop_homepage:foo" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_database(): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + datasette = Datasette(memory=True) + response = await datasette.client.get("/_memory?z=bar") + assert response.status_code == 200 + assert "Xtop_database:_memory:bar" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_table(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures/facetable?z=baz") + assert response.status_code == 200 + assert "Xtop_table:fixtures:facetable:baz" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_row(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures/facet_cities/1?z=bax") + assert response.status_code == 200 + assert "Xtop_row:fixtures:facet_cities:San Francisco:bax" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_query(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures?sql=select+1&z=x") + assert response.status_code == 200 + assert "Xtop_query:fixtures:select 1:x" in response.text + finally: + pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_top_canned_query(ds_client): + try: + pm.register(SlotPlugin(), name="SlotPlugin") + response = await ds_client.get("/fixtures/from_hook?z=xyz") + assert response.status_code == 200 + assert "Xtop_query:fixtures:from_hook:xyz" in response.text + finally: + pm.unregister(name="SlotPlugin") From 5c64af69363100a3c35e6b131efe1f741bbde661 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 30 Jan 2024 19:55:26 -0800 Subject: [PATCH 130/767] Upgrade to latest Black, closes #2239 --- datasette/filters.py | 14 ++++++------ datasette/utils/__init__.py | 14 ++++++------ datasette/utils/shutil_backport.py | 1 + datasette/views/database.py | 22 +++++++++++-------- datasette/views/table.py | 30 ++++++++++++++------------ setup.py | 2 +- tests/plugins/my_plugin.py | 6 +++--- tests/test_api_write.py | 8 ++++--- tests/test_cli.py | 8 ++++--- tests/test_docs.py | 1 + tests/test_internals_database.py | 1 + tests/test_internals_datasette.py | 1 + tests/test_permissions.py | 8 ++++--- tests/test_plugins.py | 34 ++++++++++++++++-------------- tests/test_table_api.py | 8 ++++--- tests/test_utils.py | 1 + 16 files changed, 93 insertions(+), 66 deletions(-) diff --git a/datasette/filters.py b/datasette/filters.py index 5ea3488b..73eea857 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -80,9 +80,9 @@ def search_filters(request, database, table, datasette): "{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})".format( fts_table=escape_sqlite(fts_table), fts_pk=escape_sqlite(fts_pk), - match_clause=":search" - if search_mode_raw - else "escape_fts(:search)", + match_clause=( + ":search" if search_mode_raw else "escape_fts(:search)" + ), ) ) human_descriptions.append(f'search matches "{search}"') @@ -99,9 +99,11 @@ def search_filters(request, database, table, datasette): "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format( fts_table=escape_sqlite(fts_table), search_col=escape_sqlite(search_col), - match_clause=":search_{}".format(i) - if search_mode_raw - else "escape_fts(:search_{})".format(i), + match_clause=( + ":search_{}".format(i) + if search_mode_raw + else "escape_fts(:search_{})".format(i) + ), ) ) human_descriptions.append( diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 8914c043..196e1682 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -402,9 +402,9 @@ def make_dockerfile( apt_get_extras = apt_get_extras_ if spatialite: apt_get_extras.extend(["python3-dev", "gcc", "libsqlite3-mod-spatialite"]) - environment_variables[ - "SQLITE_EXTENSIONS" - ] = "/usr/lib/x86_64-linux-gnu/mod_spatialite.so" + environment_variables["SQLITE_EXTENSIONS"] = ( + "/usr/lib/x86_64-linux-gnu/mod_spatialite.so" + ) return """ FROM python:3.11.0-slim-bullseye COPY . /app @@ -416,9 +416,11 @@ RUN datasette inspect {files} --inspect-file inspect-data.json ENV PORT {port} EXPOSE {port} CMD {cmd}""".format( - apt_get_extras=APT_GET_DOCKERFILE_EXTRAS.format(" ".join(apt_get_extras)) - if apt_get_extras - else "", + apt_get_extras=( + APT_GET_DOCKERFILE_EXTRAS.format(" ".join(apt_get_extras)) + if apt_get_extras + else "" + ), environment_variables="\n".join( [ "ENV {} '{}'".format(key, value) diff --git a/datasette/utils/shutil_backport.py b/datasette/utils/shutil_backport.py index dbe22404..d1fd1bd7 100644 --- a/datasette/utils/shutil_backport.py +++ b/datasette/utils/shutil_backport.py @@ -4,6 +4,7 @@ Backported from Python 3.8. This code is licensed under the Python License: https://github.com/python/cpython/blob/v3.8.3/LICENSE """ + import os from shutil import copy, copy2, copystat, Error diff --git a/datasette/views/database.py b/datasette/views/database.py index caeb4e46..eac01ab6 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -126,9 +126,9 @@ class DatabaseView(View): "views": sql_views, "queries": canned_queries, "allow_execute_sql": allow_execute_sql, - "table_columns": await _table_columns(datasette, database) - if allow_execute_sql - else {}, + "table_columns": ( + await _table_columns(datasette, database) if allow_execute_sql else {} + ), } if format_ == "json": @@ -719,9 +719,11 @@ class QueryView(View): display_rows=await display_rows( datasette, database, request, rows, columns ), - table_columns=await _table_columns(datasette, database) - if allow_execute_sql - else {}, + table_columns=( + await _table_columns(datasette, database) + if allow_execute_sql + else {} + ), columns=columns, renderers=renderers, url_csv=datasette.urls.path( @@ -1036,9 +1038,11 @@ async def display_rows(datasette, database, request, rows, columns): display_value = markupsafe.Markup( '<Binary: {:,} byte{}>'.format( blob_url, - ' title="{}"'.format(formatted) - if "bytes" not in formatted - else "", + ( + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "" + ), len(value), "" if len(value) == 1 else "s", ) diff --git a/datasette/views/table.py b/datasette/views/table.py index be7479f8..2c5e3e13 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -236,9 +236,11 @@ async def display_columns_and_rows( path_from_row_pks(row, pks, not pks), column, ), - ' title="{}"'.format(formatted) - if "bytes" not in formatted - else "", + ( + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "" + ), len(value), "" if len(value) == 1 else "s", ) @@ -289,9 +291,9 @@ async def display_columns_and_rows( "column": column, "value": display_value, "raw": value, - "value_type": "none" - if value is None - else str(type(value).__name__), + "value_type": ( + "none" if value is None else str(type(value).__name__) + ), } ) cell_rows.append(Row(cells)) @@ -974,9 +976,9 @@ async def table_view_data( from_sql = "from {table_name} {where}".format( table_name=escape_sqlite(table_name), - where=("where {} ".format(" and ".join(where_clauses))) - if where_clauses - else "", + where=( + ("where {} ".format(" and ".join(where_clauses))) if where_clauses else "" + ), ) # Copy of params so we can mutate them later: from_sql_params = dict(**params) @@ -1040,10 +1042,12 @@ async def table_view_data( column=escape_sqlite(sort or sort_desc), op=">" if sort else "<", p=len(params), - extra_desc_only="" - if sort - else " or {column2} is null".format( - column2=escape_sqlite(sort or sort_desc) + extra_desc_only=( + "" + if sort + else " or {column2} is null".format( + column2=escape_sqlite(sort or sort_desc) + ) ), next_clauses=" and ".join(next_by_pk_clauses), ) diff --git a/setup.py b/setup.py index d09a9e3d..cd393368 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==23.9.1", + "black==24.1.1", "blacken-docs==1.16.0", "pytest-timeout>=1.4.2", "trustme>=0.7", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index eb70d9bd..9d1f86bc 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -39,9 +39,9 @@ def extra_css_urls(template, database, table, view_name, columns, request, datas "database": database, "table": table, "view_name": view_name, - "request_path": request.path - if request is not None - else None, + "request_path": ( + request.path if request is not None else None + ), "added": ( await datasette.get_database().execute("select 3 * 5") ).first()[0], diff --git a/tests/test_api_write.py b/tests/test_api_write.py index f27d143f..1787e06f 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -279,9 +279,11 @@ async def test_insert_or_upsert_row_errors( json=input, headers={ "Authorization": "Bearer {}".format(token), - "Content-Type": "text/plain" - if special_case == "invalid_content_type" - else "application/json", + "Content-Type": ( + "text/plain" + if special_case == "invalid_content_type" + else "application/json" + ), }, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 213db416..080e8353 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -335,9 +335,11 @@ def test_serve_create(tmpdir): def test_serve_config(tmpdir, argument, format_): config_path = tmpdir / "datasette.{}".format(format_) config_path.write_text( - "settings:\n default_page_size: 5\n" - if format_ == "yaml" - else '{"settings": {"default_page_size": 5}}', + ( + "settings:\n default_page_size: 5\n" + if format_ == "yaml" + else '{"settings": {"default_page_size": 5}}' + ), "utf-8", ) runner = CliRunner() diff --git a/tests/test_docs.py b/tests/test_docs.py index 0a803861..2a58d954 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,6 +1,7 @@ """ Tests to ensure certain things are documented. """ + from datasette import app, utils from datasette.app import Datasette from datasette.filters import Filters diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index e0511100..dd68a6cb 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -1,6 +1,7 @@ """ Tests for the datasette.database.Database class """ + from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues from datasette.utils.sqlite import sqlite3 diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 428b259d..c30bb748 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -1,6 +1,7 @@ """ Tests for the datasette.app.Datasette class """ + import dataclasses from datasette import Forbidden, Context from datasette.app import Datasette, Database diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 933aa07b..9917b749 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -381,9 +381,11 @@ async def test_permissions_debug(ds_client): { "action": div.select_one(".check-action").text, # True = green tick, False = red cross, None = gray None - "result": None - if div.select(".check-result-no-opinion") - else bool(div.select(".check-result-true")), + "result": ( + None + if div.select(".check-result-no-opinion") + else bool(div.select(".check-result-true")) + ), "used_default": bool(div.select(".check-used-default")), } for div in check_divs diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 784c460a..5bfb6132 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1096,24 +1096,26 @@ async def test_hook_filters_from_request(ds_client): @pytest.mark.parametrize("extra_metadata", (False, True)) async def test_hook_register_permissions(extra_metadata): ds = Datasette( - config={ - "plugins": { - "datasette-register-permissions": { - "permissions": [ - { - "name": "extra-from-metadata", - "abbr": "efm", - "description": "Extra from metadata", - "takes_database": False, - "takes_resource": False, - "default": True, - } - ] + config=( + { + "plugins": { + "datasette-register-permissions": { + "permissions": [ + { + "name": "extra-from-metadata", + "abbr": "efm", + "description": "Extra from metadata", + "takes_database": False, + "takes_resource": False, + "default": True, + } + ] + } } } - } - if extra_metadata - else None, + if extra_metadata + else None + ), plugins_dir=PLUGINS_DIR, ) await ds.invoke_startup() diff --git a/tests/test_table_api.py b/tests/test_table_api.py index bde7a38e..58930950 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -305,9 +305,11 @@ async def test_paginate_compound_keys_with_extra_filters(ds_client): "_sort_desc=sortable_with_nulls", lambda row: ( 1 if row["sortable_with_nulls"] is None else 0, - -row["sortable_with_nulls"] - if row["sortable_with_nulls"] is not None - else 0, + ( + -row["sortable_with_nulls"] + if row["sortable_with_nulls"] is not None + else 0 + ), row["content"], ), "sorted by sortable_with_nulls descending", diff --git a/tests/test_utils.py b/tests/test_utils.py index 61392b8b..51577615 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,7 @@ """ Tests for various datasette helper functions. """ + from datasette.app import Datasette from datasette import utils from datasette.utils.asgi import Request From b8230694ff90f9a6cd4f5b7c47fd8a71c831ee1d Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Tue, 30 Jan 2024 22:56:05 -0500 Subject: [PATCH 131/767] Set link to download db to nofollow --- datasette/templates/database.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 4b125a44..ee4dd705 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -97,7 +97,7 @@ {% endif %} {% if allow_download %} -

Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

+

Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}

{% endif %} {% include "_codemirror_foot.html" %} From 04e8835297760416b50cc669ac6f45a7fd68170b Mon Sep 17 00:00:00 2001 From: gerrymanoim Date: Tue, 30 Jan 2024 22:56:32 -0500 Subject: [PATCH 132/767] Remove deprecated/unused args from setup.py (#2222) --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index cd393368..53206eae 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,6 @@ setup( [console_scripts] datasette=datasette.cli:cli """, - setup_requires=["pytest-runner"], extras_require={ "docs": [ "Sphinx==7.2.6", @@ -93,7 +92,6 @@ setup( ], "rich": ["rich"], }, - tests_require=["datasette[test]"], classifiers=[ "Development Status :: 4 - Beta", "Framework :: Datasette", From 959e0202972f4d95088c4c1a9df6274108af8bfb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 30 Jan 2024 20:40:18 -0800 Subject: [PATCH 133/767] Ran blacken-docs --- docs/internals.rst | 3 +-- docs/plugin_hooks.rst | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index d269bc7d..d8f86251 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -210,8 +210,7 @@ To set cookies on the response, use the ``response.set_cookie(...)`` method. The secure=False, httponly=False, samesite="lax", - ): - ... + ): ... You can use this with :ref:`datasette.sign() ` to set signed cookies. Here's how you would set the :ref:`ds_actor cookie ` for use with Datasette :ref:`authentication `: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index ce648ba7..da69c6c9 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -373,8 +373,7 @@ Let's say you want to build a plugin that adds a ``datasette publish my_hosting_ about, about_url, api_key, - ): - ... + ): ... Examples: `datasette-publish-fly `_, `datasette-publish-vercel `_ From 890615b3f29dcf82a792f1a145b02dba784a5b63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 10:53:57 -0800 Subject: [PATCH 134/767] Bump the python-packages group with 1 update (#2241) Bumps the python-packages group with 1 update: [furo](https://github.com/pradyunsg/furo). Updates `furo` from 2023.9.10 to 2024.1.29 - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.09.10...2024.01.29) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-major dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 53206eae..b3915c42 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( extras_require={ "docs": [ "Sphinx==7.2.6", - "furo==2023.9.10", + "furo==2024.1.29", "sphinx-autobuild", "codespell>=2.2.5", "blacken-docs", From bcc4f6bf1f14be6ef693f0b3fc9aa8a027977920 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 15:21:40 -0800 Subject: [PATCH 135/767] track_event() mechanism for analytics and plugins * Closes #2240 * Documentation for event plugin hooks, refs #2240 * Include example track_event plugin in docs, refs #2240 * Tests for track_event() and register_events() hooks, refs #2240 * Initial documentation for core events, refs #2240 * Internals documentation for datasette.track_event() --- datasette/__init__.py | 1 + datasette/app.py | 16 +++ datasette/events.py | 211 ++++++++++++++++++++++++++++++++++++ datasette/hookspecs.py | 10 ++ datasette/plugins.py | 1 + datasette/views/database.py | 6 + datasette/views/row.py | 20 ++++ datasette/views/special.py | 17 ++- datasette/views/table.py | 31 ++++++ docs/conf.py | 2 + docs/events.rst | 14 +++ docs/index.rst | 1 + docs/internals.rst | 20 ++++ docs/plugin_hooks.rst | 100 +++++++++++++++++ docs/plugins.rst | 9 ++ tests/conftest.py | 33 +++++- tests/test_api.py | 7 +- tests/test_api_write.py | 64 +++++++++++ tests/test_auth.py | 19 +++- tests/test_cli.py | 6 +- tests/test_plugins.py | 31 +++++- tests/utils.py | 5 + 22 files changed, 614 insertions(+), 10 deletions(-) create mode 100644 datasette/events.py create mode 100644 docs/events.rst diff --git a/datasette/__init__.py b/datasette/__init__.py index 271e09ad..47d2b4f6 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -1,5 +1,6 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa +from datasette.events import Event # noqa from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa from datasette.utils import actor_matches_allow # noqa from datasette.views import Context # noqa diff --git a/datasette/app.py b/datasette/app.py index 482cebb4..530f79bc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -34,6 +34,7 @@ from jinja2 import ( from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound +from .events import Event from .views import Context from .views.base import ureg from .views.database import database_download, DatabaseView, TableCreateView @@ -505,6 +506,14 @@ class Datasette: # This must be called for Datasette to be in a usable state if self._startup_invoked: return + # Register event classes + event_classes = [] + for hook in pm.hook.register_events(datasette=self): + extra_classes = await await_me_maybe(hook) + if extra_classes: + event_classes.extend(extra_classes) + self.event_classes = tuple(event_classes) + # Register permissions, but watch out for duplicate name/abbr names = {} abbrs = {} @@ -873,6 +882,13 @@ class Datasette: result = await await_me_maybe(result) return result + async def track_event(self, event: Event): + assert isinstance(event, self.event_classes), "Invalid event type: {}".format( + type(event) + ) + for hook in pm.hook.track_event(datasette=self, event=event): + await await_me_maybe(hook) + async def permission_allowed( self, actor, action, resource=None, default=DEFAULT_NOT_SET ): diff --git a/datasette/events.py b/datasette/events.py new file mode 100644 index 00000000..96244779 --- /dev/null +++ b/datasette/events.py @@ -0,0 +1,211 @@ +from abc import ABC, abstractproperty +from dataclasses import asdict, dataclass, field +from datasette.hookspecs import hookimpl +from datetime import datetime, timezone +from typing import Optional + + +@dataclass +class Event(ABC): + @abstractproperty + def name(self): + pass + + created: datetime = field( + init=False, default_factory=lambda: datetime.now(timezone.utc) + ) + actor: Optional[dict] + + def properties(self): + properties = asdict(self) + properties.pop("actor", None) + properties.pop("created", None) + return properties + + +@dataclass +class LoginEvent(Event): + """ + Event name: ``login`` + + A user (represented by ``event.actor``) has logged in. + """ + + name = "login" + + +@dataclass +class LogoutEvent(Event): + """ + Event name: ``logout`` + + A user (represented by ``event.actor``) has logged out. + """ + + name = "logout" + + +@dataclass +class CreateTokenEvent(Event): + """ + Event name: ``create-token`` + + A user created an API token. + + :ivar expires_after: Number of seconds after which this token will expire. + :type expires_after: int or None + :ivar restrict_all: Restricted permissions for this token. + :type restrict_all: list + :ivar restrict_database: Restricted database permissions for this token. + :type restrict_database: dict + :ivar restrict_resource: Restricted resource permissions for this token. + :type restrict_resource: dict + """ + + name = "create-token" + expires_after: Optional[int] + restrict_all: list + restrict_database: dict + restrict_resource: dict + + +@dataclass +class CreateTableEvent(Event): + """ + Event name: ``create-table`` + + A new table has been created in the database. + + :ivar database: The name of the database where the table was created. + :type database: str + :ivar table: The name of the table that was created + :type table: str + :ivar schema: The SQL schema definition for the new table. + :type schema: str + """ + + name = "create-table" + database: str + table: str + schema: str + + +@dataclass +class DropTableEvent(Event): + """ + Event name: ``drop-table`` + + A table has been dropped from the database. + + :ivar database: The name of the database where the table was dropped. + :type database: str + :ivar table: The name of the table that was dropped + :type table: str + """ + + name = "drop-table" + database: str + table: str + + +@dataclass +class InsertRowsEvent(Event): + """ + Event name: ``insert-rows`` + + Rows were inserted into a table. + + :ivar database: The name of the database where the rows were inserted. + :type database: str + :ivar table: The name of the table where the rows were inserted. + :type table: str + :ivar num_rows: The number of rows that were requested to be inserted. + :type num_rows: int + :ivar ignore: Was ignore set? + :type ignore: bool + :ivar replace: Was replace set? + :type replace: bool + """ + + name = "insert-rows" + database: str + table: str + num_rows: int + ignore: bool + replace: bool + + +@dataclass +class UpsertRowsEvent(Event): + """ + Event name: ``upsert-rows`` + + Rows were upserted into a table. + + :ivar database: The name of the database where the rows were inserted. + :type database: str + :ivar table: The name of the table where the rows were inserted. + :type table: str + :ivar num_rows: The number of rows that were requested to be inserted. + :type num_rows: int + """ + + name = "upsert-rows" + database: str + table: str + num_rows: int + + +@dataclass +class UpdateRowEvent(Event): + """ + Event name: ``update-row`` + + A row was updated in a table. + + :ivar database: The name of the database where the row was updated. + :type database: str + :ivar table: The name of the table where the row was updated. + :type table: str + :ivar pks: The primary key values of the updated row. + """ + + name = "update-row" + database: str + table: str + pks: list + + +@dataclass +class DeleteRowEvent(Event): + """ + Event name: ``delete-row`` + + A row was deleted from a table. + + :ivar database: The name of the database where the row was deleted. + :type database: str + :ivar table: The name of the table where the row was deleted. + :type table: str + :ivar pks: The primary key values of the deleted row. + """ + + name = "delete-row" + database: str + table: str + pks: list + + +@hookimpl +def register_events(): + return [ + LoginEvent, + LogoutEvent, + CreateTableEvent, + CreateTokenEvent, + DropTableEvent, + InsertRowsEvent, + UpsertRowsEvent, + UpdateRowEvent, + DeleteRowEvent, + ] diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 2f4c6027..b473f398 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -160,6 +160,16 @@ def handle_exception(datasette, request, exception): """Handle an uncaught exception. Can return a Response or None.""" +@hookspec +def track_event(datasette, event): + """Respond to an event tracked by Datasette""" + + +@hookspec +def register_events(datasette): + """Return a list of Event subclasses to use with track_event()""" + + @hookspec def top_homepage(datasette, request): """HTML to include at the top of the homepage""" diff --git a/datasette/plugins.py b/datasette/plugins.py index 1ed3747f..f7a1905f 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -27,6 +27,7 @@ DEFAULT_PLUGINS = ( "datasette.default_menu_links", "datasette.handle_exception", "datasette.forbidden", + "datasette.events", ) pm = pluggy.PluginManager("datasette") diff --git a/datasette/views/database.py b/datasette/views/database.py index eac01ab6..6d17b16c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,6 +10,7 @@ import re import sqlite_utils import textwrap +from datasette.events import CreateTableEvent from datasette.database import QueryInterrupted from datasette.utils import ( add_cors_headers, @@ -969,6 +970,11 @@ class TableCreateView(BaseView): } if rows: details["row_count"] = len(rows) + await self.ds.track_event( + CreateTableEvent( + request.actor, database=db.name, table=table_name, schema=schema + ) + ) return Response.json(details, status=201) diff --git a/datasette/views/row.py b/datasette/views/row.py index ce877753..7b646641 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -1,5 +1,6 @@ from datasette.utils.asgi import NotFound, Forbidden, Response from datasette.database import QueryInterrupted +from datasette.events import UpdateRowEvent, DeleteRowEvent from .base import DataView, BaseView, _error from datasette.utils import ( make_slot_function, @@ -200,6 +201,15 @@ class RowDeleteView(BaseView): except Exception as e: return _error([str(e)], 500) + await self.ds.track_event( + DeleteRowEvent( + actor=request.actor, + database=resolved.db.name, + table=resolved.table, + pks=resolved.pk_values, + ) + ) + return Response.json({"ok": True}, status=200) @@ -246,4 +256,14 @@ class RowUpdateView(BaseView): ) rows = list(results.rows) result["row"] = dict(rows[0]) + + await self.ds.track_event( + UpdateRowEvent( + actor=request.actor, + database=resolved.db.name, + table=resolved.table, + pks=resolved.pk_values, + ) + ) + return Response.json(result, status=200) diff --git a/datasette/views/special.py b/datasette/views/special.py index 849750bf..4088a1f9 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -1,4 +1,5 @@ import json +from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent from datasette.utils.asgi import Response, Forbidden from datasette.utils import ( actor_matches_allow, @@ -80,9 +81,9 @@ class AuthTokenView(BaseView): if secrets.compare_digest(token, self.ds._root_token): self.ds._root_token = None response = Response.redirect(self.ds.urls.instance()) - response.set_cookie( - "ds_actor", self.ds.sign({"a": {"id": "root"}}, "actor") - ) + root_actor = {"id": "root"} + response.set_cookie("ds_actor", self.ds.sign({"a": root_actor}, "actor")) + await self.ds.track_event(LoginEvent(actor=root_actor)) return response else: raise Forbidden("Invalid token") @@ -105,6 +106,7 @@ class LogoutView(BaseView): response = Response.redirect(self.ds.urls.instance()) response.set_cookie("ds_actor", "", expires=0, max_age=0) self.ds.add_message(request, "You are now logged out", self.ds.WARNING) + await self.ds.track_event(LogoutEvent(actor=request.actor)) return response @@ -349,6 +351,15 @@ class CreateTokenView(BaseView): restrict_resource=restrict_resource, ) token_bits = self.ds.unsign(token[len("dstok_") :], namespace="token") + await self.ds.track_event( + CreateTokenEvent( + actor=request.actor, + expires_after=expires_after, + restrict_all=restrict_all, + restrict_database=restrict_database, + restrict_resource=restrict_resource, + ) + ) context = await self.shared(request) context.update({"errors": errors, "token": token, "token_bits": token_bits}) return await self.render(["create_token.html"], request, context) diff --git a/datasette/views/table.py b/datasette/views/table.py index 2c5e3e13..3b812c01 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -8,6 +8,7 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted +from datasette.events import DropTableEvent, InsertRowsEvent, UpsertRowsEvent from datasette import tracer from datasette.utils import ( add_cors_headers, @@ -467,6 +468,8 @@ class TableInsertView(BaseView): if errors: return _error(errors, 400) + num_rows = len(rows) + # No that we've passed pks to _validate_data it's safe to # fix the rowids case: if not pks: @@ -527,6 +530,29 @@ class TableInsertView(BaseView): result["rows"] = [dict(r) for r in fetched_rows.rows] else: result["rows"] = rows + # We track the number of rows requested, but do not attempt to show which were actually + # inserted or upserted v.s. ignored + if upsert: + await self.ds.track_event( + UpsertRowsEvent( + actor=request.actor, + database=database_name, + table=table_name, + num_rows=num_rows, + ) + ) + else: + await self.ds.track_event( + InsertRowsEvent( + actor=request.actor, + database=database_name, + table=table_name, + num_rows=num_rows, + ignore=bool(ignore), + replace=bool(replace), + ) + ) + return Response.json(result, status=200 if upsert else 201) @@ -587,6 +613,11 @@ class TableDropView(BaseView): sqlite_utils.Database(conn)[table_name].drop() await db.execute_write_fn(drop_table) + await self.ds.track_event( + DropTableEvent( + actor=request.actor, database=database_name, table=table_name + ) + ) return Response.json({"ok": True}, status=200) diff --git a/docs/conf.py b/docs/conf.py index ca0eb986..e13882b2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,6 +40,8 @@ extensions = [ if not os.environ.get("DISABLE_SPHINX_INLINE_TABS"): extensions += ["sphinx_inline_tabs"] +autodoc_member_order = "bysource" + extlinks = { "issue": ("https://github.com/simonw/datasette/issues/%s", "#%s"), } diff --git a/docs/events.rst b/docs/events.rst new file mode 100644 index 00000000..f150ac02 --- /dev/null +++ b/docs/events.rst @@ -0,0 +1,14 @@ +.. _events: + +Events +====== + +Datasette includes a mechanism for tracking events that occur while the software is running. This is primarily intended to be used by plugins, which can both trigger events and listen for events. + +The core Datasette application triggers events when certain things happen. This page describes those events. + +Plugins can listen for events using the :ref:`plugin_hook_track_event` plugin hook, which will be called with instances of the following classes (or additional classes registered by other plugins): + +.. automodule:: datasette.events + :members: + :exclude-members: Event diff --git a/docs/index.rst b/docs/index.rst index 66bbd5a4..ce1ed2eb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -63,5 +63,6 @@ Contents plugin_hooks testing_plugins internals + events contributing changelog diff --git a/docs/internals.rst b/docs/internals.rst index d8f86251..bd7a70b5 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -593,6 +593,26 @@ Using either of these pattern will result in the in-memory database being served This removes a database that has been previously added. ``name=`` is the unique name of that database. +.. _datasette_track_event: + +await .track_event(event) +------------------------- + +``event`` - ``Event`` + An instance of a subclass of ``datasette.events.Event``. + +Plugins can call this to track events, using classes they have previously registered. See :ref:`plugin_event_tracking` for details. + +The event will then be passed to all plugins that have registered to receive events using the :ref:`plugin_hook_track_event` hook. + +Example usage, assuming the plugin has previously registered the ``BanUserEvent`` class: + +.. code-block:: python + + await datasette.track_event( + BanUserEvent(user={"id": 1, "username": "cleverbot"}) + ) + .. _datasette_sign: .sign(value, namespace="default") diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index da69c6c9..1a88cd31 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1759,3 +1759,103 @@ top_canned_query(datasette, request, database, query_name) The name of the canned query. Returns HTML to be displayed at the top of the canned query page. + +.. _plugin_event_tracking: + +Event tracking +-------------- + +Datasette includes an internal mechanism for tracking analytical events. This can be used for analytics, but can also be used by plugins that want to listen out for when key events occur (such as a table being created) and take action in response. + +Plugins can register to receive events using the ``track_event`` plugin hook. + +They can also define their own events for other plugins to receive using the ``register_events`` plugin hook, combined with calls to the ``datasette.track_event(...)`` internal method. + +.. _plugin_hook_track_event: + +track_event(datasette, event) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +``event`` - ``Event`` + Information about the event, represented as an instance of a subclass of the ``Event`` base class. + +This hook will be called any time an event is tracked by code that calls the :ref:`datasette.track_event(...) ` internal method. + +The ``event`` object will always have the following properties: + +- ``name``: a string representing the name of the event, for example ``logout`` or ``create-table``. +- ``actor``: a dictionary representing the actor that triggered the event, or ``None`` if the event was not triggered by an actor. +- ``created``: a ``datatime.datetime`` object in the ``timezone.utc`` timezone representing the time the event object was created. + +Other properties on the event will be available depending on the type of event. You can also access those as a dictionary using ``event.properties()``. + +The events fired by Datasette core are :ref:`documented here `. + +This example plugin logs details of all events to standard error: + +.. code-block:: python + + from datasette import hookimpl + import json + import sys + + + @hookimpl + def track_event(event): + name = event.name + actor = event.actor + properties = event.properties() + msg = json.dumps( + { + "name": name, + "actor": actor, + "properties": properties, + } + ) + print(msg, file=sys.stderr, flush=True) + + +.. _plugin_hook_register_events: + +register_events(datasette) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. + +This hook should return a list of ``Event`` subclasses that represent custom events that the plugin might send to the ``datasette.track_event()`` method. + +This example registers event subclasses for ``ban-user`` and ``unban-user`` events: + +.. code-block:: python + + from dataclasses import dataclass + from datasette import hookimpl, Event + + + @dataclass + class BanUserEvent(Event): + name = "ban-user" + user: dict + + + @dataclass + class UnbanUserEvent(Event): + name = "unban-user" + user: dict + + + @hookimpl + def register_events(): + return [BanUserEvent, UnbanUserEvent] + +The plugin can then call ``datasette.track_event(...)`` to send a ``ban-user`` event: + +.. code-block:: python + + await datasette.track_event( + BanUserEvent(user={"id": 1, "username": "cleverbot"}) + ) diff --git a/docs/plugins.rst b/docs/plugins.rst index 2ec03701..1a72af95 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -228,6 +228,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "skip_csrf" ] }, + { + "name": "datasette.events", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "register_events" + ] + }, { "name": "datasette.facets", "static": false, diff --git a/tests/conftest.py b/tests/conftest.py index 31336aea..445de057 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -import asyncio import httpx import os import pathlib @@ -8,7 +7,8 @@ import re import subprocess import tempfile import time -import trustme +from dataclasses import dataclass, field +from datasette import Event, hookimpl try: @@ -164,6 +164,35 @@ def check_permission_actions_are_documented(): ) +class TrackEventPlugin: + __name__ = "TrackEventPlugin" + + @dataclass + class OneEvent(Event): + name = "one" + + extra: str + + @hookimpl + def register_events(self, datasette): + async def inner(): + return [self.OneEvent] + + return inner + + @hookimpl + def track_event(self, datasette, event): + datasette._tracked_events = getattr(datasette, "_tracked_events", []) + datasette._tracked_events.append(event) + + +@pytest.fixture(scope="session", autouse=True) +def install_event_tracking_plugin(): + from datasette.plugins import pm + + pm.register(TrackEventPlugin(), name="TrackEventPlugin") + + @pytest.fixture(scope="session") def ds_localhost_http_server(): ds_proc = subprocess.Popen( diff --git a/tests/test_api.py b/tests/test_api.py index 93ca43eb..177dc95c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -786,7 +786,12 @@ async def test_threads_json(ds_client): @pytest.mark.asyncio async def test_plugins_json(ds_client): response = await ds_client.get("/-/plugins.json") - assert EXPECTED_PLUGINS == sorted(response.json(), key=lambda p: p["name"]) + # Filter out TrackEventPlugin + actual_plugins = sorted( + [p for p in response.json() if p["name"] != "TrackEventPlugin"], + key=lambda p: p["name"], + ) + assert EXPECTED_PLUGINS == actual_plugins # Try with ?all=1 response = await ds_client.get("/-/plugins.json?all=1") names = {p["name"] for p in response.json()} diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 1787e06f..9caf9fdf 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,5 +1,6 @@ from datasette.app import Datasette from datasette.utils import sqlite3 +from .utils import last_event import pytest import time @@ -49,6 +50,14 @@ async def test_insert_row(ds_write): assert response.json()["rows"] == [expected_row] rows = (await ds_write.get_database("data").execute("select * from docs")).rows assert dict(rows[0]) == expected_row + # Analytics event + event = last_event(ds_write) + assert event.name == "insert-rows" + assert event.num_rows == 1 + assert event.database == "data" + assert event.table == "docs" + assert not event.ignore + assert not event.replace @pytest.mark.asyncio @@ -68,6 +77,16 @@ async def test_insert_rows(ds_write, return_rows): headers=_headers(token), ) assert response.status_code == 201 + + # Analytics event + event = last_event(ds_write) + assert event.name == "insert-rows" + assert event.num_rows == 20 + assert event.database == "data" + assert event.table == "docs" + assert not event.ignore + assert not event.replace + actual_rows = [ dict(r) for r in ( @@ -353,6 +372,16 @@ async def test_insert_ignore_replace( headers=_headers(token), ) assert response.status_code == 201 + + # Analytics event + event = last_event(ds_write) + assert event.name == "insert-rows" + assert event.num_rows == 1 + assert event.database == "data" + assert event.table == "docs" + assert event.ignore == ignore + assert event.replace == replace + actual_rows = [ dict(r) for r in ( @@ -427,6 +456,14 @@ async def test_upsert(ds_write, initial, input, expected_rows, should_return): ) assert response.status_code == 200 assert response.json()["ok"] is True + + # Analytics event + event = last_event(ds_write) + assert event.name == "upsert-rows" + assert event.num_rows == 1 + assert event.database == "data" + assert event.table == "upsert_test" + if should_return: # We only expect it to return rows corresponding to those we sent expected_returned_rows = expected_rows[: len(input["rows"])] @@ -530,6 +567,13 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): headers=_headers(write_token(ds_write)), ) assert delete_response.status_code == 200 + + # Analytics event + event = last_event(ds_write) + assert event.name == "delete-row" + assert event.database == "data" + assert event.table == table + assert event.pks == str(delete_path).split(",") assert ( await ds_write.client.get( "/data.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(table) @@ -610,6 +654,13 @@ async def test_update_row(ds_write, input, expected_errors, use_return): for k, v in input.items(): assert returned_row[k] == v + # Analytics event + event = last_event(ds_write) + assert event.actor == {"id": "root", "token": "dstok"} + assert event.database == "data" + assert event.table == "docs" + assert event.pks == [str(pk)] + # And fetch the row to check it's updated response = await ds_write.client.get( "/data/docs/{}.json?_shape=array".format(pk), @@ -676,6 +727,13 @@ async def test_drop_table(ds_write, scenario): headers=_headers(token), ) assert response2.json() == {"ok": True} + # Check event + event = last_event(ds_write) + assert event.name == "drop-table" + assert event.actor == {"id": "root", "token": "dstok"} + assert event.table == "docs" + assert event.database == "data" + # Table should 404 assert (await ds_write.client.get("/data/docs")).status_code == 404 @@ -1096,6 +1154,12 @@ async def test_create_table(ds_write, input, expected_status, expected_response) assert response.status_code == expected_status data = response.json() assert data == expected_response + # create-table event + if expected_status == 201: + event = last_event(ds_write) + assert event.name == "create-table" + assert event.actor == {"id": "root", "token": "dstok"} + assert event.schema.startswith("CREATE TABLE ") @pytest.mark.asyncio diff --git a/tests/test_auth.py b/tests/test_auth.py index 33cf9b35..f2359df7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,6 +1,6 @@ from bs4 import BeautifulSoup as Soup from .fixtures import app_client -from .utils import cookie_was_deleted +from .utils import cookie_was_deleted, last_event from click.testing import CliRunner from datasette.utils import baseconv from datasette.cli import cli @@ -19,6 +19,10 @@ async def test_auth_token(ds_client): assert {"a": {"id": "root"}} == ds_client.ds.unsign( response.cookies["ds_actor"], "actor" ) + # Should have recorded a login event + event = last_event(ds_client.ds) + assert event.name == "login" + assert event.actor == {"id": "root"} # Check that a second with same token fails assert ds_client.ds._root_token is None assert (await ds_client.get(path)).status_code == 403 @@ -57,7 +61,7 @@ async def test_actor_cookie_that_expires(ds_client, offset, expected): cookie = ds_client.ds.sign( {"a": {"id": "test"}, "e": baseconv.base62.encode(expires_at)}, "actor" ) - response = await ds_client.get("/", cookies={"ds_actor": cookie}) + await ds_client.get("/", cookies={"ds_actor": cookie}) assert ds_client.ds._last_request.scope["actor"] == expected @@ -86,6 +90,10 @@ def test_logout(app_client): csrftoken_from=True, cookies={"ds_actor": app_client.actor_cookie({"id": "test"})}, ) + # Should have recorded a logout event + event = last_event(app_client.ds) + assert event.name == "logout" + assert event.actor == {"id": "test"} # The ds_actor cookie should have been unset assert cookie_was_deleted(response4, "ds_actor") # Should also have set a message @@ -185,6 +193,13 @@ def test_auth_create_token( for error in errors: assert '

{}

'.format(error) in response2.text else: + # Check create-token event + event = last_event(app_client.ds) + assert event.name == "create-token" + assert event.expires_after == expected_duration + assert isinstance(event.restrict_all, list) + assert isinstance(event.restrict_database, dict) + assert isinstance(event.restrict_resource, dict) # Extract token from page token = response2.text.split('value="dstok_')[1].split('"')[0] details = app_client.ds.unsign(token, "token") diff --git a/tests/test_cli.py b/tests/test_cli.py index 080e8353..9cc18c6e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -100,7 +100,11 @@ def test_spatialite_error_if_cannot_find_load_extension_spatialite(): def test_plugins_cli(app_client): runner = CliRunner() result1 = runner.invoke(cli, ["plugins"]) - assert json.loads(result1.output) == EXPECTED_PLUGINS + actual_plugins = sorted( + [p for p in json.loads(result1.output) if p["name"] != "TrackEventPlugin"], + key=lambda p: p["name"], + ) + assert actual_plugins == EXPECTED_PLUGINS # Try with --all result2 = runner.invoke(cli, ["plugins", "--all"]) names = [p["name"] for p in json.loads(result2.output)] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 5bfb6132..dad4f2ca 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -9,8 +9,9 @@ from .fixtures import ( TestClient as _TestClient, ) # noqa from click.testing import CliRunner +from dataclasses import dataclass from datasette.app import Datasette -from datasette import cli, hookimpl, Permission +from datasette import cli, hookimpl, Event, Permission from datasette.filters import FilterArguments from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils.sqlite import sqlite3 @@ -18,6 +19,7 @@ from datasette.utils import CustomRow, StartupError from jinja2.environment import Template from jinja2 import ChoiceLoader, FileSystemLoader import base64 +import datetime import importlib import json import os @@ -1437,3 +1439,30 @@ async def test_hook_top_canned_query(ds_client): assert "Xtop_query:fixtures:from_hook:xyz" in response.text finally: pm.unregister(name="SlotPlugin") + + +@pytest.mark.asyncio +async def test_hook_track_event(): + datasette = Datasette(memory=True) + from .conftest import TrackEventPlugin + + await datasette.invoke_startup() + await datasette.track_event( + TrackEventPlugin.OneEvent(actor=None, extra="extra extra") + ) + assert len(datasette._tracked_events) == 1 + assert isinstance(datasette._tracked_events[0], TrackEventPlugin.OneEvent) + event = datasette._tracked_events[0] + assert event.name == "one" + assert event.properties() == {"extra": "extra extra"} + # Should have a recent created as well + created = event.created + assert isinstance(created, datetime.datetime) + assert created.tzinfo == datetime.timezone.utc + + +@pytest.mark.asyncio +async def test_hook_register_events(): + datasette = Datasette(memory=True) + await datasette.invoke_startup() + assert any(k.__name__ == "OneEvent" for k in datasette.event_classes) diff --git a/tests/utils.py b/tests/utils.py index 84d5b1df..9b31abde 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,11 @@ from datasette.utils.sqlite import sqlite3 +def last_event(datasette): + events = getattr(datasette, "_tracked_events", []) + return events[-1] if events else None + + def assert_footer_links(soup): footer_links = soup.find("footer").findAll("a") assert 4 == len(footer_links) From 2e4a03b2c461ca20ff789146a006ddd126013ee7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 15:31:26 -0800 Subject: [PATCH 136/767] Run coverage on Python 3.12 - #2245 I hoped this would run slightly faster than 3.9 but there doesn't appear to be a performance improvement. --- .github/workflows/test-coverage.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index bd720664..7a08e401 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,18 +15,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: 3.9 - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: '3.12' + cache: 'pip' + cache-dependency-path: '**/setup.py' - name: Install Python dependencies run: | python -m pip install --upgrade pip From bcf7ef963f6e1eb0a64b2a0bb4af0ae7a197d1d1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 19:45:05 -0800 Subject: [PATCH 137/767] YAML/JSON examples for allow blocks --- docs/authentication.rst | 270 ++++++++++++++++++++++++++++++++++------ 1 file changed, 231 insertions(+), 39 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index a301113a..8758765d 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -80,13 +80,35 @@ The standard way to define permissions in Datasette is to use an ``"allow"`` blo The most basic form of allow block is this (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: root + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: root + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { "id": "root" + } } - } +.. [[[end]]] This will match any actors with an ``"id"`` property of ``"root"`` - for example, an actor that looks like this: @@ -99,29 +121,98 @@ This will match any actors with an ``"id"`` property of ``"root"`` - for example An allow block can specify "deny all" using ``false`` (`demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: false + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": false - } +.. tab:: YAML + + .. code-block:: yaml + + allow: false + +.. tab:: JSON + + .. code-block:: json + + { + "allow": false + } +.. [[[end]]] An ``"allow"`` of ``true`` allows all access (`demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: true + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": true - } +.. tab:: YAML + + .. code-block:: yaml + + allow: true + +.. tab:: JSON + + .. code-block:: json + + { + "allow": true + } +.. [[[end]]] Allow keys can provide a list of values. These will match any actor that has any of those values (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: + - simon + - cleopaws + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { - "id": ["simon", "cleopaws"] +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: + - simon + - cleopaws + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { + "id": [ + "simon", + "cleopaws" + ] + } } - } +.. [[[end]]] This will match any actor with an ``"id"`` of either ``"simon"`` or ``"cleopaws"``. @@ -129,53 +220,154 @@ Actors can have properties that feature a list of values. These will be matched .. code-block:: json - { - "id": "simon", - "roles": ["staff", "developer"] - } + { + "id": "simon", + "roles": ["staff", "developer"] + } This allow block will provide access to any actor that has ``"developer"`` as one of their roles (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + roles: + - developer + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { - "roles": ["developer"] +.. tab:: YAML + + .. code-block:: yaml + + allow: + roles: + - developer + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { + "roles": [ + "developer" + ] + } } - } +.. [[[end]]] Note that "roles" is not a concept that is baked into Datasette - it's a convention that plugins can choose to implement and act on. If you want to provide access to any actor with a value for a specific key, use ``"*"``. For example, to match any logged-in user specify the following (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: "*" + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: "*" + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { "id": "*" + } } - } +.. [[[end]]] You can specify that only unauthenticated actors (from anynomous HTTP requests) should be allowed access using the special ``"unauthenticated": true`` key in an allow block (`allow demo `__, `deny demo `__): -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + unauthenticated: true + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { +.. tab:: YAML + + .. code-block:: yaml + + allow: + unauthenticated: true + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { "unauthenticated": true + } } - } +.. [[[end]]] Allow keys act as an "or" mechanism. An actor will be able to execute the query if any of their JSON properties match any of the values in the corresponding lists in the ``allow`` block. The following block will allow users with either a ``role`` of ``"ops"`` OR users who have an ``id`` of ``"simon"`` or ``"cleopaws"``: -.. code-block:: json +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + allow: + id: + - simon + - cleopaws + role: ops + """).strip(), + "YAML", "JSON" + ) +.. ]]] - { - "allow": { - "id": ["simon", "cleopaws"], +.. tab:: YAML + + .. code-block:: yaml + + allow: + id: + - simon + - cleopaws + role: ops + +.. tab:: JSON + + .. code-block:: json + + { + "allow": { + "id": [ + "simon", + "cleopaws" + ], "role": "ops" + } } - } +.. [[[end]]] `Demo for cleopaws `__, `demo for ops role `__, `demo for an actor matching neither rule `__. From b466749e88b2ffbd925b6b3e777c8527ebc54e78 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jan 2024 20:03:19 -0800 Subject: [PATCH 138/767] Filled out docs/configuration.rst, closes #2246 --- docs/changelog.rst | 2 +- docs/configuration.rst | 297 ++++++++++++++++++++++++++++++++++++-- docs/custom_templates.rst | 156 +------------------- docs/metadata_doc.py | 8 +- docs/plugin_hooks.rst | 2 +- docs/plugins.rst | 2 +- 6 files changed, 294 insertions(+), 173 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index af3d2a0b..04ce9583 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -610,7 +610,7 @@ JavaScript modules To use modules, JavaScript needs to be included in `` + +You can also specify a SRI (subresource integrity hash) for these assets: + +.. [[[cog + config_example(cog, """ + extra_css_urls: + - url: https://simonwillison.net/static/css/all.bf8cd891642c.css + sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI + extra_js_urls: + - url: https://code.jquery.com/jquery-3.2.1.slim.min.js + sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml + + + extra_css_urls: + - url: https://simonwillison.net/static/css/all.bf8cd891642c.css + sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI + extra_js_urls: + - url: https://code.jquery.com/jquery-3.2.1.slim.min.js + sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= + + +.. tab:: datasette.json + + .. code-block:: json + + { + "extra_css_urls": [ + { + "url": "https://simonwillison.net/static/css/all.bf8cd891642c.css", + "sri": "sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI" + } + ], + "extra_js_urls": [ + { + "url": "https://code.jquery.com/jquery-3.2.1.slim.min.js", + "sri": "sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g=" + } + ] + } +.. [[[end]]] + +This will produce: + +.. code-block:: html + + + + +Modern browsers will only execute the stylesheet or JavaScript if the SRI hash +matches the content served. You can generate hashes using `www.srihash.org `_ + +Items in ``"extra_js_urls"`` can specify ``"module": true`` if they reference JavaScript that uses `JavaScript modules `__. This configuration: + +.. [[[cog + config_example(cog, """ + extra_js_urls: + - url: https://example.datasette.io/module.js + module: true + """) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml + + + extra_js_urls: + - url: https://example.datasette.io/module.js + module: true + + +.. tab:: datasette.json + + .. code-block:: json + + { + "extra_js_urls": [ + { + "url": "https://example.datasette.io/module.js", + "module": true + } + ] + } +.. [[[end]]] + +Will produce this HTML: + +.. code-block:: html + + + + + diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index d8e4ac96..534d8b33 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -5,159 +5,6 @@ Custom pages and templates Datasette provides a number of ways of customizing the way data is displayed. -.. _customization_css_and_javascript: - -Custom CSS and JavaScript -------------------------- - -When you launch Datasette, you can specify a custom configuration file like this:: - - datasette mydb.db --config datasette.yaml - -Your ``datasette.yaml`` file can include links that look like this: - -.. [[[cog - from metadata_doc import config_example - config_example(cog, """ - extra_css_urls: - - https://simonwillison.net/static/css/all.bf8cd891642c.css - extra_js_urls: - - https://code.jquery.com/jquery-3.2.1.slim.min.js - """) -.. ]]] - -.. tab:: datasette.yaml - - .. code-block:: yaml - - - extra_css_urls: - - https://simonwillison.net/static/css/all.bf8cd891642c.css - extra_js_urls: - - https://code.jquery.com/jquery-3.2.1.slim.min.js - - -.. tab:: datasette.json - - .. code-block:: json - - { - "extra_css_urls": [ - "https://simonwillison.net/static/css/all.bf8cd891642c.css" - ], - "extra_js_urls": [ - "https://code.jquery.com/jquery-3.2.1.slim.min.js" - ] - } -.. [[[end]]] - -The extra CSS and JavaScript files will be linked in the ```` of every page: - -.. code-block:: html - - - - -You can also specify a SRI (subresource integrity hash) for these assets: - -.. [[[cog - config_example(cog, """ - extra_css_urls: - - url: https://simonwillison.net/static/css/all.bf8cd891642c.css - sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI - extra_js_urls: - - url: https://code.jquery.com/jquery-3.2.1.slim.min.js - sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= - """) -.. ]]] - -.. tab:: datasette.yaml - - .. code-block:: yaml - - - extra_css_urls: - - url: https://simonwillison.net/static/css/all.bf8cd891642c.css - sri: sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI - extra_js_urls: - - url: https://code.jquery.com/jquery-3.2.1.slim.min.js - sri: sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g= - - -.. tab:: datasette.json - - .. code-block:: json - - { - "extra_css_urls": [ - { - "url": "https://simonwillison.net/static/css/all.bf8cd891642c.css", - "sri": "sha384-9qIZekWUyjCyDIf2YK1FRoKiPJq4PHt6tp/ulnuuyRBvazd0hG7pWbE99zvwSznI" - } - ], - "extra_js_urls": [ - { - "url": "https://code.jquery.com/jquery-3.2.1.slim.min.js", - "sri": "sha256-k2WSCIexGzOj3Euiig+TlR8gA0EmPjuc79OEeY5L45g=" - } - ] - } -.. [[[end]]] - -This will produce: - -.. code-block:: html - - - - -Modern browsers will only execute the stylesheet or JavaScript if the SRI hash -matches the content served. You can generate hashes using `www.srihash.org `_ - -Items in ``"extra_js_urls"`` can specify ``"module": true`` if they reference JavaScript that uses `JavaScript modules `__. This configuration: - -.. [[[cog - config_example(cog, """ - extra_js_urls: - - url: https://example.datasette.io/module.js - module: true - """) -.. ]]] - -.. tab:: datasette.yaml - - .. code-block:: yaml - - - extra_js_urls: - - url: https://example.datasette.io/module.js - module: true - - -.. tab:: datasette.json - - .. code-block:: json - - { - "extra_js_urls": [ - { - "url": "https://example.datasette.io/module.js", - "module": true - } - ] - } -.. [[[end]]] - -Will produce this HTML: - -.. code-block:: html - - - CSS classes on the ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -258,9 +105,10 @@ The following URLs will now serve the content from those CSS and JS files:: http://localhost:8001/assets/styles.css http://localhost:8001/assets/app.js -You can reference those files from ``datasette.yaml`` like so: +You can reference those files from ``datasette.yaml`` like this, see :ref:`custom CSS and JavaScript ` for more details: .. [[[cog + from metadata_doc import config_example config_example(cog, """ extra_css_urls: - /assets/styles.css diff --git a/docs/metadata_doc.py b/docs/metadata_doc.py index a8f13414..ad85bf52 100644 --- a/docs/metadata_doc.py +++ b/docs/metadata_doc.py @@ -24,17 +24,19 @@ def metadata_example(cog, data=None, yaml=None): cog.out("\n") -def config_example(cog, input): +def config_example( + cog, input, yaml_title="datasette.yaml", json_title="datasette.json" +): if type(input) is str: data = YAML().load(input) output_yaml = input else: data = input output_yaml = safe_dump(input, sort_keys=False) - cog.out("\n.. tab:: datasette.yaml\n\n") + cog.out("\n.. tab:: {}\n\n".format(yaml_title)) cog.out(" .. code-block:: yaml\n\n") cog.out(textwrap.indent(output_yaml, " ")) - cog.out("\n\n.. tab:: datasette.json\n\n") + cog.out("\n\n.. tab:: {}\n\n".format(json_title)) cog.out(" .. code-block:: json\n\n") cog.out(textwrap.indent(json.dumps(data, indent=2), " ")) cog.out("\n") diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 1a88cd31..d9d135e5 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -270,7 +270,7 @@ you have one: Note that ``your-plugin`` here should be the hyphenated plugin name - the name that is displayed in the list on the ``/-/plugins`` debug page. -If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details. +If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`configuration_reference_css_js` for more details. .. code-block:: python diff --git a/docs/plugins.rst b/docs/plugins.rst index 1a72af95..03ddf8f0 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -328,7 +328,7 @@ To write that to a ``requirements.txt`` file, run this:: Plugin configuration -------------------- -Plugins can have their own configuration, embedded in a :ref:`configuration` file. Configuration options for plugins live within a ``"plugins"`` key in that file, which can be included at the root, database or table level. +Plugins can have their own configuration, embedded in a :ref:`configuration file `. Configuration options for plugins live within a ``"plugins"`` key in that file, which can be included at the root, database or table level. Here is an example of some plugin configuration for a specific table: From 4da581d09bbed2377682630c147e05d78c48f7e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 14:40:49 -0800 Subject: [PATCH 139/767] Link to config reference --- docs/settings.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/settings.rst b/docs/settings.rst index 1d4baf90..d1553703 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -15,6 +15,8 @@ You can set multiple settings at once like this:: --setting sql_time_limit_ms 3500 \ --setting max_returned_rows 2000 +Settings can also be specified :ref:`in the database.yaml configuration file `. + .. _config_dir: Configuration directory mode From d4bc2b2dfc728017c8f669c1714f20b89655557c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 14:44:16 -0800 Subject: [PATCH 140/767] Remove fail_if_plugins_in_metadata, part of #2248 --- datasette/app.py | 8 ++------ datasette/cli.py | 3 +-- datasette/utils/__init__.py | 15 --------------- tests/test_plugins.py | 8 -------- 4 files changed, 3 insertions(+), 31 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 530f79bc..0143223a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -77,7 +77,6 @@ from .utils import ( parse_metadata, resolve_env_secrets, resolve_routes, - fail_if_plugins_in_metadata, tilde_decode, to_css_class, urlsafe_components, @@ -336,16 +335,13 @@ class Datasette: ] if config_dir and metadata_files and not metadata: with metadata_files[0].open() as fp: - metadata = fail_if_plugins_in_metadata( - parse_metadata(fp.read()), metadata_files[0].name - ) + metadata = parse_metadata(fp.read()) if config_dir and config_files and not config: with config_files[0].open() as fp: config = parse_metadata(fp.read()) - self._metadata_local = fail_if_plugins_in_metadata(metadata or {}) - + self._metadata_local = metadata or {} self.sqlite_extensions = [] for extension in sqlite_extensions or []: # Resolve spatialite, if requested diff --git a/datasette/cli.py b/datasette/cli.py index 91f38f69..1a5a8af3 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -33,7 +33,6 @@ from .utils import ( initial_path_for_datasette, pairs_to_nested_config, temporary_docker_directory, - fail_if_plugins_in_metadata, value_as_boolean, SpatialiteNotFound, StaticMount, @@ -543,7 +542,7 @@ def serve( metadata_data = None if metadata: - metadata_data = fail_if_plugins_in_metadata(parse_metadata(metadata.read())) + metadata_data = parse_metadata(metadata.read()) config_data = None if config: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 196e1682..75f1c2f4 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1272,21 +1272,6 @@ def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> return result -def fail_if_plugins_in_metadata(metadata: dict, filename=None): - """If plugin config is inside metadata, raise an Exception""" - if metadata is not None and metadata.get("plugins") is not None: - suggested_extension = ( - ".yaml" - if filename is not None - and (filename.endswith(".yaml") or filename.endswith(".yml")) - else ".json" - ) - raise Exception( - f'Datasette no longer accepts plugin configuration in --metadata. Move your "plugins" configuration blocks to a separate file - we suggest calling that datasette.{suggested_extension} - and start Datasette with datasette -c datasette.{suggested_extension}. See https://docs.datasette.io/en/latest/configuration.html for more details.' - ) - return metadata - - def make_slot_function(name, datasette, request, **kwargs): from datasette.plugins import pm diff --git a/tests/test_plugins.py b/tests/test_plugins.py index dad4f2ca..f26e3652 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -883,14 +883,6 @@ def test_hook_forbidden(restore_working_directory): ) -def test_plugin_config_in_metadata(): - with pytest.raises( - Exception, - match="Datasette no longer accepts plugin configuration in --metadata", - ): - Datasette(memory=True, metadata={"plugins": {}}) - - @pytest.mark.asyncio async def test_hook_handle_exception(ds_client): await ds_client.get("/trigger-error?x=123") From be4f02335fb35d40a763d07b2d4e880b90083e53 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 15:33:33 -0800 Subject: [PATCH 141/767] Treat plugins in metadata as if they were in config, closes #2248 --- datasette/app.py | 6 +++++ datasette/utils/__init__.py | 40 +++++++++++++++++++++++++++++ tests/test_plugins.py | 51 +++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 0143223a..634283ff 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -74,6 +74,7 @@ from .utils import ( find_spatialite, format_bytes, module_from_path, + move_plugins, parse_metadata, resolve_env_secrets, resolve_routes, @@ -341,6 +342,11 @@ class Datasette: with config_files[0].open() as fp: config = parse_metadata(fp.read()) + # Move any "plugins" settings from metadata to config - updates them in place + metadata = metadata or {} + config = config or {} + move_plugins(metadata, config) + self._metadata_local = metadata or {} self.sqlite_extensions = [] for extension in sqlite_extensions or []: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 75f1c2f4..cc175b01 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1287,3 +1287,43 @@ def make_slot_function(name, datasette, request, **kwargs): return markupsafe.Markup("".join(html_bits)) return inner + + +def move_plugins(source, destination): + """ + Move 'plugins' keys from source to destination dictionary. Creates hierarchy in destination if needed. + After moving, recursively remove any keys in the source that are left empty. + """ + + def recursive_move(src, dest, path=None): + if path is None: + path = [] + for key, value in list(src.items()): + new_path = path + [key] + if key == "plugins": + # Navigate and create the hierarchy in destination if needed + d = dest + for step in path: + d = d.setdefault(step, {}) + # Move the plugins + d[key] = value + # Remove the plugins from source + src.pop(key, None) + elif isinstance(value, dict): + recursive_move(value, dest, new_path) + # After moving, check if the current dictionary is empty and remove it if so + if not value: + src.pop(key, None) + + def prune_empty_dicts(d): + """ + Recursively prune all empty dictionaries from a given dictionary. + """ + for key, value in list(d.items()): + if isinstance(value, dict): + prune_empty_dicts(value) + if value == {}: + d.pop(key, None) + + recursive_move(source, destination) + prune_empty_dicts(source) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index f26e3652..a53fc118 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1458,3 +1458,54 @@ async def test_hook_register_events(): datasette = Datasette(memory=True) await datasette.invoke_startup() assert any(k.__name__ == "OneEvent" for k in datasette.event_classes) + + +@pytest.mark.parametrize( + "metadata,config,expected_metadata,expected_config", + ( + ( + # Instance level + {"plugins": {"datasette-foo": "bar"}}, + {}, + {}, + {"plugins": {"datasette-foo": "bar"}}, + ), + ( + # Database level + {"databases": {"foo": {"plugins": {"datasette-foo": "bar"}}}}, + {}, + {}, + {"databases": {"foo": {"plugins": {"datasette-foo": "bar"}}}}, + ), + ( + # Table level + { + "databases": { + "foo": {"tables": {"bar": {"plugins": {"datasette-foo": "bar"}}}} + } + }, + {}, + {}, + { + "databases": { + "foo": {"tables": {"bar": {"plugins": {"datasette-foo": "bar"}}}} + } + }, + ), + ( + # Keep other keys + {"plugins": {"datasette-foo": "bar"}, "other": "key"}, + {"original_config": "original"}, + {"other": "key"}, + {"original_config": "original", "plugins": {"datasette-foo": "bar"}}, + ), + ), +) +def test_metadata_plugin_config_treated_as_config( + metadata, config, expected_metadata, expected_config +): + ds = Datasette(metadata=metadata, config=config) + actual_metadata = ds.metadata() + assert "plugins" not in actual_metadata + assert actual_metadata == expected_metadata + assert ds.config == expected_config From 6ccef35cc92cc2357c2b2a9aa003b7334b2459eb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 15:42:45 -0800 Subject: [PATCH 142/767] More links between events documentation --- docs/events.rst | 2 +- docs/plugin_hooks.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/events.rst b/docs/events.rst index f150ac02..b86c8025 100644 --- a/docs/events.rst +++ b/docs/events.rst @@ -7,7 +7,7 @@ Datasette includes a mechanism for tracking events that occur while the software The core Datasette application triggers events when certain things happen. This page describes those events. -Plugins can listen for events using the :ref:`plugin_hook_track_event` plugin hook, which will be called with instances of the following classes (or additional classes registered by other plugins): +Plugins can listen for events using the :ref:`plugin_hook_track_event` plugin hook, which will be called with instances of the following classes - or additional classes :ref:`registered by other plugins `. .. automodule:: datasette.events :members: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index d9d135e5..16f5cebb 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1769,7 +1769,7 @@ Datasette includes an internal mechanism for tracking analytical events. This ca Plugins can register to receive events using the ``track_event`` plugin hook. -They can also define their own events for other plugins to receive using the ``register_events`` plugin hook, combined with calls to the ``datasette.track_event(...)`` internal method. +They can also define their own events for other plugins to receive using the :ref:`register_events() plugin hook `, combined with calls to the :ref:`datasette.track_event() internal method `. .. _plugin_hook_track_event: @@ -1826,7 +1826,7 @@ register_events(datasette) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. -This hook should return a list of ``Event`` subclasses that represent custom events that the plugin might send to the ``datasette.track_event()`` method. +This hook should return a list of ``Event`` subclasses that represent custom events that the plugin might send to the :ref:`datasette.track_event() ` method. This example registers event subclasses for ``ban-user`` and ``unban-user`` events: From 4ea109ac4dd17392c85ca7d5934009f9a9488a9d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 1 Feb 2024 15:47:41 -0800 Subject: [PATCH 143/767] Two spaces is aesthetically more pleasing here --- docs/settings.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/settings.rst b/docs/settings.rst index d1553703..c4b4ba82 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -11,9 +11,9 @@ Datasette supports a number of settings. These can be set using the ``--setting You can set multiple settings at once like this:: datasette mydatabase.db \ - --setting default_page_size 50 \ - --setting sql_time_limit_ms 3500 \ - --setting max_returned_rows 2000 + --setting default_page_size 50 \ + --setting sql_time_limit_ms 3500 \ + --setting max_returned_rows 2000 Settings can also be specified :ref:`in the database.yaml configuration file `. @@ -25,10 +25,10 @@ Configuration directory mode Normally you configure Datasette using command-line options. For a Datasette instance with custom templates, custom plugins, a static directory and several databases this can get quite verbose:: datasette one.db two.db \ - --metadata=metadata.json \ - --template-dir=templates/ \ - --plugins-dir=plugins \ - --static css:css + --metadata=metadata.json \ + --template-dir=templates/ \ + --plugins-dir=plugins \ + --static css:css As an alternative to this, you can run Datasette in *configuration directory* mode. Create a directory with the following structure:: From 5ea7098e4da5fe8576f6452dbfac86e6aedba397 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 4 Feb 2024 10:15:21 -0800 Subject: [PATCH 144/767] Fixed an unnecessary f-string --- demos/plugins/example_js_manager_plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/plugins/example_js_manager_plugins.py b/demos/plugins/example_js_manager_plugins.py index 7db45464..2705f2c5 100644 --- a/demos/plugins/example_js_manager_plugins.py +++ b/demos/plugins/example_js_manager_plugins.py @@ -16,6 +16,6 @@ def extra_js_urls(view_name): if view_name in PERMITTED_VIEWS: return [ { - "url": f"/static/table-example-plugins.js", + "url": "/static/table-example-plugins.js", } ] From 7219a56d1e8b5d076037aeeec2583ad4fc3cacb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 10:34:10 -0800 Subject: [PATCH 145/767] 3 space indent, not 2 --- docs/configuration.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index a835ace9..79e2a1ca 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -145,8 +145,8 @@ Settings """ # inside datasette.yaml settings: - default_allow_sql: off - default_page_size: 50 + default_allow_sql: off + default_page_size: 50 """).strip() ) .. ]]] @@ -157,8 +157,8 @@ Settings # inside datasette.yaml settings: - default_allow_sql: off - default_page_size: 50 + default_allow_sql: off + default_page_size: 50 .. tab:: datasette.json From 503545b20363ac15d3664bec7e6c4522ff271668 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 11:47:17 -0800 Subject: [PATCH 146/767] JavaScript plugins documentation, closes #2250 --- docs/index.rst | 1 + docs/javascript_plugins.rst | 159 ++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 docs/javascript_plugins.rst diff --git a/docs/index.rst b/docs/index.rst index ce1ed2eb..e3036618 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,6 +60,7 @@ Contents custom_templates plugins writing_plugins + javascript_plugins plugin_hooks testing_plugins internals diff --git a/docs/javascript_plugins.rst b/docs/javascript_plugins.rst new file mode 100644 index 00000000..e7ee6817 --- /dev/null +++ b/docs/javascript_plugins.rst @@ -0,0 +1,159 @@ +.. _javascript_plugins: + +JavaScript plugins +================== + +Datasette can run custom JavaScript in several different ways: + +- Datasette plugins written in Python can use the :ref:`extra_js_urls() ` or :ref:`extra_body_script() ` plugin hooks to inject JavaScript into a page +- Datasette instances with :ref:`custom templates ` can include additional JavaScript in those templates +- The ``extra_js_urls`` key in ``datasette.yaml`` :ref:`can be used to include extra JavaScript ` + +There are no limitations on what this JavaScript can do. It is executed directly by the browser, so it can manipulate the DOM, fetch additional data and do anything else that JavaScript is capable of. + +.. warning:: + Custom JavaScript has security implications, especially for authenticated Datasette instances where the JavaScript might run in the context of the authenticated user. It's important to carefully review any JavaScript you run in your Datasette instance. + +.. _javascript_datasette_init: + +The datasette_init event +------------------------ + +Datasette emits a custom event called ``datasette_init`` when the page is loaded. This event is dispatched on the ``document`` object, and includes a ``detail`` object with a reference to the :ref:`datasetteManager ` object. + +Your JavaScript code can listen out for this event using ``document.addEventListener()`` like this: + +.. code-block:: javascript + + document.addEventListener("datasette_init", function (evt) { + const manager = evt.detail; + console.log("Datasette version:", manager.VERSION); + }); + +.. _javascript_datasette_manager: + +datasetteManager +---------------- + +The ``datasetteManager`` object + +``VERSION`` - string + The version of Datasette + +``plugins`` - ``Map()`` + A Map of currently loaded plugin names to plugin implementations + +``registerPlugin(name, implementation)`` + Call this to register a plugin, passing its name and implementation + +``selectors`` - object + An object providing named aliases to useful CSS selectors, :ref:`listed below ` + +.. _javascript_plugin_objects: + +JavaScript plugin objects +------------------------- + +JavaScript plugins are blocks of code that can be registered with Datasette using the ``registerPlugin()`` method on the :ref:`datasetteManager ` object. + +The ``implementation`` object passed to this method should include a ``version`` key defining the plugin version, and one or more of the following named functions providing the implementation of the plugin: + +.. _javascript_plugins_makeAboveTablePanelConfigs: + +makeAboveTablePanelConfigs() +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This method should return a JavaScript array of objects defining additional panels to be added to the top of the table page. Each object should have the following: + +``id`` - string + A unique string ID for the panel, for example ``map-panel`` +``label`` - string + A human-readable label for the panel +``render(node)`` - function + A function that will be called with a DOM node to render the panel into + +This example shows how a plugin might define a single panel: + +.. code-block:: javascript + + document.addEventListener('datasette_init', function(ev) { + ev.detail.registerPlugin('panel-plugin', { + version: 0.1, + makeAboveTablePanelConfigs: () => { + return [ + { + id: 'first-panel', + label: 'First panel', + render: node => { + node.innerHTML = '

My custom panel

This is a custom panel that I added using a JavaScript plugin

'; + } + } + ] + } + }); + }); + +When a page with a table loads, all registered plugins that implement ``makeAboveTablePanelConfigs()`` will be called and panels they return will be added to the top of the table page. + +.. _javascript_plugins_makeColumnActions: + +makeColumnActions(columnDetails) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This method, if present, will be called when Datasette is rendering the cog action menu icons that appear at the top of the table view. By default these include options like "Sort ascending/descending" and "Facet by this", but plugins can return additional actions to be included in this menu. + +The method will be called with a ``columnDetails`` object with the following keys: + +``columnName`` - string + The name of the column +``columnNotNull`` - boolean + True if the column is defined as NOT NULL +``columnType`` - string + The SQLite data type of the column +``isPk`` - boolean + True if the column is part of the primary key + +It should return a JavaScript array of objects each with a ``label`` and ``onClick`` property: + +``label`` - string + The human-readable label for the action +``onClick(evt)`` - function + A function that will be called when the action is clicked + +The ``evt`` object passed to the ``onClick`` is the standard browser event object that triggered the click. + +This example plugin adds two menu items - one to copy the column name to the clipboard and another that displays the column metadata in an ``alert()`` window: + +.. code-block:: javascript + + document.addEventListener('datasette_init', function(ev) { + ev.detail.registerPlugin('column-name-plugin', { + version: 0.1, + makeColumnActions: (columnDetails) => { + return [ + { + label: 'Copy column to clipboard', + onClick: async (evt) => { + await navigator.clipboard.writeText(columnDetails.columnName) + } + }, + { + label: 'Alert column metadata', + onClick: () => alert(JSON.stringify(columnDetails, null, 2)) + } + ]; + } + }); + }); + +.. _javascript_datasette_manager_selectors: + +Selectors +--------- + +These are available on the ``selectors`` property of the :ref:`javascript_datasette_manager` object. + +.. literalinclude:: ../datasette/static/datasette-manager.js + :language: javascript + :start-at: const DOM_SELECTORS = { + :end-at: }; From efc73575548b0bbaca4bdc8de40fc6939bb88428 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 13:01:03 -0800 Subject: [PATCH 147/767] Remove Using YAML for metadata section No longer necessary now we show YAML and JSON examples everywhere. --- docs/changelog.rst | 2 +- docs/metadata.rst | 29 +---------------------------- 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 04ce9583..f4b928e3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1190,7 +1190,7 @@ Also in this release: 0.40 (2020-04-21) ----------------- -* Datasette :ref:`metadata` can now be provided as a YAML file as an optional alternative to JSON. See :ref:`metadata_yaml`. (:issue:`713`) +* Datasette :ref:`metadata` can now be provided as a YAML file as an optional alternative to JSON. (:issue:`713`) * Removed support for ``datasette publish now``, which used the the now-retired Zeit Now v1 hosting platform. A new plugin, `datasette-publish-now `__, can be installed to publish data to Zeit (`now Vercel `__) Now v2. (:issue:`710`) * Fixed a bug where the ``extra_template_vars(request, view_name)`` plugin hook was not receiving the correct ``view_name``. (:issue:`716`) * Variables added to the template context by the ``extra_template_vars()`` plugin hook are now shown in the ``?_context=1`` debugging mode (see :ref:`setting_template_debug`). (:issue:`693`) diff --git a/docs/metadata.rst b/docs/metadata.rst index b4dc90f9..f3ca68ac 100644 --- a/docs/metadata.rst +++ b/docs/metadata.rst @@ -53,7 +53,7 @@ Your ``metadata.yaml`` file can look something like this: .. [[[end]]] -Choosing YAML over JSON adds support for multi-line strings and comments, see :ref:`metadata_yaml`. +Choosing YAML over JSON adds support for multi-line strings and comments. The above metadata will be displayed on the index page of your Datasette-powered site. The source and license information will also be included in the footer of @@ -664,33 +664,6 @@ SpatiaLite tables are automatically hidden) using ``"hidden": true``: } .. [[[end]]] -.. _metadata_yaml: - -Using YAML for metadata ------------------------ - -Datasette accepts YAML as an alternative to JSON for your metadata configuration file. -YAML is particularly useful for including multiline HTML and SQL strings, plus inline comments. - -Here's an example of a ``metadata.yml`` file, re-using an example from :ref:`canned_queries`. - -.. code-block:: yaml - - title: Demonstrating Metadata from YAML - description_html: |- -

This description includes a long HTML string

-
    -
  • YAML is better for embedding HTML strings than JSON!
    • -
- license: ODbL - license_url: https://opendatacommons.org/licenses/odbl/ - databases: - fixtures: - tables: - no_primary_key: - hidden: true - - .. _metadata_reference: Metadata reference From 85a1dfe6e07fcdd7ec8f83cb5b3a8f023659d064 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Feb 2024 13:43:50 -0800 Subject: [PATCH 148/767] Configuration via the command-line section Closes #2252 Closes #2156 --- docs/configuration.rst | 78 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 79e2a1ca..425024da 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -5,7 +5,7 @@ Configuration Datasette offers several ways to configure your Datasette instances: server settings, plugin configuration, authentication, and more. -Most configuration can be handled using a ``datasette.yaml`` configuration file, passed to datasette using the ``--config``/ ``-c`` flag: +Most configuration can be handled using a ``datasette.yaml`` configuration file, passed to datasette using the ``-c/--config`` flag: .. code-block:: bash @@ -13,12 +13,86 @@ Most configuration can be handled using a ``datasette.yaml`` configuration file, This file can also use JSON, as ``datasette.json``. YAML is recommended over JSON due to its support for comments and multi-line strings. +.. _configuration_cli: + +Configuration via the command-line +---------------------------------- + +The recommended way to configure Datasette is using a ``datasette.yaml`` file passed to ``-c/--config``. You can also pass individual settings to Datasette using the ``-s/--setting`` option, which can be used multiple times: + +.. code-block:: bash + + datasette mydatabase.db \ + --setting settings.default_page_size 50 \ + --setting settings.sql_time_limit_ms 3500 + +This option takes dotted-notation for the first argument and a value for the second argument. This means you can use it to set any configuration value that would be valid in a ``datasette.yaml`` file. + +It also works for plugin configuration, for example for `datasette-cluster-map `_: + +.. code-block:: bash + + datasette mydatabase.db \ + --setting plugins.datasette-cluster-map.latitude_column xlat \ + --setting plugins.datasette-cluster-map.longitude_column xlon + +If the value you provide is a valid JSON object or list it will be treated as nested data, allowing you to configure plugins that accept lists such as `datasette-proxy-url `_: + +.. code-block:: bash + + datasette mydatabase.db \ + -s plugins.datasette-proxy-url.paths '[{"path": "/proxy", "backend": "http://example.com/"}]' + +This is equivalent to a ``datasette.yaml`` file containing the following: + +.. [[[cog + from metadata_doc import config_example + import textwrap + config_example(cog, textwrap.dedent( + """ + plugins: + datasette-proxy-url: + paths: + - path: /proxy + backend: http://example.com/ + """).strip() + ) +.. ]]] + +.. tab:: datasette.yaml + + .. code-block:: yaml + + plugins: + datasette-proxy-url: + paths: + - path: /proxy + backend: http://example.com/ + +.. tab:: datasette.json + + .. code-block:: json + + { + "plugins": { + "datasette-proxy-url": { + "paths": [ + { + "path": "/proxy", + "backend": "http://example.com/" + } + ] + } + } + } +.. [[[end]]] + .. _configuration_reference: ``datasette.yaml`` reference ---------------------------- -This example shows many of the valid configuration options that can exist inside ``datasette.yaml``. +The following example shows some of the valid configuration options that can exist inside ``datasette.yaml``. .. [[[cog from metadata_doc import config_example From 1e901aa690211db36f02cc1b25246d0f56cd8720 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 12:33:46 -0800 Subject: [PATCH 149/767] /-/config page, closes #2254 --- datasette/app.py | 14 ++++++----- datasette/utils/__init__.py | 28 +++++++++++++++++++++ datasette/views/special.py | 4 +-- docs/introspection.rst | 11 +++++++- tests/test_api.py | 50 ++++++++++++++++++++++++++----------- 5 files changed, 84 insertions(+), 23 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 634283ff..2e20d402 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -81,6 +81,7 @@ from .utils import ( tilde_decode, to_css_class, urlsafe_components, + redact_keys, row_sql_params_pks, ) from .utils.asgi import ( @@ -1374,6 +1375,11 @@ class Datasette: output.append(script) return output + def _config(self): + return redact_keys( + self.config, ("secret", "key", "password", "token", "hash", "dsn") + ) + def _routes(self): routes = [] @@ -1433,12 +1439,8 @@ class Datasette: r"/-/settings(\.(?Pjson))?$", ) add_route( - permanent_redirect("/-/settings.json"), - r"/-/config.json", - ) - add_route( - permanent_redirect("/-/settings"), - r"/-/config", + JsonDataView.as_view(self, "config.json", lambda: self._config()), + r"/-/config(\.(?Pjson))?$", ) add_route( JsonDataView.as_view(self, "threads.json", self._threads), diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index cc175b01..4c940645 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -17,6 +17,7 @@ import time import types import secrets import shutil +from typing import Iterable import urllib import yaml from .shutil_backport import copytree @@ -1327,3 +1328,30 @@ def move_plugins(source, destination): recursive_move(source, destination) prune_empty_dicts(source) + + +def redact_keys(original: dict, key_patterns: Iterable) -> dict: + """ + Recursively redact sensitive keys in a dictionary based on given patterns + + :param original: The original dictionary + :param key_patterns: A list of substring patterns to redact + :return: A copy of the original dictionary with sensitive values redacted + """ + + def redact(data): + if isinstance(data, dict): + return { + k: ( + redact(v) + if not any(pattern in k for pattern in key_patterns) + else "***" + ) + for k, v in data.items() + } + elif isinstance(data, list): + return [redact(item) for item in data] + else: + return data + + return redact(original) diff --git a/datasette/views/special.py b/datasette/views/special.py index 4088a1f9..296652d0 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -42,7 +42,7 @@ class JsonDataView(BaseView): if self.ds.cors: add_cors_headers(headers) return Response( - json.dumps(data), + json.dumps(data, default=repr), content_type="application/json; charset=utf-8", headers=headers, ) @@ -53,7 +53,7 @@ class JsonDataView(BaseView): request=request, context={ "filename": self.filename, - "data_json": json.dumps(data, indent=4), + "data_json": json.dumps(data, indent=4, default=repr), }, ) diff --git a/docs/introspection.rst b/docs/introspection.rst index e08ca911..b62197ea 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -87,7 +87,7 @@ Shows a list of currently installed plugins and their versions. `Plugins example Add ``?all=1`` to include details of the default plugins baked into Datasette. -.. _JsonDataView_config: +.. _JsonDataView_settings: /-/settings ----------- @@ -105,6 +105,15 @@ Shows the :ref:`settings` for this instance of Datasette. `Settings example ` for this instance of Datasette. This is generally the contents of the :ref:`datasette.yaml or datasette.json ` file, which can include plugin configuration as well. + +Any keys that include the one of the following substrings in their names will be returned as redacted ``***`` output, to help avoid accidentally leaking private configuration information: ``secret``, ``key``, ``password``, ``token``, ``hash``, ``dsn``. + .. _JsonDataView_databases: /-/databases diff --git a/tests/test_api.py b/tests/test_api.py index 177dc95c..0a1f3725 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -846,20 +846,6 @@ async def test_settings_json(ds_client): } -@pytest.mark.asyncio -@pytest.mark.parametrize( - "path,expected_redirect", - ( - ("/-/config.json", "/-/settings.json"), - ("/-/config", "/-/settings"), - ), -) -async def test_config_redirects_to_settings(ds_client, path, expected_redirect): - response = await ds_client.get(path) - assert response.status_code == 301 - assert response.headers["Location"] == expected_redirect - - test_json_columns_default_expected = [ {"intval": 1, "strval": "s", "floatval": 0.5, "jsonval": '{"foo": "bar"}'} ] @@ -1039,3 +1025,39 @@ async def test_tilde_encoded_database_names(db_name): # And the JSON for that database response2 = await ds.client.get(path + ".json") assert response2.status_code == 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "config,expected", + ( + ({}, {}), + ({"plugins": {"datasette-foo": "bar"}}, {"plugins": {"datasette-foo": "bar"}}), + # Test redaction + ( + { + "plugins": { + "datasette-auth": {"secret_key": "key"}, + "datasette-foo": "bar", + "datasette-auth2": {"password": "password"}, + "datasette-sentry": { + "dsn": "sentry:///foo", + }, + } + }, + { + "plugins": { + "datasette-auth": {"secret_key": "***"}, + "datasette-foo": "bar", + "datasette-auth2": {"password": "***"}, + "datasette-sentry": {"dsn": "***"}, + } + }, + ), + ), +) +async def test_config_json(config, expected): + "/-/config.json should return redacted configuration" + ds = Datasette(config=config) + response = await ds.client.get("/-/config.json") + assert response.json() == expected From 5a63ecc5577d070f28bf5daa34aaaf3dadfd2e4d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 15:03:19 -0800 Subject: [PATCH 150/767] Rename metadata= to table_config= in facet code, refs #2247 --- datasette/facets.py | 41 ++++++++++++++++++++-------------------- datasette/views/table.py | 2 +- tests/test_facets.py | 4 ++-- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index b23615fe..f1cfc68f 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -11,8 +11,8 @@ from datasette.utils import ( ) -def load_facet_configs(request, table_metadata): - # Given a request and the metadata configuration for a table, return +def load_facet_configs(request, table_config): + # Given a request and the configuration for a table, return # a dictionary of selected facets, their lists of configs and for each # config whether it came from the request or the metadata. # @@ -20,21 +20,21 @@ def load_facet_configs(request, table_metadata): # {"source": "metadata", "config": config1}, # {"source": "request", "config": config2}]} facet_configs = {} - table_metadata = table_metadata or {} - metadata_facets = table_metadata.get("facets", []) - for metadata_config in metadata_facets: - if isinstance(metadata_config, str): + table_config = table_config or {} + table_facet_configs = table_config.get("facets", []) + for facet_config in table_facet_configs: + if isinstance(facet_config, str): type = "column" - metadata_config = {"simple": metadata_config} + facet_config = {"simple": facet_config} else: assert ( - len(metadata_config.values()) == 1 + len(facet_config.values()) == 1 ), "Metadata config dicts should be {type: config}" - type, metadata_config = list(metadata_config.items())[0] - if isinstance(metadata_config, str): - metadata_config = {"simple": metadata_config} + type, facet_config = list(facet_config.items())[0] + if isinstance(facet_config, str): + facet_config = {"simple": facet_config} facet_configs.setdefault(type, []).append( - {"source": "metadata", "config": metadata_config} + {"source": "metadata", "config": facet_config} ) qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True) for key, values in qs_pairs.items(): @@ -45,13 +45,12 @@ def load_facet_configs(request, table_metadata): elif key.startswith("_facet_"): type = key[len("_facet_") :] for value in values: - # The value is the config - either JSON or not - if value.startswith("{"): - config = json.loads(value) - else: - config = {"simple": value} + # The value is the facet_config - either JSON or not + facet_config = ( + json.loads(value) if value.startswith("{") else {"simple": value} + ) facet_configs.setdefault(type, []).append( - {"source": "request", "config": config} + {"source": "request", "config": facet_config} ) return facet_configs @@ -75,7 +74,7 @@ class Facet: sql=None, table=None, params=None, - metadata=None, + table_config=None, row_count=None, ): assert table or sql, "Must provide either table= or sql=" @@ -86,12 +85,12 @@ class Facet: self.table = table self.sql = sql or f"select * from [{table}]" self.params = params or [] - self.metadata = metadata + self.table_config = table_config # row_count can be None, in which case we calculate it ourselves: self.row_count = row_count def get_configs(self): - configs = load_facet_configs(self.request, self.metadata) + configs = load_facet_configs(self.request, self.table_config) return configs.get(self.type) or [] def get_querystring_pairs(self): diff --git a/datasette/views/table.py b/datasette/views/table.py index 3b812c01..22722847 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1275,7 +1275,7 @@ async def table_view_data( sql=sql_no_order_no_limit, params=params, table=table_name, - metadata=table_metadata, + table_config=table_metadata, row_count=extra_count, ) ) diff --git a/tests/test_facets.py b/tests/test_facets.py index 85c8f85b..76344108 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -82,7 +82,7 @@ async def test_column_facet_suggest_skip_if_enabled_by_metadata(ds_client): database="fixtures", sql="select * from facetable", table="facetable", - metadata={"facets": ["_city_id"]}, + table_config={"facets": ["_city_id"]}, ) suggestions = [s["name"] for s in await facet.suggest()] assert [ @@ -278,7 +278,7 @@ async def test_column_facet_from_metadata_cannot_be_hidden(ds_client): database="fixtures", sql="select * from facetable", table="facetable", - metadata={"facets": ["_city_id"]}, + table_config={"facets": ["_city_id"]}, ) buckets, timed_out = await facet.facet_results() assert [] == timed_out From 5d21057cf1e4171bc2741d3d8d9da83aee1165b5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 15:22:03 -0800 Subject: [PATCH 151/767] /-/config example, refs #2254 --- docs/introspection.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/introspection.rst b/docs/introspection.rst index b62197ea..ff78ec78 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -110,7 +110,17 @@ Shows the :ref:`settings` for this instance of Datasette. `Settings example ` for this instance of Datasette. This is generally the contents of the :ref:`datasette.yaml or datasette.json ` file, which can include plugin configuration as well. +Shows the :ref:`configuration ` for this instance of Datasette. This is generally the contents of the :ref:`datasette.yaml or datasette.json ` file, which can include plugin configuration as well. `Config example `_: + +.. code-block:: json + + { + "settings": { + "template_debug": true, + "trace_debug": true, + "force_https_urls": true + } + } Any keys that include the one of the following substrings in their names will be returned as redacted ``***`` output, to help avoid accidentally leaking private configuration information: ``secret``, ``key``, ``password``, ``token``, ``hash``, ``dsn``. From 69c6e953231078ab18ba0807e5fe2a4e20e84093 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 17:27:20 -0800 Subject: [PATCH 152/767] Fixed a bunch of unused imports spotted with ruff --- datasette/cli.py | 1 - datasette/forbidden.py | 1 - datasette/handle_exception.py | 4 +--- datasette/permissions.py | 2 +- datasette/url_builder.py | 2 +- datasette/views/base.py | 1 - datasette/views/index.py | 3 --- ruff.toml | 1 + tests/conftest.py | 2 +- tests/test_black.py | 2 -- tests/test_cli.py | 2 -- tests/test_cli_serve_get.py | 2 +- tests/test_config_dir.py | 2 -- tests/test_internals_datasette.py | 1 - tests/test_plugins.py | 7 ++----- tests/test_table_html.py | 2 +- 16 files changed, 9 insertions(+), 26 deletions(-) create mode 100644 ruff.toml diff --git a/datasette/cli.py b/datasette/cli.py index 1a5a8af3..0c8a8541 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -15,7 +15,6 @@ import sys import textwrap import webbrowser from .app import ( - OBSOLETE_SETTINGS, Datasette, DEFAULT_SETTINGS, SETTINGS, diff --git a/datasette/forbidden.py b/datasette/forbidden.py index 156a44d4..41c48396 100644 --- a/datasette/forbidden.py +++ b/datasette/forbidden.py @@ -1,4 +1,3 @@ -from os import stat from datasette import hookimpl, Response diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index bef6b4ee..1a0ac979 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -1,14 +1,12 @@ from datasette import hookimpl, Response -from .utils import await_me_maybe, add_cors_headers +from .utils import add_cors_headers from .utils.asgi import ( Base400, - Forbidden, ) from .views.base import DatasetteError from markupsafe import Markup import pdb import traceback -from .plugins import pm try: import rich diff --git a/datasette/permissions.py b/datasette/permissions.py index 152f1721..bd42158e 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass, fields +from dataclasses import dataclass from typing import Optional diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 574bf3c1..9c6bbde0 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -1,4 +1,4 @@ -from .utils import tilde_encode, path_with_format, HASH_LENGTH, PrefixedUrlString +from .utils import tilde_encode, path_with_format, PrefixedUrlString import urllib diff --git a/datasette/views/base.py b/datasette/views/base.py index bdc1e9cf..9d7a854c 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -10,7 +10,6 @@ from markupsafe import escape import pint -from datasette import __version__ from datasette.database import QueryInterrupted from datasette.utils.asgi import Request from datasette.utils import ( diff --git a/datasette/views/index.py b/datasette/views/index.py index 595cf234..2cb18b1c 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -1,12 +1,9 @@ import json -from datasette.plugins import pm from datasette.utils import add_cors_headers, make_slot_function, CustomJSONEncoder from datasette.utils.asgi import Response from datasette.version import __version__ -from markupsafe import Markup - from .base import BaseView diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..0deb884c --- /dev/null +++ b/ruff.toml @@ -0,0 +1 @@ +line-length = 160 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 445de057..168194d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import re import subprocess import tempfile import time -from dataclasses import dataclass, field +from dataclasses import dataclass from datasette import Event, hookimpl diff --git a/tests/test_black.py b/tests/test_black.py index d09b2514..ccf51171 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1,8 +1,6 @@ import black from click.testing import CliRunner from pathlib import Path -import pytest -import sys code_root = Path(__file__).parent.parent diff --git a/tests/test_cli.py b/tests/test_cli.py index 9cc18c6e..bda17eed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,6 @@ from .fixtures import ( TestClient as _TestClient, EXPECTED_PLUGINS, ) -import asyncio from datasette.app import SETTINGS from datasette.plugins import DEFAULT_PLUGINS from datasette.cli import cli, serve @@ -19,7 +18,6 @@ import pytest import sys import textwrap from unittest import mock -import urllib def test_inspect_cli(app_client): diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index ff2429c6..1088d906 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -1,4 +1,4 @@ -from datasette.cli import cli, serve +from datasette.cli import cli from datasette.plugins import pm from click.testing import CliRunner import textwrap diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 748412c3..66114a27 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -3,11 +3,9 @@ import pathlib import pytest from datasette.app import Datasette -from datasette.cli import cli from datasette.utils.sqlite import sqlite3 from datasette.utils import StartupError from .fixtures import TestClient as _TestClient -from click.testing import CliRunner PLUGIN = """ from datasette import hookimpl diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index c30bb748..2614e02e 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -7,7 +7,6 @@ from datasette import Forbidden, Context from datasette.app import Datasette, Database from itsdangerous import BadSignature import pytest -from typing import Optional @pytest.fixture diff --git a/tests/test_plugins.py b/tests/test_plugins.py index a53fc118..c02c94cc 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,6 +1,5 @@ from bs4 import BeautifulSoup as Soup from .fixtures import ( - app_client, app_client, make_app_client, TABLES, @@ -9,14 +8,12 @@ from .fixtures import ( TestClient as _TestClient, ) # noqa from click.testing import CliRunner -from dataclasses import dataclass from datasette.app import Datasette -from datasette import cli, hookimpl, Event, Permission +from datasette import cli, hookimpl, Permission from datasette.filters import FilterArguments from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils.sqlite import sqlite3 -from datasette.utils import CustomRow, StartupError -from jinja2.environment import Template +from datasette.utils import StartupError from jinja2 import ChoiceLoader, FileSystemLoader import base64 import datetime diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 0604d34c..2a658663 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -1,4 +1,4 @@ -from datasette.app import Datasette, Database +from datasette.app import Datasette from bs4 import BeautifulSoup as Soup from .fixtures import ( # noqa app_client, From f0491038523e000b97a18d2e9a23faee62208083 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 17:33:18 -0800 Subject: [PATCH 153/767] datasette.table_metadata() is now await datasette.table_config(), refs #2247 --- datasette/app.py | 2 +- datasette/database.py | 2 +- datasette/filters.py | 2 +- datasette/views/row.py | 2 +- datasette/views/table.py | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 2e20d402..b0b8f041 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1202,7 +1202,7 @@ class Datasette: def _actor(self, request): return {"actor": request.actor} - def table_metadata(self, database, table): + async def table_config(self, database, table): """Fetch table-specific metadata.""" return ( (self.metadata("databases") or {}) diff --git a/datasette/database.py b/datasette/database.py index f2c980d7..1c1b3e1b 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -418,7 +418,7 @@ class Database: return await self.execute_fn(lambda conn: detect_fts(conn, table)) async def label_column_for_table(self, table): - explicit_label_column = self.ds.table_metadata(self.name, table).get( + explicit_label_column = (await self.ds.table_config(self.name, table)).get( "label_column" ) if explicit_label_column: diff --git a/datasette/filters.py b/datasette/filters.py index 73eea857..4d9580d8 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -50,7 +50,7 @@ def search_filters(request, database, table, datasette): extra_context = {} # Figure out which fts_table to use - table_metadata = datasette.table_metadata(database, table) + table_metadata = await datasette.table_config(database, table) db = datasette.get_database(database) fts_table = request.args.get("_fts_table") fts_table = fts_table or table_metadata.get("fts_table") diff --git a/datasette/views/row.py b/datasette/views/row.py index 7b646641..7b43b893 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -89,7 +89,7 @@ class RowView(DataView): "columns": columns, "primary_keys": resolved.pks, "primary_key_values": pk_values, - "units": self.ds.table_metadata(database, table).get("units", {}), + "units": (await self.ds.table_config(database, table)).get("units", {}), } if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): diff --git a/datasette/views/table.py b/datasette/views/table.py index 22722847..2b5b7c24 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -142,7 +142,7 @@ async def display_columns_and_rows( """Returns columns, rows for specified table - including fancy foreign key treatment""" sortable_columns = sortable_columns or set() db = datasette.databases[database_name] - table_metadata = datasette.table_metadata(database_name, table_name) + table_metadata = await datasette.table_config(database_name, table_name) column_descriptions = table_metadata.get("columns") or {} column_details = { col.name: col for col in await db.table_column_details(table_name) @@ -663,7 +663,7 @@ async def _columns_to_select(table_columns, pks, request): async def _sortable_columns_for_table(datasette, database_name, table_name, use_rowid): db = datasette.databases[database_name] - table_metadata = datasette.table_metadata(database_name, table_name) + table_metadata = await datasette.table_config(database_name, table_name) if "sortable_columns" in table_metadata: sortable_columns = set(table_metadata["sortable_columns"]) else: @@ -962,7 +962,7 @@ async def table_view_data( nocount = True nofacet = True - table_metadata = datasette.table_metadata(database_name, table_name) + table_metadata = await datasette.table_config(database_name, table_name) units = table_metadata.get("units", {}) # Arguments that start with _ and don't contain a __ are From 52a1dac5d2bac7e106c8d6ce8e0c6f1dc0141a7e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 21:00:55 -0800 Subject: [PATCH 154/767] Test proving $env works for datasette.yml, closes #2255 --- tests/test_plugins.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index c02c94cc..40d01c71 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -231,10 +231,18 @@ async def test_plugin_config(ds_client): @pytest.mark.asyncio -async def test_plugin_config_env(ds_client): - os.environ["FOO_ENV"] = "FROM_ENVIRONMENT" - assert {"foo": "FROM_ENVIRONMENT"} == ds_client.ds.plugin_config("env-plugin") - del os.environ["FOO_ENV"] +async def test_plugin_config_env(ds_client, monkeypatch): + monkeypatch.setenv("FOO_ENV", "FROM_ENVIRONMENT") + assert ds_client.ds.plugin_config("env-plugin") == {"foo": "FROM_ENVIRONMENT"} + + +@pytest.mark.asyncio +async def test_plugin_config_env_from_config(monkeypatch): + monkeypatch.setenv("FOO_ENV", "FROM_ENVIRONMENT_2") + datasette = Datasette( + config={"plugins": {"env-plugin": {"setting": {"$env": "FOO_ENV"}}}} + ) + assert datasette.plugin_config("env-plugin") == {"setting": "FROM_ENVIRONMENT_2"} @pytest.mark.asyncio From 60c6692f6802dfae6a433f648f287be30ef52325 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 21:57:09 -0800 Subject: [PATCH 155/767] table_config instead of table_metadata (#2257) Table configuration that was incorrectly placed in metadata is now treated as if it was in config. New await datasette.table_config() method. Closes #2247 --- datasette/app.py | 12 +++-- datasette/database.py | 10 ++-- datasette/utils/__init__.py | 69 +++++++++++++++++++----- datasette/views/table.py | 11 ++-- tests/test_api.py | 101 +++++++++++++++++++++++++++++++++++- tests/test_html.py | 2 +- 6 files changed, 175 insertions(+), 30 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index b0b8f041..373b3e95 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -75,6 +75,7 @@ from .utils import ( format_bytes, module_from_path, move_plugins, + move_table_config, parse_metadata, resolve_env_secrets, resolve_routes, @@ -346,7 +347,9 @@ class Datasette: # Move any "plugins" settings from metadata to config - updates them in place metadata = metadata or {} config = config or {} - move_plugins(metadata, config) + metadata, config = move_plugins(metadata, config) + # Now migrate any known table configuration settings over as well + metadata, config = move_table_config(metadata, config) self._metadata_local = metadata or {} self.sqlite_extensions = [] @@ -1202,10 +1205,11 @@ class Datasette: def _actor(self, request): return {"actor": request.actor} - async def table_config(self, database, table): - """Fetch table-specific metadata.""" + async def table_config(self, database: str, table: str) -> dict: + """Return dictionary of configuration for specified table""" return ( - (self.metadata("databases") or {}) + (self.config or {}) + .get("databases", {}) .get(database, {}) .get("tables", {}) .get(table, {}) diff --git a/datasette/database.py b/datasette/database.py index 1c1b3e1b..fba81496 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -487,13 +487,11 @@ class Database: ) ).rows ] - # Add any from metadata.json - db_metadata = self.ds.metadata(database=self.name) - if "tables" in db_metadata: + # Add any tables marked as hidden in config + db_config = self.ds.config.get("databases", {}).get(self.name, {}) + if "tables" in db_config: hidden_tables += [ - t - for t in db_metadata["tables"] - if db_metadata["tables"][t].get("hidden") + t for t in db_config["tables"] if db_config["tables"][t].get("hidden") ] # Also mark as hidden any tables which start with the name of a hidden table # e.g. "searchable_fts" implies "searchable_fts_content" should be hidden diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 4c940645..fcaebe3f 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -2,6 +2,7 @@ import asyncio from contextlib import contextmanager import click from collections import OrderedDict, namedtuple, Counter +import copy import base64 import hashlib import inspect @@ -17,7 +18,7 @@ import time import types import secrets import shutil -from typing import Iterable +from typing import Iterable, Tuple import urllib import yaml from .shutil_backport import copytree @@ -1290,11 +1291,24 @@ def make_slot_function(name, datasette, request, **kwargs): return inner -def move_plugins(source, destination): +def prune_empty_dicts(d: dict): + """ + Recursively prune all empty dictionaries from a given dictionary. + """ + for key, value in list(d.items()): + if isinstance(value, dict): + prune_empty_dicts(value) + if value == {}: + d.pop(key, None) + + +def move_plugins(source: dict, destination: dict) -> Tuple[dict, dict]: """ Move 'plugins' keys from source to destination dictionary. Creates hierarchy in destination if needed. After moving, recursively remove any keys in the source that are left empty. """ + source = copy.deepcopy(source) + destination = copy.deepcopy(destination) def recursive_move(src, dest, path=None): if path is None: @@ -1316,18 +1330,49 @@ def move_plugins(source, destination): if not value: src.pop(key, None) - def prune_empty_dicts(d): - """ - Recursively prune all empty dictionaries from a given dictionary. - """ - for key, value in list(d.items()): - if isinstance(value, dict): - prune_empty_dicts(value) - if value == {}: - d.pop(key, None) - recursive_move(source, destination) prune_empty_dicts(source) + return source, destination + + +_table_config_keys = ( + "hidden", + "sort", + "sort_desc", + "size", + "sortable_columns", + "label_column", + "facets", + "fts_table", + "fts_pk", + "searchmode", + "units", +) + + +def move_table_config(metadata: dict, config: dict): + """ + Move all known table configuration keys from metadata to config. + """ + if "databases" not in metadata: + return metadata, config + metadata = copy.deepcopy(metadata) + config = copy.deepcopy(config) + for database_name, database in metadata["databases"].items(): + if "tables" not in database: + continue + for table_name, table in database["tables"].items(): + for key in _table_config_keys: + if key in table: + config.setdefault("databases", {}).setdefault( + database_name, {} + ).setdefault("tables", {}).setdefault(table_name, {})[ + key + ] = table.pop( + key + ) + prune_empty_dicts(metadata) + return metadata, config def redact_keys(original: dict, key_patterns: Iterable) -> dict: diff --git a/datasette/views/table.py b/datasette/views/table.py index 2b5b7c24..50d2b3c2 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -142,11 +142,11 @@ async def display_columns_and_rows( """Returns columns, rows for specified table - including fancy foreign key treatment""" sortable_columns = sortable_columns or set() db = datasette.databases[database_name] - table_metadata = await datasette.table_config(database_name, table_name) - column_descriptions = table_metadata.get("columns") or {} + column_descriptions = datasette.metadata("columns", database_name, table_name) or {} column_details = { col.name: col for col in await db.table_column_details(table_name) } + table_config = await datasette.table_config(database_name, table_name) pks = await db.primary_keys(table_name) pks_for_display = pks if not pks_for_display: @@ -193,7 +193,6 @@ async def display_columns_and_rows( "raw": pk_path, "value": markupsafe.Markup( '{flat_pks}'.format( - base_url=base_url, table_path=datasette.urls.table(database_name, table_name), flat_pks=str(markupsafe.escape(pk_path)), flat_pks_quoted=path_from_row_pks(row, pks, not pks), @@ -274,9 +273,9 @@ async def display_columns_and_rows( ), ) ) - elif column in table_metadata.get("units", {}) and value != "": + elif column in table_config.get("units", {}) and value != "": # Interpret units using pint - value = value * ureg(table_metadata["units"][column]) + value = value * ureg(table_config["units"][column]) # Pint uses floating point which sometimes introduces errors in the compact # representation, which we have to round off to avoid ugliness. In the vast # majority of cases this rounding will be inconsequential. I hope. @@ -591,7 +590,7 @@ class TableDropView(BaseView): try: data = json.loads(await request.post_body()) confirm = data.get("confirm") - except json.JSONDecodeError as e: + except json.JSONDecodeError: pass if not confirm: diff --git a/tests/test_api.py b/tests/test_api.py index 0a1f3725..8cb73dbb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -771,7 +771,7 @@ def test_databases_json(app_client_two_attached_databases_one_immutable): @pytest.mark.asyncio async def test_metadata_json(ds_client): response = await ds_client.get("/-/metadata.json") - assert response.json() == METADATA + assert response.json() == ds_client.ds.metadata() @pytest.mark.asyncio @@ -1061,3 +1061,102 @@ async def test_config_json(config, expected): ds = Datasette(config=config) response = await ds.client.get("/-/config.json") assert response.json() == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "metadata,expected_config,expected_metadata", + ( + ({}, {}, {}), + ( + # Metadata input + { + "title": "Datasette Fixtures", + "databases": { + "fixtures": { + "tables": { + "sortable": { + "sortable_columns": [ + "sortable", + "sortable_with_nulls", + "sortable_with_nulls_2", + "text", + ], + }, + "no_primary_key": {"sortable_columns": [], "hidden": True}, + "units": {"units": {"distance": "m", "frequency": "Hz"}}, + "primary_key_multiple_columns_explicit_label": { + "label_column": "content2" + }, + "simple_view": {"sortable_columns": ["content"]}, + "searchable_view_configured_by_metadata": { + "fts_table": "searchable_fts", + "fts_pk": "pk", + }, + "roadside_attractions": { + "columns": { + "name": "The name of the attraction", + "address": "The street address for the attraction", + } + }, + "attraction_characteristic": {"sort_desc": "pk"}, + "facet_cities": {"sort": "name"}, + "paginated_view": {"size": 25}, + }, + } + }, + }, + # Should produce a config with just the table configuration keys + { + "databases": { + "fixtures": { + "tables": { + "sortable": { + "sortable_columns": [ + "sortable", + "sortable_with_nulls", + "sortable_with_nulls_2", + "text", + ] + }, + "units": {"units": {"distance": "m", "frequency": "Hz"}}, + # These one get redacted: + "no_primary_key": "***", + "primary_key_multiple_columns_explicit_label": "***", + "simple_view": {"sortable_columns": ["content"]}, + "searchable_view_configured_by_metadata": { + "fts_table": "searchable_fts", + "fts_pk": "pk", + }, + "attraction_characteristic": {"sort_desc": "pk"}, + "facet_cities": {"sort": "name"}, + "paginated_view": {"size": 25}, + } + } + } + }, + # And metadata with everything else + { + "title": "Datasette Fixtures", + "databases": { + "fixtures": { + "tables": { + "roadside_attractions": { + "columns": { + "name": "The name of the attraction", + "address": "The street address for the attraction", + } + }, + } + } + }, + }, + ), + ), +) +async def test_upgrade_metadata(metadata, expected_config, expected_metadata): + ds = Datasette(metadata=metadata) + response = await ds.client.get("/-/config.json") + assert response.json() == expected_config + response2 = await ds.client.get("/-/metadata.json") + assert response2.json() == expected_metadata diff --git a/tests/test_html.py b/tests/test_html.py index 86895844..8229b166 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -753,7 +753,7 @@ async def test_metadata_json_html(ds_client): response = await ds_client.get("/-/metadata") assert response.status_code == 200 pre = Soup(response.content, "html.parser").find("pre") - assert METADATA == json.loads(pre.text) + assert ds_client.ds.metadata() == json.loads(pre.text) @pytest.mark.asyncio From 9ac9f0152f1d3396d01ceddfcaf07fd1cf3f7168 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:18:38 -0800 Subject: [PATCH 156/767] Migrate allow from metadata to config if necessary, closes #2249 --- datasette/app.py | 6 +++--- datasette/utils/__init__.py | 9 +++++---- tests/test_permissions.py | 20 ++++++++++++-------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 373b3e95..af8cfeab 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -74,7 +74,7 @@ from .utils import ( find_spatialite, format_bytes, module_from_path, - move_plugins, + move_plugins_and_allow, move_table_config, parse_metadata, resolve_env_secrets, @@ -344,10 +344,10 @@ class Datasette: with config_files[0].open() as fp: config = parse_metadata(fp.read()) - # Move any "plugins" settings from metadata to config - updates them in place + # Move any "plugins" and "allow" settings from metadata to config - updates them in place metadata = metadata or {} config = config or {} - metadata, config = move_plugins(metadata, config) + metadata, config = move_plugins_and_allow(metadata, config) # Now migrate any known table configuration settings over as well metadata, config = move_table_config(metadata, config) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index fcaebe3f..f2cd7eb0 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1302,10 +1302,11 @@ def prune_empty_dicts(d: dict): d.pop(key, None) -def move_plugins(source: dict, destination: dict) -> Tuple[dict, dict]: +def move_plugins_and_allow(source: dict, destination: dict) -> Tuple[dict, dict]: """ - Move 'plugins' keys from source to destination dictionary. Creates hierarchy in destination if needed. - After moving, recursively remove any keys in the source that are left empty. + Move 'plugins' and 'allow' keys from source to destination dictionary. Creates + hierarchy in destination if needed. After moving, recursively remove any keys + in the source that are left empty. """ source = copy.deepcopy(source) destination = copy.deepcopy(destination) @@ -1315,7 +1316,7 @@ def move_plugins(source: dict, destination: dict) -> Tuple[dict, dict]: path = [] for key, value in list(src.items()): new_path = path + [key] - if key == "plugins": + if key in ("plugins", "allow"): # Navigate and create the hierarchy in destination if needed d = dest for step in path: diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 9917b749..6713b850 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -89,10 +89,11 @@ def test_view_padlock(allow, expected_anon, expected_auth, path, padlock_client) ({"id": "root"}, 403, 200), ], ) -def test_view_database(allow, expected_anon, expected_auth): - with make_app_client( - config={"databases": {"fixtures": {"allow": allow}}} - ) as client: +@pytest.mark.parametrize("use_metadata", (True, False)) +def test_view_database(allow, expected_anon, expected_auth, use_metadata): + key = "metadata" if use_metadata else "config" + kwargs = {key: {"databases": {"fixtures": {"allow": allow}}}} + with make_app_client(**kwargs) as client: for path in ( "/fixtures", "/fixtures/compound_three_primary_keys", @@ -173,16 +174,19 @@ def test_database_list_respects_view_table(): ({"id": "root"}, 403, 200), ], ) -def test_view_table(allow, expected_anon, expected_auth): - with make_app_client( - config={ +@pytest.mark.parametrize("use_metadata", (True, False)) +def test_view_table(allow, expected_anon, expected_auth, use_metadata): + key = "metadata" if use_metadata else "config" + kwargs = { + key: { "databases": { "fixtures": { "tables": {"compound_three_primary_keys": {"allow": allow}} } } } - ) as client: + } + with make_app_client(**kwargs) as client: anon_response = client.get("/fixtures/compound_three_primary_keys") assert expected_anon == anon_response.status if allow and anon_response.status == 200: From ad01f9d3217f2447b0a321ee7731900dac7e3e6d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:24:24 -0800 Subject: [PATCH 157/767] 1.0a8 release notes Closes #2243 * Changelog for jinja2_environment_from_request and plugin_hook_slots * track_event() in changelog * Remove Using YAML for metadata section - no longer necessary now we show YAML and JSON examples everywhere. * Configuration via the command-line section - #2252 * JavaScript plugins in release notes, refs #2052 * /-/config in changelog, refs #2254 Refs #2052, #2156, #2243, #2247, #2249, #2252, #2254 --- docs/changelog.rst | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f4b928e3..1f47b429 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,83 @@ Changelog ========= +.. _v1_0_a8: + +1.0a8 (2024-02-01) +------------------ + +This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, and adds several new plugin hooks. + +Configuration +~~~~~~~~~~~~~ + +- Plugin configuration now lives in the :ref:`datasette.yaml configuration file `, passed to Datasette using the ``-c/--config`` option. Thanks, Alex Garcia. (:issue:`2093`) + + .. code-block:: bash + + datasette -c datasette.yaml + + Where ``datasette.yaml`` contains configuration that looks like this: + + .. code-block:: yaml + + plugins: + datasette-cluster-map: + latitude_column: xlat + longitude_column: xlon + + Previously plugins were configured in ``metadata.yaml``, which was confusing as plugin settings were unrelated to database and table metadata. +- The ``-s/--setting`` option can now be used to set plugin configuration as well. See :ref:`configuration_cli` for details. (:issue:`2252`) + + The above YAML configuration example using ``-s/--setting`` looks like this: + + .. code-block:: bash + + datasette mydatabase.db \ + -s plugins.datasette-cluster-map.latitude_column xlat \ + -s plugins.datasette-cluster-map.longitude_column xlon + +- The new ``/-/config`` page shows the current instance configuration, after redacting keys that could contain sensitive data such as API keys or passwords. (:issue:`2254`) + +- Existing Datasette installations may already have configuration set in ``metadata.yaml`` that should be migrated to ``datasette.yaml``. To avoid breaking these installations, Datasette will silently treat table configuration, plugin configuration and allow blocks in metadata as if they had been specified in configuration instead. (:issue:`2247`) (:issue:`2248`) (:issue:`2249`) + +JavaScript plugins +~~~~~~~~~~~~~~~~~~ + +Datasette now includes a :ref:`JavaScript plugins mechanism `, allowing JavaScript to customize Datasette in a way that can collaborate with other plugins. + +This provides two initial hooks, with more to come in the future: + +- :ref:`makeAboveTablePanelConfigs() ` can add additional panels to the top of the table page. +- :ref:`makeColumnActions() ` can add additional actions to the column menu. + +Thanks `Cameron Yick `__ for contributing this feature. (`#2052 `__) + +Plugin hooks +~~~~~~~~~~~~ + +- New :ref:`plugin_hook_jinja2_environment_from_request` plugin hook, which can be used to customize the current Jinja environment based on the incoming request. This can be used to modify the template lookup path based on the incoming request hostname, among other things. (:issue:`2225`) +- New :ref:`family of template slot plugin hooks `: ``top_homepage``, ``top_database``, ``top_table``, ``top_row``, ``top_query``, ``top_canned_query``. Plugins can use these to provide additional HTML to be injected at the top of the corresponding pages. (:issue:`1191`) +- New :ref:`track_event() mechanism ` for plugins to emit and receive events when certain events occur within Datasette. (:issue:`2240`) + - Plugins can register additional event classes using :ref:`plugin_hook_register_events`. + - They can then trigger those events with the :ref:`datasette.track_event(event) ` internal method. + - Plugins can subscribe to notifications of events using the :ref:`plugin_hook_track_event` plugin hook. +- New internal function for plugin authors: :ref:`database_execute_isolated_fn`, for creating a new SQLite connection, executing code and then closing that connection, all while preventing other code from writing to that particular database. This connection will not have the :ref:`prepare_connection() ` plugin hook executed against it, allowing plugins to perform actions that might otherwise be blocked by existing connection configuration. (:issue:`2218`) + +Documentation +~~~~~~~~~~~~~ + +- Documentation describing :ref:`how to write tests that use signed actor cookies ` using ``datasette.client.actor_cookie()``. (:issue:`1830`) +- Documentation on how to :ref:`register a plugin for the duration of a test `. (:issue:`2234`) +- The :ref:`configuration documentation ` now shows examples of both YAML and JSON for each setting. + +Minor fixes +~~~~~~~~~~~ + +- Datasette no longer attempts to run SQL queries in parallel when rendering a table page, as this was leading to some rare crashing bugs. (:issue:`2189`) +- Fixed warning: ``DeprecationWarning: pkg_resources is deprecated as an API`` (:issue:`2057`) +- Fixed bug where ``?_extra=columns`` parameter returned an incorrectly shaped response. (:issue:`2230`) + .. _v0_64_6: 0.64.6 (2023-12-22) From c64453a4a137ea815f4d94a99cdc4c7709734839 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:28:22 -0800 Subject: [PATCH 158/767] Fix the date on the 1.0a8 release (due to go tomorrow) Refs #2258 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1f47b429..5e9e3ba2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changelog .. _v1_0_a8: -1.0a8 (2024-02-01) +1.0a8 (2024-02-07) ------------------ This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, and adds several new plugin hooks. From d0089ba7769d58b97c5dc1e07969246502d97544 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:30:30 -0800 Subject: [PATCH 159/767] Note in changelog about datasette publish, refs #2195 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5e9e3ba2..e17dc2f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -44,6 +44,8 @@ Configuration - Existing Datasette installations may already have configuration set in ``metadata.yaml`` that should be migrated to ``datasette.yaml``. To avoid breaking these installations, Datasette will silently treat table configuration, plugin configuration and allow blocks in metadata as if they had been specified in configuration instead. (:issue:`2247`) (:issue:`2248`) (:issue:`2249`) +Note that the ``datasette publish`` command has not yet been updated to accept a ``datasette.yaml`` configuration file. This will be addressed in :issue:`2195` but for the moment you can include those settings in ``metadata.yaml`` instead. + JavaScript plugins ~~~~~~~~~~~~~~~~~~ From df8d1c055a48a36693d9caec3915eb93c1acefc0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Feb 2024 22:59:58 -0800 Subject: [PATCH 160/767] Mention JS plugins in release intro --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e17dc2f8..a73635a8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Changelog 1.0a8 (2024-02-07) ------------------ -This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, and adds several new plugin hooks. +This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, introduces a new system for JavaScript plugins and adds several new plugin hooks. Configuration ~~~~~~~~~~~~~ From 1e31821d9ff2bc81c62bcd442214e9098cf785a4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 08:25:47 -0800 Subject: [PATCH 161/767] Link to events docs from changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a73635a8..dfcf492f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -67,6 +67,7 @@ Plugin hooks - Plugins can register additional event classes using :ref:`plugin_hook_register_events`. - They can then trigger those events with the :ref:`datasette.track_event(event) ` internal method. - Plugins can subscribe to notifications of events using the :ref:`plugin_hook_track_event` plugin hook. + - Datasette core now emits ``login``, ``logout``, ``create-token``, ``create-table``, ``drop-table``, ``insert-rows``, ``upsert-rows``, ``update-row``, ``delete-row`` events, :ref:`documented here `. - New internal function for plugin authors: :ref:`database_execute_isolated_fn`, for creating a new SQLite connection, executing code and then closing that connection, all while preventing other code from writing to that particular database. This connection will not have the :ref:`prepare_connection() ` plugin hook executed against it, allowing plugins to perform actions that might otherwise be blocked by existing connection configuration. (:issue:`2218`) Documentation From e0794ddd52697812848c0b59f68e49a2e9361693 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 08:32:47 -0800 Subject: [PATCH 162/767] Link to annotated release notes blog post --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index dfcf492f..d164f71d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,8 @@ Changelog This alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, introduces a new system for JavaScript plugins and adds several new plugin hooks. +See `Datasette 1.0a8: JavaScript plugins, new plugin hooks and plugin configuration in datasette.yaml `__ for an annotated version of these release notes. + Configuration ~~~~~~~~~~~~~ From 9989f257094daaf26e0cb0cebe31f17f19d4cad2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 08:34:05 -0800 Subject: [PATCH 163/767] Release 1.0a8 Refs Refs #2052, #2156, #2243, #2247, #2249, #2252, #2254, #2258 --- datasette/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 75d44727..e43b9918 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a8.dev1" +__version__ = "1.0a8" __version_info__ = tuple(__version__.split(".")) From 569aacd39bcc5529b2f463c89c616e3ada21c560 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 7 Feb 2024 22:53:14 -0800 Subject: [PATCH 164/767] Link to /en/latest/ changelog --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57f17a5c..662f2a11 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Datasette [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/) -[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/stable/changelog.html) +[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/latest/changelog.html) [![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/) [![Tests](https://github.com/simonw/datasette/workflows/Test/badge.svg)](https://github.com/simonw/datasette/actions?query=workflow%3ATest) [![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest) From 900d15bcb81c90d26cfebc3fe463c4b0465832c2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 12:21:13 -0800 Subject: [PATCH 165/767] alter table support for /db/-/create API, refs #2101 --- datasette/default_permissions.py | 10 ++++- datasette/events.py | 25 +++++++++++ datasette/views/database.py | 61 +++++++++++++++++++++++--- docs/authentication.rst | 12 ++++++ tests/test_api_write.py | 74 ++++++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+), 8 deletions(-) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index d29dbe84..c13f2ed2 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -8,7 +8,6 @@ from typing import Union, Tuple @hookimpl def register_permissions(): return ( - # name, abbr, description, takes_database, takes_resource, default Permission( name="view-instance", abbr="vi", @@ -109,6 +108,14 @@ def register_permissions(): takes_resource=False, default=False, ), + Permission( + name="alter-table", + abbr="at", + description="Alter tables", + takes_database=True, + takes_resource=True, + default=False, + ), Permission( name="drop-table", abbr="dt", @@ -129,6 +136,7 @@ def permission_allowed_default(datasette, actor, action, resource): "debug-menu", "insert-row", "create-table", + "alter-table", "drop-table", "delete-row", "update-row", diff --git a/datasette/events.py b/datasette/events.py index 96244779..ae90972d 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -108,6 +108,30 @@ class DropTableEvent(Event): table: str +@dataclass +class AlterTableEvent(Event): + """ + Event name: ``alter-table`` + + A table has been altered. + + :ivar database: The name of the database where the table was altered + :type database: str + :ivar table: The name of the table that was altered + :type table: str + :ivar before_schema: The table's SQL schema before the alteration + :type before_schema: str + :ivar after_schema: The table's SQL schema after the alteration + :type after_schema: str + """ + + name = "alter-table" + database: str + table: str + before_schema: str + after_schema: str + + @dataclass class InsertRowsEvent(Event): """ @@ -203,6 +227,7 @@ def register_events(): LogoutEvent, CreateTableEvent, CreateTokenEvent, + AlterTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, diff --git a/datasette/views/database.py b/datasette/views/database.py index 6d17b16c..bd55064f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,7 +10,7 @@ import re import sqlite_utils import textwrap -from datasette.events import CreateTableEvent +from datasette.events import AlterTableEvent, CreateTableEvent from datasette.database import QueryInterrupted from datasette.utils import ( add_cors_headers, @@ -792,7 +792,17 @@ class MagicParameters(dict): class TableCreateView(BaseView): name = "table-create" - _valid_keys = {"table", "rows", "row", "columns", "pk", "pks", "ignore", "replace"} + _valid_keys = { + "table", + "rows", + "row", + "columns", + "pk", + "pks", + "ignore", + "replace", + "alter", + } _supported_column_types = { "text", "integer", @@ -876,6 +886,20 @@ class TableCreateView(BaseView): ): return _error(["Permission denied - need insert-row"], 403) + alter = False + if rows or row: + if not table_exists: + # if table is being created for the first time, alter=True + alter = True + else: + # alter=True only if they request it AND they have permission + if data.get("alter"): + if not await self.ds.permission_allowed( + request.actor, "alter-table", resource=database_name + ): + return _error(["Permission denied - need alter-table"], 403) + alter = True + if columns: if rows or row: return _error(["Cannot specify columns with rows or row"]) @@ -939,10 +963,18 @@ class TableCreateView(BaseView): return _error(["pk cannot be changed for existing table"]) pks = actual_pks + initial_schema = None + if table_exists: + initial_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + def create_table(conn): table = sqlite_utils.Database(conn)[table_name] if rows: - table.insert_all(rows, pk=pks or pk, ignore=ignore, replace=replace) + table.insert_all( + rows, pk=pks or pk, ignore=ignore, replace=replace, alter=alter + ) else: table.create( {c["name"]: c["type"] for c in columns}, @@ -954,6 +986,18 @@ class TableCreateView(BaseView): schema = await db.execute_write_fn(create_table) except Exception as e: return _error([str(e)]) + + if initial_schema is not None and initial_schema != schema: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=initial_schema, + after_schema=schema, + ) + ) + table_url = self.ds.absolute_url( request, self.ds.urls.table(db.name, table_name) ) @@ -970,11 +1014,14 @@ class TableCreateView(BaseView): } if rows: details["row_count"] = len(rows) - await self.ds.track_event( - CreateTableEvent( - request.actor, database=db.name, table=table_name, schema=schema + + if not table_exists: + # Only log creation if we created a table + await self.ds.track_event( + CreateTableEvent( + request.actor, database=db.name, table=table_name, schema=schema + ) ) - ) return Response.json(details, status=201) diff --git a/docs/authentication.rst b/docs/authentication.rst index 8758765d..87ee6385 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1217,6 +1217,18 @@ Actor is allowed to create a database table. Default *deny*. +.. _permissions_alter_table: + +alter-table +----------- + +Actor is allowed to alter a database table. + +``resource`` - tuple: (string, string) + The name of the database, then the name of the table + +Default *deny*. + .. _permissions_drop_table: drop-table diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 9caf9fdf..30cbfbab 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1349,3 +1349,77 @@ async def test_method_not_allowed(ds_write, path): "ok": False, "error": "Method not allowed", } + + +@pytest.mark.asyncio +async def test_create_uses_alter_by_default_for_new_table(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "new_table", + "rows": [ + { + "name": "Row 1", + } + ] + * 100 + + [ + {"name": "Row 2", "extra": "Extra"}, + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + event = last_event(ds_write) + assert event.name == "create-table" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("has_alter_permission", (True,)) # False)) +async def test_create_using_alter_against_existing_table( + ds_write, has_alter_permission +): + token = write_token( + ds_write, permissions=["ir", "ct"] + (["at"] if has_alter_permission else []) + ) + # First create the table + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "new_table", + "rows": [ + { + "name": "Row 1", + } + ], + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + # Now try to insert more rows using /-/create with alter=True + response2 = await ds_write.client.post( + "/data/-/create", + json={ + "table": "new_table", + "rows": [{"name": "Row 2", "extra": "extra"}], + "pk": "id", + "alter": True, + }, + headers=_headers(token), + ) + if not has_alter_permission: + assert response2.status_code == 403 + assert response2.json() == { + "ok": False, + "errors": ["Permission denied - need alter-table"], + } + else: + assert response2.status_code == 201 + # It should have altered the table + event = last_event(ds_write) + assert event.name == "alter-table" + assert "extra" not in event.before_schema + assert "extra" in event.after_schema From 574687834f4bd8e73281731b8ff01bfe093fecb5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 12:33:41 -0800 Subject: [PATCH 166/767] Docs for /db/-/create alter: true option, refs #2101 --- docs/json_api.rst | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/json_api.rst b/docs/json_api.rst index 16b997eb..68a0c984 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -834,19 +834,22 @@ To create a table, make a ``POST`` to ``//-/create``. This requires th The JSON here describes the table that will be created: -* ``table`` is the name of the table to create. This field is required. -* ``columns`` is a list of columns to create. Each column is a dictionary with ``name`` and ``type`` keys. +* ``table`` is the name of the table to create. This field is required. +* ``columns`` is a list of columns to create. Each column is a dictionary with ``name`` and ``type`` keys. - - ``name`` is the name of the column. This is required. - - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. + - ``name`` is the name of the column. This is required. + - ``type`` is the type of the column. This is optional - if not provided, ``text`` will be assumed. The valid types are ``text``, ``integer``, ``float`` and ``blob``. -* ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. +* ``pk`` is the primary key for the table. This is optional - if not provided, Datasette will create a SQLite table with a hidden ``rowid`` column. - If the primary key is an integer column, it will be configured to automatically increment for each new record. + If the primary key is an integer column, it will be configured to automatically increment for each new record. - If you set this to ``id`` without including an ``id`` column in the list of ``columns``, Datasette will create an integer ID column for you. + If you set this to ``id`` without including an ``id`` column in the list of ``columns``, Datasette will create an auto-incrementing integer ID column for you. -* ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. +* ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. +* ``ignore`` can be set to ``true`` to ignore existing rows by primary key if the table already exists. +* ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. +* ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. If the table is successfully created this will return a ``201`` status code and the following response: @@ -925,6 +928,8 @@ You can avoid this error by passing the same ``"ignore": true`` or ``"replace": To use the ``"replace": true`` option you will also need the :ref:`permissions_update_row` permission. +Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`permissions_alter_table` permission. + .. _TableDropView: Dropping tables From b5ccc4d60844a24fdf91c3f317d8cda4a285a58d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 12:35:12 -0800 Subject: [PATCH 167/767] Test for Permission denied - need alter-table --- tests/test_api_write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 30cbfbab..abf9a88a 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1377,7 +1377,7 @@ async def test_create_uses_alter_by_default_for_new_table(ds_write): @pytest.mark.asyncio -@pytest.mark.parametrize("has_alter_permission", (True,)) # False)) +@pytest.mark.parametrize("has_alter_permission", (True, False)) async def test_create_using_alter_against_existing_table( ds_write, has_alter_permission ): From 528d89d1a3d6ff85047a7eef9a7623efdd2fb19f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:14:12 -0800 Subject: [PATCH 168/767] alter: true support for /-/insert and /-/upsert, refs #2101 --- datasette/views/table.py | 48 +++++++++++++++++++++++++++++++++++----- docs/json_api.rst | 6 ++++- tests/test_api_write.py | 48 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 91 insertions(+), 11 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 50d2b3c2..fcbe253d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -8,7 +8,12 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted -from datasette.events import DropTableEvent, InsertRowsEvent, UpsertRowsEvent +from datasette.events import ( + AlterTableEvent, + DropTableEvent, + InsertRowsEvent, + UpsertRowsEvent, +) from datasette import tracer from datasette.utils import ( add_cors_headers, @@ -388,7 +393,7 @@ class TableInsertView(BaseView): extras = { key: value for key, value in data.items() if key not in ("row", "rows") } - valid_extras = {"return", "ignore", "replace"} + valid_extras = {"return", "ignore", "replace", "alter"} invalid_extras = extras.keys() - valid_extras if invalid_extras: return _errors( @@ -397,7 +402,6 @@ class TableInsertView(BaseView): if extras.get("ignore") and extras.get("replace"): return _errors(['Cannot use "ignore" and "replace" at the same time']) - # Validate columns of each row columns = set(await db.table_columns(table_name)) columns.update(pks_list) @@ -412,7 +416,7 @@ class TableInsertView(BaseView): ) ) invalid_columns = set(row.keys()) - columns - if invalid_columns: + if invalid_columns and not extras.get("alter"): errors.append( "Row {} has invalid columns: {}".format( i, ", ".join(sorted(invalid_columns)) @@ -476,10 +480,23 @@ class TableInsertView(BaseView): ignore = extras.get("ignore") replace = extras.get("replace") + alter = extras.get("alter") if upsert and (ignore or replace): return _error(["Upsert does not support ignore or replace"], 400) + initial_schema = None + if alter: + # Must have alter-table permission + if not await self.ds.permission_allowed( + request.actor, "alter-table", resource=(database_name, table_name) + ): + return _error(["Permission denied for alter-table"], 403) + # Track initial schema to check if it changed later + initial_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + should_return = bool(extras.get("return", False)) row_pk_values_for_later = [] if should_return and upsert: @@ -489,9 +506,13 @@ class TableInsertView(BaseView): table = sqlite_utils.Database(conn)[table_name] kwargs = {} if upsert: - kwargs["pk"] = pks[0] if len(pks) == 1 else pks + kwargs = { + "pk": pks[0] if len(pks) == 1 else pks, + "alter": alter, + } else: - kwargs = {"ignore": ignore, "replace": replace} + # Insert + kwargs = {"ignore": ignore, "replace": replace, "alter": alter} if should_return and not upsert: rowids = [] method = table.upsert if upsert else table.insert @@ -552,6 +573,21 @@ class TableInsertView(BaseView): ) ) + if initial_schema is not None: + after_schema = await db.execute_fn( + lambda conn: sqlite_utils.Database(conn)[table_name].schema + ) + if initial_schema != after_schema: + await self.ds.track_event( + AlterTableEvent( + request.actor, + database=database_name, + table=table_name, + before_schema=initial_schema, + after_schema=after_schema, + ) + ) + return Response.json(result, status=200 if upsert else 201) diff --git a/docs/json_api.rst b/docs/json_api.rst index 68a0c984..000f532d 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -618,6 +618,8 @@ Pass ``"ignore": true`` to ignore these errors and insert the other rows: Or you can pass ``"replace": true`` to replace any rows with conflicting primary keys with the new values. +Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. + .. _TableUpsertView: Upserting rows @@ -728,6 +730,8 @@ When using upsert you must provide the primary key column (or columns if the tab If your table does not have an explicit primary key you should pass the SQLite ``rowid`` key instead. +Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. + .. _RowUpdateView: Updating a row @@ -849,7 +853,7 @@ The JSON here describes the table that will be created: * ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. * ``ignore`` can be set to ``true`` to ignore existing rows by primary key if the table already exists. * ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. -* ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. +* ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. If the table is successfully created this will return a ``201`` status code and the following response: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index abf9a88a..9e1d73e0 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -60,6 +60,27 @@ async def test_insert_row(ds_write): assert not event.replace +@pytest.mark.asyncio +async def test_insert_row_alter(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/docs/-/insert", + json={ + "row": {"title": "Test", "score": 1.2, "age": 5, "extra": "extra"}, + "alter": True, + }, + headers=_headers(token), + ) + assert response.status_code == 201 + assert response.json()["ok"] is True + assert response.json()["rows"][0]["extra"] == "extra" + # Analytics event + event = last_event(ds_write) + assert event.name == "alter-table" + assert "extra" not in event.before_schema + assert "extra" in event.after_schema + + @pytest.mark.asyncio @pytest.mark.parametrize("return_rows", (True, False)) async def test_insert_rows(ds_write, return_rows): @@ -278,16 +299,27 @@ async def test_insert_rows(ds_write, return_rows): 403, ["Permission denied: need both insert-row and update-row"], ), + # Alter table forbidden without alter permission + ( + "/data/docs/-/upsert", + {"rows": [{"id": 1, "title": "One", "extra": "extra"}], "alter": True}, + "update-and-insert-but-no-alter", + 403, + ["Permission denied for alter-table"], + ), ), ) async def test_insert_or_upsert_row_errors( ds_write, path, input, special_case, expected_status, expected_errors ): - token = write_token(ds_write) + token_permissions = [] if special_case == "insert-but-not-update": - token = write_token(ds_write, permissions=["ir", "vi"]) + token_permissions = ["ir", "vi"] if special_case == "update-but-not-insert": - token = write_token(ds_write, permissions=["ur", "vi"]) + token_permissions = ["ur", "vi"] + if special_case == "update-and-insert-but-no-alter": + token_permissions = ["ur", "ir"] + token = write_token(ds_write, permissions=token_permissions) if special_case == "duplicate_id": await ds_write.get_database("data").execute_write( "insert into docs (id) values (1)" @@ -309,7 +341,9 @@ async def test_insert_or_upsert_row_errors( actor_response = ( await ds_write.client.get("/-/actor.json", headers=kwargs["headers"]) ).json() - print(actor_response) + assert set((actor_response["actor"] or {}).get("_r", {}).get("a") or []) == set( + token_permissions + ) if special_case == "invalid_json": del kwargs["json"] @@ -434,6 +468,12 @@ async def test_insert_ignore_replace( {"id": 1, "title": "Two", "score": 1}, ], ), + ( + # Upsert with an alter + {"rows": [{"id": 1, "title": "One"}], "pk": "id"}, + {"rows": [{"id": 1, "title": "Two", "extra": "extra"}], "alter": True}, + [{"id": 1, "title": "Two", "extra": "extra"}], + ), ), ) @pytest.mark.parametrize("should_return", (False, True)) From 4e944c29e4a208f173f15ac2df6253ff90f6466f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:19:47 -0800 Subject: [PATCH 169/767] Corrected path used in test_update_row_check_permission --- tests/test_api_write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 9e1d73e0..b43ee5a6 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -633,7 +633,7 @@ async def test_update_row_check_permission(ds_write, scenario): pk = await _insert_row(ds_write) - path = "/data/{}/{}/-/delete".format( + path = "/data/{}/{}/-/update".format( "docs" if scenario != "bad_table" else "bad_table", pk ) From c954795f9af9007e7c04d9b472bfd2faef647a87 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:30:48 -0800 Subject: [PATCH 170/767] alter: true for row/-/update, refs #2101 --- datasette/views/row.py | 12 +++++++++++- docs/json_api.rst | 2 ++ tests/test_api_write.py | 43 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 7b43b893..4d20e41a 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -237,11 +237,21 @@ class RowUpdateView(BaseView): if not "update" in data or not isinstance(data["update"], dict): return _error(["JSON must contain an update dictionary"]) + invalid_keys = set(data.keys()) - {"update", "return", "alter"} + if invalid_keys: + return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) + update = data["update"] + alter = data.get("alter") + if alter and not await self.ds.permission_allowed( + request.actor, "alter-table", resource=(resolved.db.name, resolved.table) + ): + return _error(["Permission denied for alter-table"], 403) + def update_row(conn): sqlite_utils.Database(conn)[resolved.table].update( - resolved.pk_values, update + resolved.pk_values, update, alter=alter ) try: diff --git a/docs/json_api.rst b/docs/json_api.rst index 000f532d..c401d97e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -787,6 +787,8 @@ The returned JSON will look like this: Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. +Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. + .. _RowDeleteView: Deleting a row diff --git a/tests/test_api_write.py b/tests/test_api_write.py index b43ee5a6..7cc38674 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -622,12 +622,17 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): @pytest.mark.asyncio -@pytest.mark.parametrize("scenario", ("no_token", "no_perm", "bad_table")) +@pytest.mark.parametrize( + "scenario", ("no_token", "no_perm", "bad_table", "cannot_alter") +) async def test_update_row_check_permission(ds_write, scenario): if scenario == "no_token": token = "bad_token" elif scenario == "no_perm": token = write_token(ds_write, actor_id="not-root") + elif scenario == "cannot_alter": + # update-row but no alter-table: + token = write_token(ds_write, permissions=["ur"]) else: token = write_token(ds_write) @@ -637,9 +642,13 @@ async def test_update_row_check_permission(ds_write, scenario): "docs" if scenario != "bad_table" else "bad_table", pk ) + json_body = {"update": {"title": "New title"}} + if scenario == "cannot_alter": + json_body["alter"] = True + response = await ds_write.client.post( path, - json={"update": {"title": "New title"}}, + json=json_body, headers=_headers(token), ) assert response.status_code == 403 if scenario in ("no_token", "bad_token") else 404 @@ -651,6 +660,36 @@ async def test_update_row_check_permission(ds_write, scenario): ) +@pytest.mark.asyncio +async def test_update_row_invalid_key(ds_write): + token = write_token(ds_write) + + pk = await _insert_row(ds_write) + + path = "/data/docs/{}/-/update".format(pk) + response = await ds_write.client.post( + path, + json={"update": {"title": "New title"}, "bad_key": 1}, + headers=_headers(token), + ) + assert response.status_code == 400 + assert response.json() == {"ok": False, "errors": ["Invalid keys: bad_key"]} + + +@pytest.mark.asyncio +async def test_update_row_alter(ds_write): + token = write_token(ds_write, permissions=["ur", "at"]) + pk = await _insert_row(ds_write) + path = "/data/docs/{}/-/update".format(pk) + response = await ds_write.client.post( + path, + json={"update": {"title": "New title", "extra": "extra"}, "alter": True}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json() == {"ok": True} + + @pytest.mark.asyncio @pytest.mark.parametrize( "input,expected_errors", From c62cfa6de836667834b5b9a7fef2b861307ac998 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 13:32:36 -0800 Subject: [PATCH 171/767] Fix upsert test to detect new alter-table event --- tests/test_api_write.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 7cc38674..2d127e1a 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -499,10 +499,14 @@ async def test_upsert(ds_write, initial, input, expected_rows, should_return): # Analytics event event = last_event(ds_write) - assert event.name == "upsert-rows" - assert event.num_rows == 1 assert event.database == "data" assert event.table == "upsert_test" + if input.get("alter"): + assert event.name == "alter-table" + assert "extra" in event.after_schema + else: + assert event.name == "upsert-rows" + assert event.num_rows == 1 if should_return: # We only expect it to return rows corresponding to those we sent From dcd9ea3622520c99a1f921766dc36ca4c0e3b796 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 14:14:58 -0800 Subject: [PATCH 172/767] datasette-events-db as an example of track_events() --- docs/plugin_hooks.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 16f5cebb..960dc9b6 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1817,6 +1817,7 @@ This example plugin logs details of all events to standard error: ) print(msg, file=sys.stderr, flush=True) +Example: `datasette-events-db `_ .. _plugin_hook_register_events: From bd9ed62e5d8821f9dc9e035b195452980c900b3c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 18:58:12 -0800 Subject: [PATCH 173/767] Make ds.pemrission_allawed(..., default=) a keyword-only argument, refs #2262 --- datasette/app.py | 2 +- datasette/views/table.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index af8cfeab..d943b97b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -896,7 +896,7 @@ class Datasette: await await_me_maybe(hook) async def permission_allowed( - self, actor, action, resource=None, default=DEFAULT_NOT_SET + self, actor, action, resource=None, *, default=DEFAULT_NOT_SET ): """Check permissions using the permissions_allowed plugin hook""" result = None diff --git a/datasette/views/table.py b/datasette/views/table.py index fcbe253d..1c187692 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -444,10 +444,10 @@ class TableInsertView(BaseView): # Must have insert-row AND upsert-row permissions if not ( await self.ds.permission_allowed( - request.actor, "insert-row", database_name, table_name + request.actor, "insert-row", resource=(database_name, table_name) ) and await self.ds.permission_allowed( - request.actor, "update-row", database_name, table_name + request.actor, "update-row", resource=(database_name, table_name) ) ): return _error( From 398a92cf1e54f868ff80f01634d6a814d1c61998 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Feb 2024 20:12:22 -0800 Subject: [PATCH 174/767] Include database in name of _execute_writes thread, closes #2265 --- datasette/database.py | 3 +++ tests/test_api.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/datasette/database.py b/datasette/database.py index fba81496..becb552c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -196,6 +196,9 @@ class Database: self._write_thread = threading.Thread( target=self._execute_writes, daemon=True ) + self._write_thread.name = "_execute_writes for database {}".format( + self.name + ) self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") reply_queue = janus.Queue() diff --git a/tests/test_api.py b/tests/test_api.py index 8cb73dbb..7a25b55e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -780,7 +780,11 @@ async def test_threads_json(ds_client): expected_keys = {"threads", "num_threads"} if sys.version_info >= (3, 7, 0): expected_keys.update({"tasks", "num_tasks"}) - assert set(response.json().keys()) == expected_keys + data = response.json() + assert set(data.keys()) == expected_keys + # Should be at least one _execute_writes thread for __INTERNAL__ + thread_names = [thread["name"] for thread in data["threads"]] + assert "_execute_writes for database __INTERNAL__" in thread_names @pytest.mark.asyncio From 5d7997418664bcdfdba714c16bd5a67c241e8740 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Feb 2024 07:19:47 -0800 Subject: [PATCH 175/767] Call them "notable events" --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 960dc9b6..5372ea5e 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1765,7 +1765,7 @@ Returns HTML to be displayed at the top of the canned query page. Event tracking -------------- -Datasette includes an internal mechanism for tracking analytical events. This can be used for analytics, but can also be used by plugins that want to listen out for when key events occur (such as a table being created) and take action in response. +Datasette includes an internal mechanism for tracking notable events. This can be used for analytics, but can also be used by plugins that want to listen out for when key events occur (such as a table being created) and take action in response. Plugins can register to receive events using the ``track_event`` plugin hook. From b89cac3b6a63929325c067d0cf2d5748e4bf4d2e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 13 Feb 2024 18:23:54 -0800 Subject: [PATCH 176/767] Use MD5 usedforsecurity=False on Python 3.9 and higher to pass FIPS Closes #2270 --- datasette/database.py | 4 ++-- datasette/utils/__init__.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index becb552c..707d8f85 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,7 +1,6 @@ import asyncio from collections import namedtuple from pathlib import Path -import hashlib import janus import queue import sys @@ -15,6 +14,7 @@ from .utils import ( detect_spatialite, get_all_foreign_keys, get_outbound_foreign_keys, + md5_not_usedforsecurity, sqlite_timelimit, sqlite3, table_columns, @@ -74,7 +74,7 @@ class Database: def color(self): if self.hash: return self.hash[:6] - return hashlib.md5(self.name.encode("utf8")).hexdigest()[:6] + return md5_not_usedforsecurity(self.name)[:6] def suggest_name(self): if self.path: diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index f2cd7eb0..e3637f7a 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -713,7 +713,7 @@ def to_css_class(s): """ if css_class_re.match(s): return s - md5_suffix = hashlib.md5(s.encode("utf8")).hexdigest()[:6] + md5_suffix = md5_not_usedforsecurity(s)[:6] # Strip leading _, - s = s.lstrip("_").lstrip("-") # Replace any whitespace with hyphens @@ -1401,3 +1401,11 @@ def redact_keys(original: dict, key_patterns: Iterable) -> dict: return data return redact(original) + + +def md5_not_usedforsecurity(s): + try: + return hashlib.md5(s.encode("utf8"), usedforsecurity=False).hexdigest() + except TypeError: + # For Python 3.8 which does not support usedforsecurity=False + return hashlib.md5(s.encode("utf8")).hexdigest() From 97de4d6362ce5a6c1e3520ecdc73b305ab269910 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 15 Feb 2024 21:35:49 -0800 Subject: [PATCH 177/767] Use transaction in delete_everything(), closes #2273 --- datasette/utils/internal_db.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 2e5ac53b..dd0d3a9d 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -69,18 +69,20 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - conn.execute( - "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_foreign_keys WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] - ) + with conn: + conn.execute( + "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_foreign_keys WHERE database_name = ?", + [database_name], + ) + conn.execute( + "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] + ) await internal_db.execute_write_fn(delete_everything) From 47e29e948b26e8c003a03b4fc46cb635134a3958 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 10:05:18 -0800 Subject: [PATCH 178/767] Better comments in permission_allowed_default() --- datasette/default_permissions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/default_permissions.py b/datasette/default_permissions.py index c13f2ed2..757b3a46 100644 --- a/datasette/default_permissions.py +++ b/datasette/default_permissions.py @@ -144,7 +144,7 @@ def permission_allowed_default(datasette, actor, action, resource): if actor and actor.get("id") == "root": return True - # Resolve metadata view permissions + # Resolve view permissions in allow blocks in configuration if action in ( "view-instance", "view-database", @@ -158,7 +158,7 @@ def permission_allowed_default(datasette, actor, action, resource): if result is not None: return result - # Check custom permissions: blocks + # Resolve custom permissions: blocks in configuration result = await _resolve_config_permissions_blocks( datasette, actor, action, resource ) From 232a30459babebece653795d136fb6516444ecf0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 12:56:39 -0800 Subject: [PATCH 179/767] DATASETTE_TRACE_PLUGINS setting, closes #2274 --- datasette/plugins.py | 24 ++++++++++++++++++++++++ docs/writing_plugins.rst | 24 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/datasette/plugins.py b/datasette/plugins.py index f7a1905f..3769a209 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -1,6 +1,7 @@ import importlib import os import pluggy +from pprint import pprint import sys from . import hookspecs @@ -33,6 +34,29 @@ DEFAULT_PLUGINS = ( pm = pluggy.PluginManager("datasette") pm.add_hookspecs(hookspecs) +DATASETTE_TRACE_PLUGINS = os.environ.get("DATASETTE_TRACE_PLUGINS", None) + + +def before(hook_name, hook_impls, kwargs): + print(file=sys.stderr) + print(f"{hook_name}:", file=sys.stderr) + pprint(kwargs, width=40, indent=4, stream=sys.stderr) + print("Hook implementations:", file=sys.stderr) + pprint(hook_impls, width=40, indent=4, stream=sys.stderr) + + +def after(outcome, hook_name, hook_impls, kwargs): + results = outcome.get_result() + if not isinstance(results, list): + results = [results] + print(f"Results:", file=sys.stderr) + pprint(results, width=40, indent=4, stream=sys.stderr) + + +if DATASETTE_TRACE_PLUGINS: + pm.add_hookcall_monitoring(before, after) + + DATASETTE_LOAD_PLUGINS = os.environ.get("DATASETTE_LOAD_PLUGINS", None) if not hasattr(sys, "_called_from_test") and DATASETTE_LOAD_PLUGINS is None: diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 5c8bc4c6..2bc6bd24 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -7,6 +7,30 @@ You can write one-off plugins that apply to just one Datasette instance, or you Want to start by looking at an example? The `Datasette plugins directory `__ lists more than 90 open source plugins with code you can explore. The :ref:`plugin hooks ` page includes links to example plugins for each of the documented hooks. +.. _writing_plugins_tracing: + +Tracing plugin hooks +-------------------- + +The ``DATASETTE_TRACE_PLUGINS`` environment variable turns on detailed tracing showing exactly which hooks are being run. This can be useful for understanding how Datasette is using your plugin. + +.. code-block:: bash + + DATASETTE_TRACE_PLUGINS=1 datasette mydb.db + +Example output:: + + actor_from_request: + { 'datasette': , + 'request': } + Hook implementations: + [ >, + >, + >] + Results: + [{'id': 'root'}] + + .. _writing_plugins_one_off: Writing one-off plugins From 8bfa3a51c222d653f45fb48ebcb6957a85f9ea6c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 13:29:39 -0800 Subject: [PATCH 180/767] Consider every plugins opinion in datasette.permission_allowed() Closes #2275, refs #2262 --- datasette/app.py | 14 +++++++++++++- docs/authentication.rst | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index d943b97b..8591af6a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -903,6 +903,8 @@ class Datasette: # Use default from registered permission, if available if default is DEFAULT_NOT_SET and action in self.permissions: default = self.permissions[action].default + opinions = [] + # Every plugin is consulted for their opinion for check in pm.hook.permission_allowed( datasette=self, actor=actor, @@ -911,9 +913,19 @@ class Datasette: ): check = await await_me_maybe(check) if check is not None: - result = check + opinions.append(check) + + result = None + # If any plugin said False it's false - the veto rule + if any(not r for r in opinions): + result = False + elif any(r for r in opinions): + # Otherwise, if any plugin said True it's true + result = True + used_default = False if result is None: + # No plugin expressed an opinion, so use the default result = default used_default = True self._permission_checks.append( diff --git a/docs/authentication.rst b/docs/authentication.rst index 87ee6385..a8dc5637 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -71,6 +71,23 @@ Datasette's built-in view permissions (``view-database``, ``view-table`` etc) de Permissions with potentially harmful effects should default to *deny*. Plugin authors should account for this when designing new plugins - for example, the `datasette-upload-csvs `__ plugin defaults to deny so that installations don't accidentally allow unauthenticated users to create new tables by uploading a CSV file. +.. _authentication_permissions_explained: + +How permissions are resolved +---------------------------- + +The :ref:`datasette.permission_allowed(actor, action, resource=None, default=...)` method is called to check if an actor is allowed to perform a specific action. + +This method asks every plugin that implements the :ref:`plugin_hook_permission_allowed` hook if the actor is allowed to perform the action. + +Each plugin can return ``True`` to indicate that the actor is allowed to perform the action, ``False`` if they are not allowed and ``None`` if the plugin has no opinion on the matter. + +``False`` acts as a veto - if any plugin returns ``False`` then the permission check is denied. Otherwise, if any plugin returns ``True`` then the permission check is allowed. + +The ``resource`` argument can be used to specify a specific resource that the action is being performed against. Some permissions, such as ``view-instance``, do not involve a resource. Others such as ``view-database`` have a resource that is a string naming the database. Permissions that take both a database name and the name of a table, view or canned query within that database use a resource that is a tuple of two strings, ``(database_name, resource_name)``. + +Plugins that implement the ``permission_allowed()`` hook can decide if they are going to consider the provided resource or not. + .. _authentication_permissions_allow: Defining permissions with "allow" blocks From 244f3ff83aac19e96fab85a95ddde349079a9827 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 13:39:57 -0800 Subject: [PATCH 181/767] Test demonstrating fix for permisisons bug in #2262 --- tests/test_api_write.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 2d127e1a..2aea699b 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -365,6 +365,41 @@ async def test_insert_or_upsert_row_errors( assert before_count == after_count +@pytest.mark.asyncio +@pytest.mark.parametrize("allowed", (True, False)) +async def test_upsert_permissions_per_table(ds_write, allowed): + # https://github.com/simonw/datasette/issues/2262 + token = "dstok_{}".format( + ds_write.sign( + { + "a": "root", + "token": "dstok", + "t": int(time.time()), + "_r": { + "r": { + "data": { + "docs" if allowed else "other": ["ir", "ur"], + } + } + }, + }, + namespace="token", + ) + ) + response = await ds_write.client.post( + "/data/docs/-/upsert", + json={"rows": [{"id": 1, "title": "One"}]}, + headers={ + "Authorization": "Bearer {}".format(token), + }, + ) + if allowed: + assert response.status_code == 200 + assert response.json()["ok"] is True + else: + assert response.status_code == 403 + + @pytest.mark.asyncio @pytest.mark.parametrize( "ignore,replace,expected_rows", From 3a999a85fb431594ccee1adf38721de03de19500 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 13:58:33 -0800 Subject: [PATCH 182/767] Fire insert-rows on /db/-/create if rows were inserted, refs #2260 --- datasette/views/database.py | 13 ++++++- tests/test_api_write.py | 71 +++++++++++++++++++++++++++++-------- 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index bd55064f..2a8b40cc 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -10,7 +10,7 @@ import re import sqlite_utils import textwrap -from datasette.events import AlterTableEvent, CreateTableEvent +from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted from datasette.utils import ( add_cors_headers, @@ -1022,6 +1022,17 @@ class TableCreateView(BaseView): request.actor, database=db.name, table=table_name, schema=schema ) ) + if rows: + await self.ds.track_event( + InsertRowsEvent( + request.actor, + database=db.name, + table=table_name, + num_rows=len(rows), + ignore=ignore, + replace=replace, + ) + ) return Response.json(details, status=201) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 2aea699b..0eb915ba 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -857,13 +857,14 @@ async def test_drop_table(ds_write, scenario): @pytest.mark.asyncio @pytest.mark.parametrize( - "input,expected_status,expected_response", + "input,expected_status,expected_response,expected_events", ( # Permission error with a bad token ( {"table": "bad", "row": {"id": 1}}, 403, {"ok": False, "errors": ["Permission denied"]}, + [], ), # Successful creation with columns: ( @@ -910,6 +911,7 @@ async def test_drop_table(ds_write, scenario): ")" ), }, + ["create-table"], ), # Successful creation with rows: ( @@ -945,6 +947,7 @@ async def test_drop_table(ds_write, scenario): ), "row_count": 2, }, + ["create-table", "insert-rows"], ), # Successful creation with row: ( @@ -973,6 +976,7 @@ async def test_drop_table(ds_write, scenario): ), "row_count": 1, }, + ["create-table", "insert-rows"], ), # Create with row and no primary key ( @@ -992,6 +996,7 @@ async def test_drop_table(ds_write, scenario): "schema": ("CREATE TABLE [four] (\n" " [name] TEXT\n" ")"), "row_count": 1, }, + ["create-table", "insert-rows"], ), # Create table with compound primary key ( @@ -1013,6 +1018,7 @@ async def test_drop_table(ds_write, scenario): ), "row_count": 1, }, + ["create-table", "insert-rows"], ), # Error: Table is required ( @@ -1024,6 +1030,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Table is required"], }, + [], ), # Error: Invalid table name ( @@ -1036,6 +1043,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Invalid table name"], }, + [], ), # Error: JSON must be an object ( @@ -1045,6 +1053,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["JSON must be an object"], }, + [], ), # Error: Cannot specify columns with rows or row ( @@ -1058,6 +1067,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Cannot specify columns with rows or row"], }, + [], ), # Error: columns, rows or row is required ( @@ -1069,6 +1079,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["columns, rows or row is required"], }, + [], ), # Error: columns must be a list ( @@ -1081,6 +1092,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["columns must be a list"], }, + [], ), # Error: columns must be a list of objects ( @@ -1093,6 +1105,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["columns must be a list of objects"], }, + [], ), # Error: Column name is required ( @@ -1105,6 +1118,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Column name is required"], }, + [], ), # Error: Unsupported column type ( @@ -1117,6 +1131,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Unsupported column type: bad"], }, + [], ), # Error: Duplicate column name ( @@ -1132,6 +1147,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Duplicate column name: id"], }, + [], ), # Error: rows must be a list ( @@ -1144,6 +1160,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["rows must be a list"], }, + [], ), # Error: rows must be a list of objects ( @@ -1156,6 +1173,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["rows must be a list of objects"], }, + [], ), # Error: pk must be a string ( @@ -1169,6 +1187,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["pk must be a string"], }, + [], ), # Error: Cannot specify both pk and pks ( @@ -1183,6 +1202,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["Cannot specify both pk and pks"], }, + [], ), # Error: pks must be a list ( @@ -1196,12 +1216,14 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["pks must be a list"], }, + [], ), # Error: pks must be a list of strings ( {"table": "bad", "row": {"id": 1, "name": "Row 1"}, "pks": [1, 2]}, 400, {"ok": False, "errors": ["pks must be a list of strings"]}, + [], ), # Error: ignore and replace are mutually exclusive ( @@ -1217,6 +1239,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace are mutually exclusive"], }, + [], ), # ignore and replace require row or rows ( @@ -1230,6 +1253,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace require row or rows"], }, + [], ), # ignore and replace require pk or pks ( @@ -1243,6 +1267,7 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace require pk or pks"], }, + [], ), ( { @@ -1255,10 +1280,14 @@ async def test_drop_table(ds_write, scenario): "ok": False, "errors": ["ignore and replace require pk or pks"], }, + [], ), ), ) -async def test_create_table(ds_write, input, expected_status, expected_response): +async def test_create_table( + ds_write, input, expected_status, expected_response, expected_events +): + ds_write._tracked_events = [] # Special case for expected status of 403 if expected_status == 403: token = "bad_token" @@ -1272,12 +1301,9 @@ async def test_create_table(ds_write, input, expected_status, expected_response) assert response.status_code == expected_status data = response.json() assert data == expected_response - # create-table event - if expected_status == 201: - event = last_event(ds_write) - assert event.name == "create-table" - assert event.actor == {"id": "root", "token": "dstok"} - assert event.schema.startswith("CREATE TABLE ") + # Should have tracked the expected events + events = ds_write._tracked_events + assert [e.name for e in events] == expected_events @pytest.mark.asyncio @@ -1376,6 +1402,8 @@ async def test_create_table_ignore_replace(ds_write, input, expected_rows_after) ) assert first_response.status_code == 201 + ds_write._tracked_events = [] + # Try a second time second_response = await ds_write.client.post( "/data/-/create", @@ -1387,6 +1415,10 @@ async def test_create_table_ignore_replace(ds_write, input, expected_rows_after) rows = await ds_write.client.get("/data/test_insert_replace.json?_shape=array") assert rows.json() == expected_rows_after + # Check it fired the right events + event_names = [e.name for e in ds_write._tracked_events] + assert event_names == ["insert-rows"] + @pytest.mark.asyncio async def test_create_table_error_if_pk_changed(ds_write): @@ -1471,6 +1503,7 @@ async def test_method_not_allowed(ds_write, path): @pytest.mark.asyncio async def test_create_uses_alter_by_default_for_new_table(ds_write): + ds_write._tracked_events = [] token = write_token(ds_write) response = await ds_write.client.post( "/data/-/create", @@ -1490,8 +1523,8 @@ async def test_create_uses_alter_by_default_for_new_table(ds_write): headers=_headers(token), ) assert response.status_code == 201 - event = last_event(ds_write) - assert event.name == "create-table" + event_names = [e.name for e in ds_write._tracked_events] + assert event_names == ["create-table", "insert-rows"] @pytest.mark.asyncio @@ -1517,6 +1550,8 @@ async def test_create_using_alter_against_existing_table( headers=_headers(token), ) assert response.status_code == 201 + + ds_write._tracked_events = [] # Now try to insert more rows using /-/create with alter=True response2 = await ds_write.client.post( "/data/-/create", @@ -1536,8 +1571,16 @@ async def test_create_using_alter_against_existing_table( } else: assert response2.status_code == 201 + + event_names = [e.name for e in ds_write._tracked_events] + assert event_names == ["alter-table", "insert-rows"] + # It should have altered the table - event = last_event(ds_write) - assert event.name == "alter-table" - assert "extra" not in event.before_schema - assert "extra" in event.after_schema + alter_event = ds_write._tracked_events[0] + assert alter_event.name == "alter-table" + assert "extra" not in alter_event.before_schema + assert "extra" in alter_event.after_schema + + insert_rows_event = ds_write._tracked_events[1] + assert insert_rows_event.name == "insert-rows" + assert insert_rows_event.num_rows == 1 From 9906f937d92c79dcc457cb057d7222ed70aef0e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 14:32:47 -0800 Subject: [PATCH 183/767] Release 1.0a9 Refs #2101, #2260, #2262, #2265, #2270, #2273, #2274, #2275 Closes #2276 --- datasette/version.py | 2 +- docs/changelog.rst | 45 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index e43b9918..f5e07ac8 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a8" +__version__ = "1.0a9" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index d164f71d..e567f422 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,51 @@ Changelog ========= +.. _v1_0_a9: + +1.0a9 (2024-02-16) +------------------ + +This alpha release adds basic alter table support to the Datasette Write API and fixes a permissions bug relating to the ``/upsert`` API endpoint. + +Alter table support for create, insert, upsert and update +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`JSON write API ` can now be used to apply simple alter table schema changes, provided the acting actor has the new :ref:`permissions_alter_table` permission. (:issue:`2101`) + +The only alter operation supported so far is adding new columns to an existing table. + +* The :ref:`/db/-/create ` API now adds new columns during large operations to create a table based on incoming example ``"rows"``, in the case where one of the later rows includes columns that were not present in the earlier batches. This requires the ``create-table`` but not the ``alter-table`` permission. +* When ``/db/-/create`` is called with rows in a situation where the table may have been already created, an ``"alter": true`` key can be included to indicate that any missing columns from the new rows should be added to the table. This requires the ``alter-table`` permission. +* :ref:`/db/table/-/insert ` and :ref:`/db/table/-/upsert ` and :ref:`/db/table/row-pks/-/update ` all now also accept ``"alter": true``, depending on the ``alter-table`` permission. + +Operations that alter a table now fire the new :ref:`alter-table event `. + +Permissions fix for the upsert API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`/database/table/-/upsert API ` had a minor permissions bug, only affecting Datasette instances that had configured the ``insert-row`` and ``update-row`` permissions to apply to a specific table rather than the database or instance as a whole. Full details in issue :issue:`2262`. + +To avoid similar mistakes in the future the :ref:`datasette.permission_allowed() ` method now specifies ``default=`` as a keyword-only argument. + +Permission checks now consider opinions from every plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`datasette.permission_allowed() ` method previously consulted every plugin that implemented the :ref:`permission_allowed() ` plugin hook and obeyed the opinion of the last plugin to return a value. (:issue:`2275`) + +Datasette now consults every plugin and checks to see if any of them returned ``False`` (the veto rule), and if none of them did, it then checks to see if any of them returned ``True``. + +This is explained at length in the new documentation covering :ref:`authentication_permissions_explained`. + +Other changes +~~~~~~~~~~~~~ + +- The new :ref:`DATASETTE_TRACE_PLUGINS=1 environment variable ` turns on detailed trace output for every executed plugin hook, useful for debugging and understanding how the plugin system works at a low level. (:issue:`2274`) +- Datasette on Python 3.9 or above marks its non-cryptographic uses of the MD5 hash function as ``usedforsecurity=False``, for compatibility with FIPS systems. (:issue:`2270`) +- SQL relating to :ref:`internals_internal` now executes inside a transaction, avoiding a potential database locked error. (:issue:`2273`) +- The ``/-/threads`` debug page now identifies the database in the name associated with each dedicated write thread. (:issue:`2265`) +- The ``/db/-/create`` API now fires a ``insert-rows`` event if rows were inserted after the table was created. (:issue:`2260`) + .. _v1_0_a8: 1.0a8 (2024-02-07) From e1c80efff8f4b0a53619546bb03e6dfd6cb42a32 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Feb 2024 14:43:36 -0800 Subject: [PATCH 184/767] Note about activating alpha documentation versions on ReadTheDocs --- docs/contributing.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index ef022a4d..b678e637 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -254,6 +254,7 @@ Datasette releases are performed using tags. When a new release is published on * Re-point the "latest" tag on Docker Hub to the new image * Build a wheel bundle of the underlying Python source code * Push that new wheel up to PyPI: https://pypi.org/project/datasette/ +* If the release is an alpha, navigate to https://readthedocs.org/projects/datasette/versions/ and search for the tag name in the "Activate a version" filter, then mark that version as "active" to ensure it will appear on the public ReadTheDocs documentation site. To deploy new releases you will need to have push access to the main Datasette GitHub repository. From 5e0e440f2c8a0771b761b02801456e55e95e2a04 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 20:28:15 -0800 Subject: [PATCH 185/767] database.execute_write_fn(transaction=True) parameter, closes #2277 --- datasette/database.py | 31 +++++++++++++++++++++++-------- docs/internals.rst | 15 ++++++++++----- tests/test_internals_database.py | 27 +++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 707d8f85..d34aac73 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -179,17 +179,25 @@ class Database: # Threaded mode - send to write thread return await self._send_to_write_thread(fn, isolated_connection=True) - async def execute_write_fn(self, fn, block=True): + async def execute_write_fn(self, fn, block=True, transaction=True): if self.ds.executor is None: # non-threaded mode if self._write_connection is None: self._write_connection = self.connect(write=True) self.ds._prepare_connection(self._write_connection, self.name) - return fn(self._write_connection) + if transaction: + with self._write_connection: + return fn(self._write_connection) + else: + return fn(self._write_connection) else: - return await self._send_to_write_thread(fn, block) + return await self._send_to_write_thread( + fn, block=block, transaction=transaction + ) - async def _send_to_write_thread(self, fn, block=True, isolated_connection=False): + async def _send_to_write_thread( + self, fn, block=True, isolated_connection=False, transaction=True + ): if self._write_queue is None: self._write_queue = queue.Queue() if self._write_thread is None: @@ -202,7 +210,9 @@ class Database: self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") reply_queue = janus.Queue() - self._write_queue.put(WriteTask(fn, task_id, reply_queue, isolated_connection)) + self._write_queue.put( + WriteTask(fn, task_id, reply_queue, isolated_connection, transaction) + ) if block: result = await reply_queue.async_q.get() if isinstance(result, Exception): @@ -244,7 +254,11 @@ class Database: pass else: try: - result = task.fn(conn) + if task.transaction: + with conn: + result = task.fn(conn) + else: + result = task.fn(conn) except Exception as e: sys.stderr.write("{}\n".format(e)) sys.stderr.flush() @@ -554,13 +568,14 @@ class Database: class WriteTask: - __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection") + __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection", "transaction") - def __init__(self, fn, task_id, reply_queue, isolated_connection): + def __init__(self, fn, task_id, reply_queue, isolated_connection, transaction): self.fn = fn self.task_id = task_id self.reply_queue = reply_queue self.isolated_connection = isolated_connection + self.transaction = transaction class QueryInterrupted(Exception): diff --git a/docs/internals.rst b/docs/internals.rst index bd7a70b5..6ca62423 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1010,7 +1010,9 @@ You can pass additional SQL parameters as a tuple or dictionary. The method will block until the operation is completed, and the return value will be the return from calling ``conn.execute(...)`` using the underlying ``sqlite3`` Python library. -If you pass ``block=False`` this behaviour changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. +If you pass ``block=False`` this behavior changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task. + +Each call to ``execute_write()`` will be executed inside a transaction. .. _database_execute_write_script: @@ -1019,6 +1021,8 @@ await db.execute_write_script(sql, block=True) Like ``execute_write()`` but can be used to send multiple SQL statements in a single string separated by semicolons, using the ``sqlite3`` `conn.executescript() `__ method. +Each call to ``execute_write_script()`` will be executed inside a transaction. + .. _database_execute_write_many: await db.execute_write_many(sql, params_seq, block=True) @@ -1033,10 +1037,12 @@ Like ``execute_write()`` but uses the ``sqlite3`` `conn.executemany() 5") - conn.commit() return conn.execute( "select count(*) from some_table" ).fetchone()[0] @@ -1069,7 +1074,7 @@ The value returned from ``await database.execute_write_fn(...)`` will be the ret If your function raises an exception that exception will be propagated up to the ``await`` line. -If you see ``OperationalError: database table is locked`` errors you should check that you remembered to explicitly call ``conn.commit()`` in your write function. +By default your function will be executed inside a transaction. You can pass ``transaction=False`` to disable this behavior, though if you do that you should be careful to manually apply transactions - ideally using the ``with conn:`` pattern, or you may see ``OperationalError: database table is locked`` errors. If you specify ``block=False`` the method becomes fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. Any exceptions in your code will be silently swallowed. diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index dd68a6cb..57e75046 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -66,6 +66,33 @@ async def test_execute_fn(db): assert 2 == await db.execute_fn(get_1_plus_1) +@pytest.mark.asyncio +async def test_execute_fn_transaction_false(): + datasette = Datasette(memory=True) + db = datasette.add_memory_database("test_execute_fn_transaction_false") + + def run(conn): + try: + with conn: + conn.execute("create table foo (id integer primary key)") + conn.execute("insert into foo (id) values (44)") + # Table should exist + assert ( + conn.execute( + 'select count(*) from sqlite_master where name = "foo"' + ).fetchone()[0] + == 1 + ) + assert conn.execute("select id from foo").fetchall()[0][0] == 44 + raise ValueError("Cancel commit") + except ValueError: + pass + # Row should NOT exist + assert conn.execute("select count(*) from foo").fetchone()[0] == 0 + + await db.execute_write_fn(run, transaction=False) + + @pytest.mark.parametrize( "tables,exists", ( From 10f9ba1a0050724ba47a089861606bef58a4087f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 20:51:19 -0800 Subject: [PATCH 186/767] Take advantage of execute_write_fn(transaction=True) A bunch of places no longer need to do manual transaction handling thanks to this change. Refs #2277 --- datasette/database.py | 9 +++------ datasette/utils/internal_db.py | 27 +++++++++++++-------------- tests/test_internals_database.py | 10 ++++------ 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index d34aac73..4e590d3a 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -123,8 +123,7 @@ class Database: async def execute_write(self, sql, params=None, block=True): def _inner(conn): - with conn: - return conn.execute(sql, params or []) + return conn.execute(sql, params or []) with trace("sql", database=self.name, sql=sql.strip(), params=params): results = await self.execute_write_fn(_inner, block=block) @@ -132,8 +131,7 @@ class Database: async def execute_write_script(self, sql, block=True): def _inner(conn): - with conn: - return conn.executescript(sql) + return conn.executescript(sql) with trace("sql", database=self.name, sql=sql.strip(), executescript=True): results = await self.execute_write_fn(_inner, block=block) @@ -149,8 +147,7 @@ class Database: count += 1 yield param - with conn: - return conn.executemany(sql, count_params(params_seq)), count + return conn.executemany(sql, count_params(params_seq)), count with trace( "sql", database=self.name, sql=sql.strip(), executemany=True diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index dd0d3a9d..dbfcceb4 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -69,20 +69,19 @@ async def populate_schema_tables(internal_db, db): database_name = db.name def delete_everything(conn): - with conn: - conn.execute( - "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] - ) - conn.execute( - "DELETE FROM catalog_foreign_keys WHERE database_name = ?", - [database_name], - ) - conn.execute( - "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] - ) + conn.execute( + "DELETE FROM catalog_tables WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_columns WHERE database_name = ?", [database_name] + ) + conn.execute( + "DELETE FROM catalog_foreign_keys WHERE database_name = ?", + [database_name], + ) + conn.execute( + "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name] + ) await internal_db.execute_write_fn(delete_everything) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 57e75046..1c155cf3 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -501,9 +501,8 @@ async def test_execute_write_has_correctly_prepared_connection(db): @pytest.mark.asyncio async def test_execute_write_fn_block_false(db): def write_fn(conn): - with conn: - conn.execute("delete from roadside_attractions where pk = 1;") - row = conn.execute("select count(*) from roadside_attractions").fetchone() + conn.execute("delete from roadside_attractions where pk = 1;") + row = conn.execute("select count(*) from roadside_attractions").fetchone() return row[0] task_id = await db.execute_write_fn(write_fn, block=False) @@ -513,9 +512,8 @@ async def test_execute_write_fn_block_false(db): @pytest.mark.asyncio async def test_execute_write_fn_block_true(db): def write_fn(conn): - with conn: - conn.execute("delete from roadside_attractions where pk = 1;") - row = conn.execute("select count(*) from roadside_attractions").fetchone() + conn.execute("delete from roadside_attractions where pk = 1;") + row = conn.execute("select count(*) from roadside_attractions").fetchone() return row[0] new_count = await db.execute_write_fn(write_fn) From a4fa1ef3bd6a6117118b5cdd64aca2308c21604b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 20:56:15 -0800 Subject: [PATCH 187/767] Release 1.0a10 Refs #2277 --- datasette/version.py | 2 +- docs/changelog.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index f5e07ac8..809c434f 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a9" +__version__ = "1.0a10" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index e567f422..92f198af 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v1_0_a10: + +1.0a10 (2024-02-17) +------------------- + +The only changes in this alpha correspond to the way Datasette handles database transactions. (:issue:`2277`) + +- The :ref:`database.execute_write_fn() ` method has a new ``transaction=True`` parameter. This defaults to ``True`` which means all functions executed using this method are now automatically wrapped in a transaction - previously the functions needed to roll transaction handling on their own, and many did not. +- Pass ``transaction=False`` to ``execute_write_fn()`` if you want to manually handle transactions in your function. +- Several internal Datasette features, including parts of the :ref:`JSON write API `, had been failing to wrap their operations in a transaction. This has been fixed by the new ``transaction=True`` default. + .. _v1_0_a9: 1.0a9 (2024-02-16) From 81629dbeffb5cee9086bc956ce3a9ab7d051f4d1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 17 Feb 2024 21:03:41 -0800 Subject: [PATCH 188/767] Upgrade GitHub Actions, including PyPI publishing --- .github/workflows/publish.yml | 60 ++++++++++++++--------------------- .github/workflows/test.yml | 13 +++----- 2 files changed, 27 insertions(+), 46 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 64a03a77..55fc0eb9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,20 +12,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | pip install -e '.[test]' @@ -36,47 +31,38 @@ jobs: deploy: runs-on: ubuntu-latest needs: [test] + environment: release + permissions: + id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' - - uses: actions/cache@v3 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-publish-pip- + python-version: '3.12' + cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | - pip install setuptools wheel twine - - name: Publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + pip install setuptools wheel build + - name: Build run: | - python setup.py sdist bdist_wheel - twine upload dist/* + python -m build + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 deploy_static_docs: runs-on: ubuntu-latest needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.9' - - uses: actions/cache@v2 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-publish-pip- + cache: pip + cache-dependency-path: setup.py - name: Install dependencies run: | python -m pip install -e .[docs] @@ -105,7 +91,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 656b0b1c..3ac8756d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,19 +12,14 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: true - - uses: actions/cache@v3 - name: Configure pip caching - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: pip + cache-dependency-path: setup.py - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) From 3856a8cb244f1338d2c4bceb76a510022d88ade5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 12:51:14 -0800 Subject: [PATCH 189/767] Consistent Permission denied:, refs #2279 --- datasette/views/database.py | 6 +++--- tests/test_api_write.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 2a8b40cc..56fc6f8c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -860,7 +860,7 @@ class TableCreateView(BaseView): if not await self.ds.permission_allowed( request.actor, "update-row", resource=database_name ): - return _error(["Permission denied - need update-row"], 403) + return _error(["Permission denied: need update-row"], 403) table_name = data.get("table") if not table_name: @@ -884,7 +884,7 @@ class TableCreateView(BaseView): if not await self.ds.permission_allowed( request.actor, "insert-row", resource=database_name ): - return _error(["Permission denied - need insert-row"], 403) + return _error(["Permission denied: need insert-row"], 403) alter = False if rows or row: @@ -897,7 +897,7 @@ class TableCreateView(BaseView): if not await self.ds.permission_allowed( request.actor, "alter-table", resource=database_name ): - return _error(["Permission denied - need alter-table"], 403) + return _error(["Permission denied: need alter-table"], 403) alter = True if columns: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 0eb915ba..634f5ee9 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1316,7 +1316,7 @@ async def test_create_table( ["create-table"], {"table": "t", "rows": [{"name": "c"}]}, 403, - ["Permission denied - need insert-row"], + ["Permission denied: need insert-row"], ), # This should work: ( @@ -1330,7 +1330,7 @@ async def test_create_table( ["create-table", "insert-row"], {"table": "t", "rows": [{"id": 1}], "pk": "id", "replace": True}, 403, - ["Permission denied - need update-row"], + ["Permission denied: need update-row"], ), ), ) @@ -1567,7 +1567,7 @@ async def test_create_using_alter_against_existing_table( assert response2.status_code == 403 assert response2.json() == { "ok": False, - "errors": ["Permission denied - need alter-table"], + "errors": ["Permission denied: need alter-table"], } else: assert response2.status_code == 201 From b36a2d8f4b566a3a4902cdaa7a549241e0e8c881 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 12:55:51 -0800 Subject: [PATCH 190/767] Require update-row to use insert replace, closes #2279 --- datasette/views/table.py | 5 +++++ docs/json_api.rst | 4 ++-- tests/test_api_write.py | 8 ++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 1c187692..6d0d9885 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -485,6 +485,11 @@ class TableInsertView(BaseView): if upsert and (ignore or replace): return _error(["Upsert does not support ignore or replace"], 400) + if replace and not await self.ds.permission_allowed( + request.actor, "update-row", resource=(database_name, table_name) + ): + return _error(['Permission denied: need update-row to use "replace"'], 403) + initial_schema = None if alter: # Must have alter-table permission diff --git a/docs/json_api.rst b/docs/json_api.rst index c401d97e..366f74b2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -616,7 +616,7 @@ Pass ``"ignore": true`` to ignore these errors and insert the other rows: "ignore": true } -Or you can pass ``"replace": true`` to replace any rows with conflicting primary keys with the new values. +Or you can pass ``"replace": true`` to replace any rows with conflicting primary keys with the new values. This requires the :ref:`permissions_update_row` permission. Pass ``"alter: true`` to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. @@ -854,7 +854,7 @@ The JSON here describes the table that will be created: * ``pks`` can be used instead of ``pk`` to create a compound primary key. It should be a JSON list of column names to use in that primary key. * ``ignore`` can be set to ``true`` to ignore existing rows by primary key if the table already exists. -* ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. +* ``replace`` can be set to ``true`` to replace existing rows by primary key if the table already exists. This requires the :ref:`permissions_update_row` permission. * ``alter`` can be set to ``true`` if you want to automatically add any missing columns to the table. This requires the :ref:`permissions_alter_table` permission. If the table is successfully created this will return a ``201`` status code and the following response: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 634f5ee9..6a7ddeb6 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -221,6 +221,14 @@ async def test_insert_rows(ds_write, return_rows): 400, ['Cannot use "ignore" and "replace" at the same time'], ), + ( + # Replace is not allowed if you don't have update-row + "/data/docs/-/insert", + {"rows": [{"title": "Test"}], "replace": True}, + "insert-but-not-update", + 403, + ['Permission denied: need update-row to use "replace"'], + ), ( "/data/docs/-/insert", {"rows": [{"title": "Test"}], "invalid_param": True}, From 392ca2e24cc93a3918d07718f40524857d626d14 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 13:40:48 -0800 Subject: [PATCH 191/767] Improvements to table column cog menu display, closes #2263 - Repositions if menu would cause a horizontal scrollbar - Arrow tip on menu now attempts to align with cog icon on column --- datasette/static/table.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/datasette/static/table.js b/datasette/static/table.js index 778457c5..0c54a472 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -217,6 +217,17 @@ const initDatasetteTable = function (manager) { menuList.appendChild(menuItem); }); + // Measure width of menu and adjust position if too far right + const menuWidth = menu.offsetWidth; + const windowWidth = window.innerWidth; + if (menuLeft + menuWidth > windowWidth) { + menu.style.left = windowWidth - menuWidth - 20 + "px"; + } + // Align menu .hook arrow with the column cog icon + const hook = menu.querySelector('.hook'); + const icon = th.querySelector('.dropdown-menu-icon'); + const iconRect = icon.getBoundingClientRect(); + hook.style.left = (iconRect.left - menuLeft + 1) + 'px'; } var svg = document.createElement("div"); From 27409a78929b4baa017cce2cc0ca636603ed6d37 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 14:01:55 -0800 Subject: [PATCH 192/767] Fix for hook position in wide column names, refs #2263 --- datasette/static/table.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/datasette/static/table.js b/datasette/static/table.js index 0c54a472..4f81b2e5 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -227,7 +227,15 @@ const initDatasetteTable = function (manager) { const hook = menu.querySelector('.hook'); const icon = th.querySelector('.dropdown-menu-icon'); const iconRect = icon.getBoundingClientRect(); - hook.style.left = (iconRect.left - menuLeft + 1) + 'px'; + const hookLeft = (iconRect.left - menuLeft + 1) + 'px'; + hook.style.left = hookLeft; + // Move the whole menu right if the hook is too far right + const menuRect = menu.getBoundingClientRect(); + if (iconRect.right > menuRect.right) { + menu.style.left = (iconRect.right - menuWidth) + 'px'; + // And move hook tip as well + hook.style.left = (menuWidth - 13) + 'px'; + } } var svg = document.createElement("div"); From 26300738e3c6e7ad515bd513063f57249a05000a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Feb 2024 14:17:37 -0800 Subject: [PATCH 193/767] Fixes for permissions debug page, closes #2278 --- datasette/templates/permissions_debug.html | 10 +++++----- datasette/views/special.py | 17 +++++++++-------- tests/test_permissions.py | 8 ++++++++ 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html index 36a12acc..5a5c1aa6 100644 --- a/datasette/templates/permissions_debug.html +++ b/datasette/templates/permissions_debug.html @@ -57,7 +57,7 @@ textarea {

@@ -71,19 +71,19 @@ textarea { +{% endif %} + {% endblock %} diff --git a/datasette/url_builder.py b/datasette/url_builder.py index 9c6bbde0..16b3d42b 100644 --- a/datasette/url_builder.py +++ b/datasette/url_builder.py @@ -31,6 +31,12 @@ class Urls: db = self.ds.get_database(database) return self.path(tilde_encode(db.route), format=format) + def database_query(self, database, sql, format=None): + path = f"{self.database(database)}/-/query?" + urllib.parse.urlencode( + {"sql": sql} + ) + return self.path(path, format=format) + def table(self, database, table, format=None): path = f"{self.database(database)}/{tilde_encode(table)}" if format is not None: diff --git a/datasette/views/table.py b/datasette/views/table.py index d71efeb0..ea044b36 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -929,6 +929,7 @@ async def table_view_traced(datasette, request): database=resolved.db.name, table=resolved.table, ), + count_limit=resolved.db.count_limit, ), request=request, view_name="table", @@ -1280,6 +1281,9 @@ async def table_view_data( if extra_extras: extras.update(extra_extras) + async def extra_count_sql(): + return count_sql + async def extra_count(): "Total count of rows matching these filters" # Calculate the total count for this query @@ -1299,8 +1303,11 @@ async def table_view_data( # Otherwise run a select count(*) ... if count_sql and count is None and not nocount: + count_sql_limited = ( + f"select count(*) from (select * {from_sql} limit 10001)" + ) try: - count_rows = list(await db.execute(count_sql, from_sql_params)) + count_rows = list(await db.execute(count_sql_limited, from_sql_params)) count = count_rows[0][0] except QueryInterrupted: pass @@ -1615,6 +1622,7 @@ async def table_view_data( "facet_results", "facets_timed_out", "count", + "count_sql", "human_description_en", "next_url", "metadata", @@ -1647,6 +1655,7 @@ async def table_view_data( registry = Registry( extra_count, + extra_count_sql, extra_facet_results, extra_facets_timed_out, extra_suggested_facets, From dc288056b81a3635bdb02a6d0121887db2720e5e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 21 Aug 2024 19:56:02 -0700 Subject: [PATCH 292/767] Better handling of errors for count all button, refs #2408 --- datasette/templates/table.html | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 187f0143..7246ff5d 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -42,7 +42,7 @@ {% if count or human_description_en %}

{% if count == count_limit + 1 %}>{{ "{:,}".format(count_limit) }} rows - {% if allow_execute_sql and query.sql %} count all rows{% endif %} + {% if allow_execute_sql and query.sql %} count all{% endif %} {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %} {% if human_description_en %}{{ human_description_en }}{% endif %}

@@ -180,7 +180,7 @@ document.addEventListener('DOMContentLoaded', function() { const countLink = document.querySelector('a.count-sql'); if (countLink) { - countLink.addEventListener('click', function(ev) { + countLink.addEventListener('click', async function(ev) { ev.preventDefault(); // Replace countLink with span with same style attribute const span = document.createElement('span'); @@ -189,14 +189,23 @@ document.addEventListener('DOMContentLoaded', function() { countLink.replaceWith(span); countLink.setAttribute('disabled', 'disabled'); let url = countLink.href.replace(/(\?|$)/, '.json$1'); - fetch(url) - .then(response => response.json()) - .then(data => { - const count = data['rows'][0]['count(*)']; - const formattedCount = count.toLocaleString(); - span.closest('h3').textContent = formattedCount + ' rows'; - }) - .catch(error => countLink.textContent = 'error'); + try { + const response = await fetch(url); + console.log({response}); + const data = await response.json(); + console.log({data}); + if (!response.ok) { + console.log('throw error'); + throw new Error(data.title || data.error); + } + const count = data['rows'][0]['count(*)']; + const formattedCount = count.toLocaleString(); + span.closest('h3').textContent = formattedCount + ' rows'; + } catch (error) { + console.log('Update', span, 'with error message', error); + span.textContent = error.message; + span.style.color = 'red'; + } }); } }); From 92c4d41ca605e0837a2711ee52fde9cf1eea74d0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 1 Sep 2024 17:20:41 -0700 Subject: [PATCH 293/767] results.dicts() method, closes #2414 --- datasette/database.py | 3 +++ datasette/views/row.py | 3 +-- datasette/views/table.py | 2 +- docs/internals.rst | 3 +++ tests/test_api_write.py | 23 +++++++++-------------- tests/test_internals_database.py | 11 +++++++++++ 6 files changed, 28 insertions(+), 17 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index da0ab1de..a2e899bc 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -677,6 +677,9 @@ class Results: else: raise MultipleValues + def dicts(self): + return [dict(row) for row in self.rows] + def __iter__(self): return iter(self.rows) diff --git a/datasette/views/row.py b/datasette/views/row.py index d802994e..f374fd94 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -277,8 +277,7 @@ class RowUpdateView(BaseView): results = await resolved.db.execute( resolved.sql, resolved.params, truncate=True ) - rows = list(results.rows) - result["row"] = dict(rows[0]) + result["row"] = results.dicts()[0] await self.ds.track_event( UpdateRowEvent( diff --git a/datasette/views/table.py b/datasette/views/table.py index ea044b36..82dab613 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -558,7 +558,7 @@ class TableInsertView(BaseView): ), args, ) - result["rows"] = [dict(r) for r in fetched_rows.rows] + result["rows"] = fetched_rows.dicts() else: result["rows"] = rows # We track the number of rows requested, but do not attempt to show which were actually diff --git a/docs/internals.rst b/docs/internals.rst index 4289c815..facbc224 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1093,6 +1093,9 @@ The ``Results`` object also has the following properties and methods: ``.rows`` - list of ``sqlite3.Row`` This property provides direct access to the list of rows returned by the database. You can access specific rows by index using ``results.rows[0]``. +``.dicts()`` - list of ``dict`` + This method returns a list of Python dictionaries, one for each row. + ``.first()`` - row or None Returns the first row in the results, or ``None`` if no rows were returned. diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 9c2b9b45..04e61261 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -58,8 +58,8 @@ async def test_insert_row(ds_write, content_type): assert response.status_code == 201 assert response.json()["ok"] is True assert response.json()["rows"] == [expected_row] - rows = (await ds_write.get_database("data").execute("select * from docs")).rows - assert dict(rows[0]) == expected_row + rows = (await ds_write.get_database("data").execute("select * from docs")).dicts() + assert rows[0] == expected_row # Analytics event event = last_event(ds_write) assert event.name == "insert-rows" @@ -118,12 +118,9 @@ async def test_insert_rows(ds_write, return_rows): assert not event.ignore assert not event.replace - actual_rows = [ - dict(r) - for r in ( - await ds_write.get_database("data").execute("select * from docs") - ).rows - ] + actual_rows = ( + await ds_write.get_database("data").execute("select * from docs") + ).dicts() assert len(actual_rows) == 20 assert actual_rows == [ {"id": i + 1, "title": "Test {}".format(i), "score": 1.0, "age": 5} @@ -469,12 +466,10 @@ async def test_insert_ignore_replace( assert event.ignore == ignore assert event.replace == replace - actual_rows = [ - dict(r) - for r in ( - await ds_write.get_database("data").execute("select * from docs") - ).rows - ] + actual_rows = ( + await ds_write.get_database("data").execute("select * from docs") + ).dicts() + assert actual_rows == expected_rows assert response.json()["ok"] is True if should_return: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 0020668a..edfc6bc7 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -40,6 +40,17 @@ async def test_results_bool(db, expected): assert bool(results) is expected +@pytest.mark.asyncio +async def test_results_dicts(db): + results = await db.execute("select pk, name from roadside_attractions") + assert results.dicts() == [ + {"pk": 1, "name": "The Mystery Spot"}, + {"pk": 2, "name": "Winchester Mystery House"}, + {"pk": 3, "name": "Burlingame Museum of PEZ Memorabilia"}, + {"pk": 4, "name": "Bigfoot Discovery Museum"}, + ] + + @pytest.mark.parametrize( "query,expected", [ From 2170269258d1de38f4e518aa3e55e6b3ed202841 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 3 Sep 2024 08:37:26 -0700 Subject: [PATCH 294/767] New .core CSS class for inputs and buttons * Initial .core input/button classes, refs #2415 * Docs for the new .core CSS class, refs #2415 * Applied .core class everywhere that needs it, closes #2415 --- datasette/static/app.css | 33 +++++++++++++++------- datasette/templates/allow_debug.html | 2 +- datasette/templates/api_explorer.html | 4 +-- datasette/templates/create_token.html | 2 +- datasette/templates/database.html | 2 +- datasette/templates/logout.html | 2 +- datasette/templates/messages_debug.html | 2 +- datasette/templates/permissions_debug.html | 2 +- datasette/templates/query.html | 2 +- datasette/templates/table.html | 4 +-- docs/custom_templates.rst | 9 ++++++ docs/writing_plugins.rst | 3 +- tests/test_permissions.py | 2 +- 13 files changed, 46 insertions(+), 23 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 562d6adb..f975f0ad 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -528,8 +528,11 @@ label.sort_by_desc { pre#sql-query { margin-bottom: 1em; } -form input[type=text], -form input[type=search] { + +.core input[type=text], +input.core[type=text], +.core input[type=search], +input.core[type=search] { border: 1px solid #ccc; border-radius: 3px; width: 60%; @@ -540,17 +543,25 @@ form input[type=search] { } /* Stop Webkit from styling search boxes in an inconsistent way */ /* https://css-tricks.com/webkit-html5-search-inputs/ comments */ -input[type=search] { +.core input[type=search], +input.core[type=search] { -webkit-appearance: textfield; } -input[type="search"]::-webkit-search-decoration, -input[type="search"]::-webkit-search-cancel-button, -input[type="search"]::-webkit-search-results-button, -input[type="search"]::-webkit-search-results-decoration { +.core input[type="search"]::-webkit-search-decoration, +input.core[type="search"]::-webkit-search-decoration, +.core input[type="search"]::-webkit-search-cancel-button, +input.core[type="search"]::-webkit-search-cancel-button, +.core input[type="search"]::-webkit-search-results-button, +input.core[type="search"]::-webkit-search-results-button, +.core input[type="search"]::-webkit-search-results-decoration, +input.core[type="search"]::-webkit-search-results-decoration { display: none; } -form input[type=submit], form button[type=button] { +.core input[type=submit], +.core button[type=button], +input.core[type=submit], +button.core[type=button] { font-weight: 400; cursor: pointer; text-align: center; @@ -563,14 +574,16 @@ form input[type=submit], form button[type=button] { border-radius: .25rem; } -form input[type=submit] { +.core input[type=submit], +input.core[type=submit] { color: #fff; background: linear-gradient(180deg, #007bff 0%, #4E79C7 100%); border-color: #007bff; -webkit-appearance: button; } -form button[type=button] { +.core button[type=button], +button.core[type=button] { color: #007bff; background-color: #fff; border-color: #007bff; diff --git a/datasette/templates/allow_debug.html b/datasette/templates/allow_debug.html index 04181531..610417d2 100644 --- a/datasette/templates/allow_debug.html +++ b/datasette/templates/allow_debug.html @@ -35,7 +35,7 @@ p.message-warning {

Use this tool to try out different actor and allow combinations. See Defining permissions with "allow" blocks for documentation.

- +

diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 109fb1e9..dc393c20 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -19,7 +19,7 @@

GET - +
@@ -29,7 +29,7 @@
POST - +
diff --git a/datasette/templates/create_token.html b/datasette/templates/create_token.html index 2be98d38..409fb8a9 100644 --- a/datasette/templates/create_token.html +++ b/datasette/templates/create_token.html @@ -39,7 +39,7 @@ {% endfor %} {% endif %} - +

diff --git a/datasette/templates/logout.html b/datasette/templates/logout.html index 4c4a7d11..c8fc642a 100644 --- a/datasette/templates/logout.html +++ b/datasette/templates/logout.html @@ -8,7 +8,7 @@

You are logged in as {{ display_actor(actor) }}

- +
diff --git a/datasette/templates/messages_debug.html b/datasette/templates/messages_debug.html index e0ab9a40..2940cd69 100644 --- a/datasette/templates/messages_debug.html +++ b/datasette/templates/messages_debug.html @@ -8,7 +8,7 @@

Set a message:

- +
diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html index 5a5c1aa6..83891181 100644 --- a/datasette/templates/permissions_debug.html +++ b/datasette/templates/permissions_debug.html @@ -47,7 +47,7 @@ textarea {

This tool lets you simulate an actor and a permission check for that actor.

- +

diff --git a/datasette/templates/query.html b/datasette/templates/query.html index f7c8d0a3..a6e9a3aa 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -36,7 +36,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 7246ff5d..c9e0e87b 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -48,7 +48,7 @@

{% endif %} - + {% if supports_search %}
{% endif %} @@ -152,7 +152,7 @@ object {% endif %}

- +

CSV options: diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 534d8b33..8cc40f0f 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -83,6 +83,15 @@ database column they are representing, for example: +.. _customization_css: + +Writing custom CSS +~~~~~~~~~~~~~~~~~~ + +Custom templates need to take Datasette's default CSS into account. The pattern portfolio at ``/-/patterns`` (`example here `__) is a useful reference for understanding the available CSS classes. + +The ``core`` class is particularly useful - you can apply this directly to a ```` or `` +

+

+ + + + +

+ + +{% if queries %} +
    + {% for query in queries %} +
  • + {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} + {% if query.is_write %}Writable{% endif %} + {% if query.is_published %}Published{% endif %} +
    • + {% endfor %} +
+{% else %} +

No queries found.

+{% endif %} + +{% if next_url %} +

Next page

+{% endif %} + +{% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index ed38189b..edbc315e 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -92,24 +92,14 @@ class DatabaseView(View): tables = await get_tables(datasette, request, db, allowed_dict) - # Get allowed queries using the new permission system - allowed_query_page = await datasette.allowed_resources( - "view-query", - request.actor, - parent=database, - include_is_private=True, - limit=1000, + queries_page = await datasette.list_queries( + database, + actor=request.actor, + limit=20, + include_private=True, ) - - # Build canned_queries list by looking up each allowed query - all_queries = await datasette.get_canned_queries(database, request.actor) - canned_queries = [] - for query_resource in allowed_query_page.resources: - query_name = query_resource.child - if query_name in all_queries: - canned_queries.append( - dict(all_queries[query_name], private=query_resource.private) - ) + canned_queries = queries_page["queries"] + queries_more = queries_page["has_more"] async def database_actions(): links = [] @@ -141,6 +131,7 @@ class DatabaseView(View): "hidden_count": len([t for t in tables if t["hidden"]]), "views": sql_views, "queries": canned_queries, + "queries_more": queries_more, "allow_execute_sql": allow_execute_sql, "table_columns": ( await _table_columns(datasette, database) if allow_execute_sql else {} @@ -174,6 +165,7 @@ class DatabaseView(View): hidden_count=len([t for t in tables if t["hidden"]]), views=sql_views, queries=canned_queries, + queries_more=queries_more, allow_execute_sql=allow_execute_sql, table_columns=( await _table_columns(datasette, database) @@ -222,6 +214,9 @@ class DatabaseContext(Context): hidden_count: int = field(metadata={"help": "Count of hidden tables"}) views: list = field(metadata={"help": "List of view objects in the database"}) queries: list = field(metadata={"help": "List of canned query objects"}) + queries_more: bool = field( + metadata={"help": "Boolean indicating if more saved queries are available"} + ) allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) @@ -474,6 +469,31 @@ def _as_bool(value): return bool(value) +def _as_optional_bool(value, name): + if value is None or value == "": + return None + if isinstance(value, bool): + return value + if isinstance(value, int): + return bool(value) + if isinstance(value, str): + lowered = value.lower() + if lowered in {"1", "true", "t", "yes", "on"}: + return True + if lowered in {"0", "false", "f", "no", "off"}: + return False + raise QueryValidationError("{} must be 0 or 1".format(name)) + + +def _query_list_limit(value): + if value in (None, ""): + return 50 + try: + return min(max(1, int(value)), 1000) + except ValueError as ex: + raise QueryValidationError("_size must be an integer") from ex + + def _derived_query_parameters(sql): parameters = [] seen = set() @@ -949,19 +969,66 @@ class QueryListView(BaseView): async def get(self, request): db = await self.ds.resolve_database(request) - page = await self.ds.allowed_resources( - "view-query", - request.actor, - parent=db.name, - limit=1000, + format_ = request.url_vars.get("format") or "html" + try: + limit = _query_list_limit(request.args.get("_size")) + is_write = _as_optional_bool(request.args.get("is_write"), "is_write") + is_published = _as_optional_bool( + request.args.get("is_published"), "is_published" + ) + except QueryValidationError as ex: + return _error([ex.message], ex.status) + + page = await self.ds.list_queries( + db.name, + actor=request.actor, + limit=limit, + cursor=request.args.get("_next"), + q=request.args.get("q") or None, + is_write=is_write, + is_published=is_published, + source=request.args.get("source") or None, + owner_id=request.args.get("owner_id") or None, + include_private=True, + ) + next_url = None + if page["next"]: + pairs = [ + (key, value) + for key, value in parse_qsl( + request.query_string, keep_blank_values=True + ) + if key != "_next" + ] + pairs.append(("_next", page["next"])) + next_url = "{}?{}".format( + self.ds.urls.database(db.name) + "/-/queries", + urlencode(pairs), + ) + + data = { + "ok": True, + "database": db.name, + "queries": page["queries"], + "next": page["next"], + "next_url": next_url, + "has_more": page["has_more"], + "limit": page["limit"], + "filters": { + "q": request.args.get("q") or "", + "is_write": request.args.get("is_write") or "", + "is_published": request.args.get("is_published") or "", + "source": request.args.get("source") or "", + "owner_id": request.args.get("owner_id") or "", + }, + } + if format_ == "json": + return Response.json(data) + return await self.render( + ["query_list.html"], + request, + data, ) - all_queries = await self.ds.get_queries(db.name) - queries = [ - all_queries[resource.child] - for resource in page.resources - if resource.child in all_queries - ] - return Response.json({"ok": True, "database": db.name, "queries": queries}) class QueryCreateView(BaseView): diff --git a/docs/json_api.rst b/docs/json_api.rst index e4c9e86e..ece430c2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -510,7 +510,7 @@ Datasette provides a write API for JSON data. This is a POST-only API that requi Listing saved queries ~~~~~~~~~~~~~~~~~~~~~ -``GET //-/queries`` returns saved query definitions the actor can view. +``GET //-/queries.json`` returns saved query definitions the actor can view. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. .. _QueryCreateView: diff --git a/queries-plan.md b/queries-plan.md index a58ace70..671fc29c 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -210,7 +210,7 @@ JSON endpoints should follow Datasette's existing write API style: use `POST` pl Endpoints: -- `GET /{database}/-/queries` lists query definitions the actor can view or manage, probably paginated. +- `GET /{database}/-/queries` shows a searchable HTML query browser. `GET /{database}/-/queries.json` returns query definitions the actor can view, using cursor pagination with `_next` and `_size`. - `POST /{database}/-/queries/-/insert` creates a query. - `GET /{database}/{query}/-/definition` returns one query definition without executing it. - `POST /{database}/{query}/-/update` updates one query. @@ -353,9 +353,21 @@ await datasette.update_query( await datasette.remove_query(database, name, source=None) await datasette.get_query(database, name) -await datasette.get_queries(database) +await datasette.list_queries( + database, + actor=None, + limit=50, + cursor=None, + q=None, + is_write=None, + is_published=None, + source=None, + owner_id=None, +) ``` +`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. + `update_query()` should use an internal sentinel default such as `UNCHANGED = object()` so callers can distinguish "leave this column alone" from "set this column to `NULL`": ```python @@ -380,6 +392,8 @@ The save form should call `POST /{database}/-/queries/-/insert` and default to ` If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. +On `/{database}`, show a preview of the first 20 visible queries using `list_queries(..., limit=20)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. + ## Dedicated create query UI Add `/{database}/-/queries/-/create` for the fuller query authoring flow, including writable queries. diff --git a/tests/test_queries.py b/tests/test_queries.py index df4131b9..dd906faf 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -7,6 +7,20 @@ from datasette.resources import DatabaseResource, QueryResource from datasette.utils.asgi import Forbidden +async def add_numbered_queries(ds, database, count): + for i in range(1, count + 1): + await ds.add_query( + database, + "demo_query_{:02d}".format(i), + "select {} as query_number".format(i), + title="Demo query {:02d}".format(i), + description="Seeded demo query number {:02d}".format(i), + is_published=True, + source="user", + owner_id="root", + ) + + @pytest.mark.asyncio async def test_queries_internal_table_schema(): ds = Datasette(memory=True) @@ -96,11 +110,15 @@ async def test_add_get_and_remove_query(): "on_error_redirect": None, } - assert await ds.get_queries("data") == {"top_customers": query} + queries_page = await ds.list_queries("data", actor=None) + assert queries_page["queries"] == [query] + assert queries_page["next"] is None await ds.remove_query("data", "top_customers") assert await ds.get_query("data", "top_customers") is None - assert await ds.get_queries("data") == {} + queries_page = await ds.list_queries("data", actor=None) + assert queries_page["queries"] == [] + assert queries_page["next"] is None @pytest.mark.asyncio @@ -238,6 +256,24 @@ async def test_unpublished_query_requires_execute_sql_but_published_does_not(): ) +@pytest.mark.asyncio +async def test_database_page_query_preview_is_limited(): + ds = Datasette(memory=True) + ds.add_memory_database("query_preview", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 25) + + html_response = await ds.client.get("/data") + json_response = await ds.client.get("/data.json") + + assert html_response.status_code == 200 + assert "Demo query 20" in html_response.text + assert "Demo query 21" not in html_response.text + assert 'href="/data/-/queries"' in html_response.text + assert len(json_response.json()["queries"]) == 20 + assert json_response.json()["queries_more"] is True + + @pytest.mark.asyncio async def test_query_actions_are_registered(): ds = Datasette() @@ -347,21 +383,78 @@ async def test_query_list_and_definition_api(): ds.root_enabled = True ds.add_memory_database("query_list_api", name="data") await ds.invoke_startup() - await ds.add_query("data", "listed", "select 1", title="Listed", is_published=True) + await add_numbered_queries(ds, "data", 12) list_response = await ds.client.get( - "/data/-/queries", + "/data/-/queries.json?_size=5", + actor={"id": "root"}, + ) + next_response = await ds.client.get( + "/data/-/queries.json?_size=5&_next={}".format(list_response.json()["next"]), actor={"id": "root"}, ) definition_response = await ds.client.get( - "/data/listed/-/definition", + "/data/demo_query_01/-/definition", actor={"id": "root"}, ) assert list_response.status_code == 200 - assert list_response.json()["queries"][0]["name"] == "listed" + assert [query["name"] for query in list_response.json()["queries"]] == [ + "demo_query_01", + "demo_query_02", + "demo_query_03", + "demo_query_04", + "demo_query_05", + ] + assert list_response.json()["next"] + assert [query["name"] for query in next_response.json()["queries"]] == [ + "demo_query_06", + "demo_query_07", + "demo_query_08", + "demo_query_09", + "demo_query_10", + ] assert definition_response.status_code == 200 - assert definition_response.json()["query"]["title"] == "Listed" + assert definition_response.json()["query"]["title"] == "Demo query 01" + + +@pytest.mark.asyncio +async def test_query_list_search_filter_and_html(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_html", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 3) + await ds.add_query( + "data", + "private_query", + "select 'private'", + title="Private query", + is_published=False, + source="user", + owner_id="root", + ) + + html_response = await ds.client.get( + "/data/-/queries?q=02", + actor={"id": "root"}, + ) + json_response = await ds.client.get( + "/data/-/queries.json?q=02", + actor={"id": "root"}, + ) + filtered_response = await ds.client.get( + "/data/-/queries.json?is_published=0", + actor={"id": "root"}, + ) + + assert html_response.status_code == 200 + assert "Demo query 02" in html_response.text + assert "Demo query 01" not in html_response.text + assert json_response.json()["queries"][0]["name"] == "demo_query_02" + assert [query["name"] for query in filtered_response.json()["queries"]] == [ + "private_query" + ] @pytest.mark.asyncio From 310c36ae94c54d4b859925d4977554c2a2618534 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 10:18:36 -0700 Subject: [PATCH 692/767] Limit database query preview to five Refs #2735 --- datasette/views/database.py | 2 +- queries-plan.md | 2 +- tests/test_canned_queries.py | 35 ++++++++++++++++++++++++++++++----- tests/test_queries.py | 6 +++--- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index edbc315e..353cfcf2 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -95,7 +95,7 @@ class DatabaseView(View): queries_page = await datasette.list_queries( database, actor=request.actor, - limit=20, + limit=5, include_private=True, ) canned_queries = queries_page["queries"] diff --git a/queries-plan.md b/queries-plan.md index 671fc29c..82ef3260 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -392,7 +392,7 @@ The save form should call `POST /{database}/-/queries/-/insert` and default to ` If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. -On `/{database}`, show a preview of the first 20 visible queries using `list_queries(..., limit=20)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. +On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. ## Dedicated create query UI diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index c46fd86f..a9d22036 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -248,10 +248,9 @@ def test_json_response(canned_write_client, headers, body, querystring): def test_canned_query_permissions_on_database_page(canned_write_client): - # Without auth only shows three queries - query_names = { - q["name"] for q in canned_write_client.get("/data.json").json["queries"] - } + # Without auth shows the five public queries + anon_response = canned_write_client.get("/data.json") + query_names = {q["name"] for q in anon_response.json["queries"]} assert query_names == { "add_name_specify_id_with_error_in_on_success_message_sql", "update_name", @@ -259,8 +258,9 @@ def test_canned_query_permissions_on_database_page(canned_write_client): "canned_read", "add_name", } + assert anon_response.json["queries_more"] is False - # With auth shows four + # With auth the database page preview shows the first five queries response = canned_write_client.get( "/data.json", cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, @@ -273,6 +273,31 @@ def test_canned_query_permissions_on_database_page(canned_write_client): ], key=lambda q: q["name"], ) + assert query_names_and_private == [ + {"name": "add_name", "private": False}, + {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, + {"name": "canned_read", "private": False}, + {"name": "delete_name", "private": True}, + ] + assert response.json["queries_more"] is True + + # The full query list endpoint includes the remaining query + response = canned_write_client.get( + "/data/-/queries.json?_size=10", + cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, + ) + assert response.status == 200 + query_names_and_private = sorted( + [ + {"name": q["name"], "private": q["private"]} + for q in response.json["queries"] + ], + key=lambda q: q["name"], + ) assert query_names_and_private == [ {"name": "add_name", "private": False}, {"name": "add_name_specify_id", "private": False}, diff --git a/tests/test_queries.py b/tests/test_queries.py index dd906faf..2b46e00f 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -267,10 +267,10 @@ async def test_database_page_query_preview_is_limited(): json_response = await ds.client.get("/data.json") assert html_response.status_code == 200 - assert "Demo query 20" in html_response.text - assert "Demo query 21" not in html_response.text + assert "Demo query 05" in html_response.text + assert "Demo query 06" not in html_response.text assert 'href="/data/-/queries"' in html_response.text - assert len(json_response.json()["queries"]) == 20 + assert len(json_response.json()["queries"]) == 5 assert json_response.json()["queries_more"] is True From 6eee6c81e8c21737e2391af55baf24866429038d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 10:24:42 -0700 Subject: [PATCH 693/767] Add global query browser Refs #2735 --- datasette/app.py | 57 +++++++++++++++++++----- datasette/templates/query_list.html | 11 +++-- datasette/views/database.py | 27 ++++++++++-- docs/json_api.rst | 3 +- queries-plan.md | 6 +-- tests/test_queries.py | 67 +++++++++++++++++++++++++++++ 6 files changed, 149 insertions(+), 22 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index bdbf9389..c047fde9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -52,6 +52,7 @@ from .views.database import ( QueryCreateView, QueryDeleteView, QueryDefinitionView, + GlobalQueryListView, QueryInsertView, QueryListView, QueryUpdateView, @@ -1290,7 +1291,7 @@ class Datasette: async def list_queries( self, - database, + database=None, *, actor=None, limit=50, @@ -1310,16 +1311,40 @@ class Datasette: include_is_private=include_private, ) params = dict(allowed_params) - params.update({"query_database": database, "limit": limit + 1}) + params.update({"limit": limit + 1}) sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" - where_clauses = ["q.database_name = :query_database"] + where_clauses = [] + order_by = "q.database_name, sort_key, q.name" + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + order_by = "sort_key, q.name" if cursor: try: components = urlsafe_components(cursor) except ValueError: components = [] - if len(components) == 2: + if database is None and len(components) == 3: + where_clauses.append(""" + ( + q.database_name > :cursor_database + OR ( + q.database_name = :cursor_database + AND ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_database"] = components[0] + params["cursor_sort_key"] = components[1] + params["cursor_name"] = components[2] + elif database is not None and len(components) == 2: where_clauses.append(""" ( {sort_key_sql} > :cursor_sort_key @@ -1368,13 +1393,14 @@ class Datasette: ON allowed.parent = q.database_name AND allowed.child = q.name WHERE {where} - ORDER BY sort_key, q.name + ORDER BY {order_by} LIMIT :limit """.format( allowed_sql=allowed_sql, private_select=private_select, sort_key_sql=sort_key_sql, - where=" AND ".join(where_clauses), + where=" AND ".join(where_clauses) or "1 = 1", + order_by=order_by, ), params, ) @@ -1394,10 +1420,17 @@ class Datasette: next_token = None if has_more and rows: last_row = rows[-1] - next_token = "{},{}".format( - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) + if database is None: + next_token = "{},{},{}".format( + tilde_encode(last_row["database_name"]), + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + else: + next_token = "{},{}".format( + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) return { "queries": queries, "next": next_token, @@ -2651,6 +2684,10 @@ class Datasette: JumpView.as_view(self), r"/-/jump(\.(?Pjson))?$", ) + add_route( + GlobalQueryListView.as_view(self), + r"/-/queries(\.(?Pjson))?$", + ) add_route( InstanceSchemaView.as_view(self), r"/-/schema(\.(?Pjson|md))?$", diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index ef5da0d5..af974550 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -1,8 +1,8 @@ {% extends "base.html" %} -{% block title %}{{ database }}: queries{% endblock %} +{% block title %}{% if database %}{{ database }}: {% endif %}queries{% endblock %} -{% block body_class %}query-list db-{{ database|to_css_class }}{% endblock %} +{% block body_class %}query-list{% if database %} db-{{ database|to_css_class }}{% endif %}{% endblock %} {% block crumbs %} {{ crumbs.nav(request=request, database=database) }} @@ -12,7 +12,7 @@

Queries

-
+

@@ -38,7 +38,10 @@

    {% for query in queries %}
  • - {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} + {% if show_database %} + {{ query.database }}: + {% endif %} + {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} {% if query.is_write %}Writable{% endif %} {% if query.is_published %}Published{% endif %}
    • diff --git a/datasette/views/database.py b/datasette/views/database.py index 353cfcf2..1576b6a9 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -967,8 +967,14 @@ class ExecuteWriteView(BaseView): class QueryListView(BaseView): name = "query-list" + async def database_name(self, request): + return (await self.ds.resolve_database(request)).name + + def query_list_path(self, database): + return self.ds.urls.database(database) + "/-/queries" + async def get(self, request): - db = await self.ds.resolve_database(request) + database = await self.database_name(request) format_ = request.url_vars.get("format") or "html" try: limit = _query_list_limit(request.args.get("_size")) @@ -980,7 +986,7 @@ class QueryListView(BaseView): return _error([ex.message], ex.status) page = await self.ds.list_queries( - db.name, + database, actor=request.actor, limit=limit, cursor=request.args.get("_next"), @@ -991,6 +997,7 @@ class QueryListView(BaseView): owner_id=request.args.get("owner_id") or None, include_private=True, ) + query_list_path = self.query_list_path(database) next_url = None if page["next"]: pairs = [ @@ -1002,18 +1009,20 @@ class QueryListView(BaseView): ] pairs.append(("_next", page["next"])) next_url = "{}?{}".format( - self.ds.urls.database(db.name) + "/-/queries", + query_list_path, urlencode(pairs), ) data = { "ok": True, - "database": db.name, + "database": database, "queries": page["queries"], "next": page["next"], "next_url": next_url, "has_more": page["has_more"], "limit": page["limit"], + "query_list_path": query_list_path, + "show_database": database is None, "filters": { "q": request.args.get("q") or "", "is_write": request.args.get("is_write") or "", @@ -1031,6 +1040,16 @@ class QueryListView(BaseView): ) +class GlobalQueryListView(QueryListView): + name = "global-query-list" + + async def database_name(self, request): + return None + + def query_list_path(self, database): + return self.ds.urls.path("/-/queries") + + class QueryCreateView(BaseView): name = "query-create" has_json_alternate = False diff --git a/docs/json_api.rst b/docs/json_api.rst index ece430c2..f44a39fe 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -505,12 +505,13 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`. +.. _GlobalQueryListView: .. _QueryListView: Listing saved queries ~~~~~~~~~~~~~~~~~~~~~ -``GET //-/queries.json`` returns saved query definitions the actor can view. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. +``GET /-/queries.json`` returns saved query definitions across every database that the actor can view. ``GET //-/queries.json`` returns saved query definitions for a specific database. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. .. _QueryCreateView: diff --git a/queries-plan.md b/queries-plan.md index 82ef3260..a708e887 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -210,7 +210,7 @@ JSON endpoints should follow Datasette's existing write API style: use `POST` pl Endpoints: -- `GET /{database}/-/queries` shows a searchable HTML query browser. `GET /{database}/-/queries.json` returns query definitions the actor can view, using cursor pagination with `_next` and `_size`. +- `GET /-/queries` and `GET /{database}/-/queries` show searchable HTML query browsers. `GET /-/queries.json` lists query definitions across every database the actor can view; `GET /{database}/-/queries.json` scopes that list to one database. Both JSON endpoints use cursor pagination with `_next` and `_size`. - `POST /{database}/-/queries/-/insert` creates a query. - `GET /{database}/{query}/-/definition` returns one query definition without executing it. - `POST /{database}/{query}/-/update` updates one query. @@ -366,7 +366,7 @@ await datasette.list_queries( ) ``` -`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. +`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. Passing `database=None` lists visible queries across all live databases, still filtered through `view-query` permission SQL. `update_query()` should use an internal sentinel default such as `UNCHANGED = object()` so callers can distinguish "leave this column alone" from "set this column to `NULL`": @@ -392,7 +392,7 @@ The save form should call `POST /{database}/-/queries/-/insert` and default to ` If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. -On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. +On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. The global `/-/queries` page reuses the same interface and shows the database for each query. ## Dedicated create query UI diff --git a/tests/test_queries.py b/tests/test_queries.py index 2b46e00f..bc04bb51 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -457,6 +457,73 @@ async def test_query_list_search_filter_and_html(): ] +@pytest.mark.asyncio +async def test_global_query_list_api_and_html(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_global_alpha", name="alpha") + ds.add_memory_database("query_list_global_beta", name="beta") + await ds.invoke_startup() + await ds.add_query( + "alpha", + "alpha_first", + "select 1", + title="Alpha first", + is_published=True, + source="user", + owner_id="root", + ) + await ds.add_query( + "alpha", + "alpha_second", + "select 2", + title="Alpha second", + is_published=True, + source="user", + owner_id="root", + ) + await ds.add_query( + "beta", + "beta_first", + "select 3", + title="Beta first", + is_published=True, + source="user", + owner_id="root", + ) + + list_response = await ds.client.get( + "/-/queries.json?_size=2", + actor={"id": "root"}, + ) + next_response = await ds.client.get( + "/-/queries.json?_size=2&_next={}".format(list_response.json()["next"]), + actor={"id": "root"}, + ) + html_response = await ds.client.get( + "/-/queries?q=Beta", + actor={"id": "root"}, + ) + + assert list_response.status_code == 200 + assert [ + (query["database"], query["name"]) for query in list_response.json()["queries"] + ] == [ + ("alpha", "alpha_first"), + ("alpha", "alpha_second"), + ] + assert list_response.json()["next"] + assert [ + (query["database"], query["name"]) for query in next_response.json()["queries"] + ] == [ + ("beta", "beta_first"), + ] + assert html_response.status_code == 200 + assert 'href="/beta">beta:' in html_response.text + assert "Beta first" in html_response.text + assert "Alpha first" not in html_response.text + + @pytest.mark.asyncio async def test_query_insert_api_publish_requires_publish_query(): ds = Datasette( From f0b59971f7c8c0f4435a18b4f4e9c8053c2683fe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 10:39:56 -0700 Subject: [PATCH 694/767] Delete unnecessary test --- tests/test_utils_sql_analysis.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index c82fb04f..5730cd0d 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -169,13 +169,6 @@ def test_analyze_attached_database_tables(conn): } -def test_analyze_invalid_sql_cleans_up_authorizer(conn): - with pytest.raises(sqlite3.OperationalError): - analyze_sql_tables(conn, "insert into missing_table values (1)") - - conn.execute("select name from dogs").fetchall() - - def test_analyze_clears_authorizer_on_error(): class FakeConnection: def __init__(self): From 2b5b4ed66b86bae0080e9d8f4881cad8e57bbdb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 11:11:08 -0700 Subject: [PATCH 695/767] Much improved "Write to this database" UI - Start with a template option, letting you pick table and operation - SQL textarea defaults to 4 empty lines at start - Query operations table is simpler and looks nicer Refs #2742 --- datasette/templates/execute_write.html | 240 +++++++++++++++++++++++-- datasette/views/database.py | 13 +- tests/test_queries.py | 32 +++- 3 files changed, 271 insertions(+), 14 deletions(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 5b4f30d9..90845910 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -1,10 +1,80 @@ {% extends "base.html" %} -{% block title %}Execute write SQL{% endblock %} +{% block title %}Write to this database{% endblock %} {% block extra_head %} {{- super() -}} {% include "_codemirror.html" %} + {% endblock %} {% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %} @@ -15,13 +85,34 @@ {% block content %} -

    Execute write SQL

    +

    Write to this database

    + +

    Execute SQL to insert, update or delete rows in this database.

    {% if execution_message %}

    {{ execution_message }}

    {% endif %} + {% if write_template_tables %} +
    +
    + Start with a template +

    + + + + + +

    +
    +
    + {% endif %} +

    {% if parameter_names %} @@ -31,30 +122,28 @@ {% endfor %} {% endif %} -

    Analysis

    +

    Query operations

    {% if analysis_error %}

    {{ analysis_error }}

    {% elif analysis_rows %} -
    +
    - + - {% for row in analysis_rows %} - - - - - - + + + + + {% endfor %} @@ -66,6 +155,133 @@

    + + {% include "_codemirror_foot.html" %} +{% if write_template_tables %} + +{% endif %} + {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 1576b6a9..fb3bdfdb 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -830,6 +830,13 @@ class ExecuteWriteView(BaseView): parameter_values = parameter_values or {} parameter_names = [] analysis_rows = [] + table_columns = await _table_columns(self.ds, db.name) + hidden_table_names = set(await db.hidden_table_names()) + write_template_tables = { + table: columns + for table, columns in table_columns.items() + if columns and table not in hidden_table_names + } if sql and analysis_error is None: try: parameter_names = _derived_query_parameters(sql) @@ -858,7 +865,9 @@ class ExecuteWriteView(BaseView): "parameter_names": parameter_names, "parameter_values": parameter_values, "analysis_error": analysis_error, - "analysis_rows": analysis_rows, + "analysis_rows": [ + row for row in analysis_rows if row["operation"] != "read" + ], "execution_message": execution_message, "execution_ok": execution_ok, "execute_disabled": bool( @@ -866,6 +875,8 @@ class ExecuteWriteView(BaseView): or analysis_error or any(row["allowed"] is False for row in analysis_rows) ), + "table_columns": table_columns, + "write_template_tables": write_template_tables, }, ) response.status = status diff --git a/tests/test_queries.py b/tests/test_queries.py index bc04bb51..684454fc 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -690,6 +690,14 @@ async def test_execute_write_get_prepopulates_without_executing(): ds.root_enabled = True db = ds.add_memory_database("execute_write_get", name="data") await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write("create table cats (id integer primary key, name text)") + await db.execute_write("create table log (message text)") + await db.execute_write(""" + create trigger dogs_after_insert after insert on dogs begin + update cats set name = new.name where id = new.id; + insert into log (message) values (new.name); + end + """) await ds.invoke_startup() response = await ds.client.get( @@ -700,11 +708,33 @@ async def test_execute_write_get_prepopulates_without_executing(): assert response.status_code == 200 assert response.headers["content-security-policy"] == "frame-ancestors 'none'" assert response.headers["x-frame-options"] == "DENY" - assert "Execute write SQL" in response.text + assert "Write to this database" in response.text + assert ( + "Execute SQL to insert, update or delete rows in this database." + in response.text + ) + assert "

    Query operations

    " in response.text + assert "Start with a template" in response.text + assert '' in response.text + assert 'data-sql-template="insert"' in response.text + assert 'data-sql-template="update"' in response.text + assert 'data-sql-template="delete"' in response.text + assert '
    Operation Database Tablerequired permissionRequired permission AllowedSource
    {{ row.operation }}{{ row.database }}{{ row.table }}{{ row.required_permission }}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}{{ row.source or "" }}{{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% endif %}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}
    ' in response.text + assert '' in response.text + assert "" in response.text + assert "" in response.text + assert "" not in response.text assert 'action="/data/-/execute-write"' in response.text assert "insert into dogs (name) values ('Cleo')" in response.text assert (await db.execute("select count(*) from dogs")).first()[0] == 0 + empty_response = await ds.client.get( + "/data/-/execute-write", + actor={"id": "root"}, + ) + assert '' in empty_response.text + assert 'executeWriteSqlInput.value = "\\n\\n\\n";' in empty_response.text + @pytest.mark.asyncio async def test_database_action_menu_links_to_execute_write_for_permitted_actor(): From 1bce34a33869709e1dea21b6182327a105895285 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 11:22:24 -0700 Subject: [PATCH 696/767] If just a single insert, link to row page Refs #2742 --- datasette/templates/execute_write.html | 2 +- datasette/views/database.py | 49 ++++++++++++++++++++++++++ tests/test_queries.py | 42 ++++++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 90845910..705181d8 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -90,7 +90,7 @@

    Execute SQL to insert, update or delete rows in this database.

    {% if execution_message %} -

    {{ execution_message }}

    +

    {{ execution_message }}{% for link in execution_links %} {{ link.label }}{% endfor %}

    {% endif %}
    diff --git a/datasette/views/database.py b/datasette/views/database.py index fb3bdfdb..2b3920f7 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -18,8 +18,10 @@ from datasette.utils import ( await_me_maybe, call_with_supported_arguments, named_parameters as derive_named_parameters, + escape_sqlite, format_bytes, make_slot_function, + path_from_row_pks, tilde_decode, to_css_class, validate_sql_select, @@ -678,6 +680,43 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis +async def _inserted_row_url(datasette, db, analysis, cursor): + if cursor.rowcount != 1: + return None + lastrowid = getattr(cursor, "lastrowid", None) + if lastrowid is None: + return None + direct_inserts = [ + access + for access in analysis.table_accesses + if access.operation == "insert" + and access.source is None + and access.database == db.name + ] + if len(direct_inserts) != 1: + return None + table = direct_inserts[0].table + pks = await db.primary_keys(table) + use_rowid = not pks + select = ( + "rowid" + if use_rowid + else ", ".join(escape_sqlite(primary_key) for primary_key in pks) + ) + try: + result = await db.execute( + "select {} from {} where rowid = ?".format(select, escape_sqlite(table)), + [lastrowid], + ) + except sqlite3.DatabaseError: + return None + row = result.first() + if row is None: + return None + row_path = path_from_row_pks(row, pks, use_rowid) + return datasette.urls.row(db.name, table, row_path) + + def _apply_query_data_types(data): typed = dict(data) for key in ("hide_sql", "is_published"): @@ -824,10 +863,12 @@ class ExecuteWriteView(BaseView): analysis=None, analysis_error=None, execution_message=None, + execution_links=None, execution_ok=None, status=200, ): parameter_values = parameter_values or {} + execution_links = execution_links or [] parameter_names = [] analysis_rows = [] table_columns = await _table_columns(self.ds, db.name) @@ -869,6 +910,7 @@ class ExecuteWriteView(BaseView): row for row in analysis_rows if row["operation"] != "read" ], "execution_message": execution_message, + "execution_links": execution_links, "execution_ok": execution_ok, "execute_disabled": bool( (not sql) @@ -964,6 +1006,12 @@ class ExecuteWriteView(BaseView): ) ) + inserted_row_url = await _inserted_row_url(self.ds, db, analysis, cursor) + execution_links = ( + [{"href": inserted_row_url, "label": "View row"}] + if inserted_row_url + else [] + ) return await self._render_form( request, db, @@ -971,6 +1019,7 @@ class ExecuteWriteView(BaseView): parameter_values={name: params.get(name, "") for name in parameter_names}, analysis=analysis, execution_message=message, + execution_links=execution_links, execution_ok=True, ) diff --git a/tests/test_queries.py b/tests/test_queries.py index 684454fc..ed981ee7 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -849,6 +849,48 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.asyncio +async def test_execute_write_insert_links_to_inserted_row(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_insert_link", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write("create table log (id integer primary key, message text)") + await db.execute_write("insert into log (message) values ('existing')") + await db.execute_write(""" + create trigger dogs_after_insert after insert on dogs begin + insert into log (message) values (new.name); + end + """) + await ds.invoke_startup() + + insert_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "insert into dogs (name) values (:name)", + "name": "Cleo", + }, + ) + update_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "update dogs set name = :name where id = :id", + "name": "Cleo 2", + "id": "1", + }, + ) + + assert insert_response.status_code == 200 + assert "Query executed, 1 row affected" in insert_response.text + assert 'View row' in insert_response.text + assert "/data/log/2" not in insert_response.text + assert update_response.status_code == 200 + assert "Query executed, 1 row affected" in update_response.text + assert "View row" not in update_response.text + + @pytest.mark.asyncio async def test_execute_write_post_rejects_read_only_sql(): ds = Datasette(memory=True, default_deny=True) From 66bbbbc947bd4d7305761a627dc2f1949949c0a5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 11:35:09 -0700 Subject: [PATCH 697/767] Support multi-line parameters on /db/-/execute-write Refs https://github.com/simonw/datasette/issues/2742#issuecomment-4536317049 Each paramater input now has an expand/collapse button toggle to turn into a textarea. If you paste text that includes at least one newline it toggles automatically. --- datasette/templates/execute_write.html | 94 +++++++++++++++++++++++++- tests/test_queries.py | 1 + 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 705181d8..a560e920 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -74,6 +74,25 @@ color: #b00020; font-weight: 700; } +form.sql .execute-write-parameter-row textarea[data-parameter-control] { + border: 1px solid #ccc; + border-radius: 3px; + box-sizing: content-box; + display: inline-block; + font-family: Helvetica, sans-serif; + font-size: 1em; + min-height: 7rem; + padding: 9px 4px; + vertical-align: top; + width: 60%; +} +form.sql.core button.execute-write-parameter-toggle[type=button] { + font-size: 0.72rem; + height: 1.8rem; + line-height: 1; + margin-left: 0.35rem; + padding: 0.25rem 0.45rem; +} {% endblock %} @@ -118,7 +137,7 @@ {% if parameter_names %}

    Parameters

    {% for parameter in parameter_names %} -

    +

    {% endfor %} {% endif %} @@ -164,6 +183,79 @@ if (executeWriteSqlInput && !executeWriteSqlInput.value) { {% include "_codemirror_foot.html" %} + + {% if write_template_tables %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 2b3920f7..e4eaee30 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -680,6 +680,39 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis +async def _execute_write_analysis_data(datasette, db, sql, actor): + parameter_names = [] + analysis_rows = [] + analysis_error = None + if sql: + try: + parameter_names = _derived_query_parameters(sql) + params = {parameter: "" for parameter in parameter_names} + analysis = await db.analyze_sql(sql, params) + if _analysis_is_write(analysis): + analysis_rows = await _analysis_rows_with_permissions( + datasette, analysis, actor + ) + else: + analysis_error = ( + "Use /-/query for read-only SQL; " + "this endpoint only executes writes" + ) + except (QueryValidationError, sqlite3.DatabaseError) as ex: + analysis_error = getattr(ex, "message", str(ex)) + return { + "ok": analysis_error is None, + "parameters": parameter_names, + "analysis_error": analysis_error, + "analysis_rows": [row for row in analysis_rows if row["operation"] != "read"], + "execute_disabled": bool( + (not sql) + or analysis_error + or any(row["allowed"] is False for row in analysis_rows) + ), + } + + async def _inserted_row_url(datasette, db, analysis, cursor): if cursor.rowcount != 1: return None @@ -1024,6 +1057,45 @@ class ExecuteWriteView(BaseView): ) +class ExecuteWriteAnalyzeView(BaseView): + name = "execute-write-analyze" + has_json_alternate = False + + async def post(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing( + _error(["Permission denied: need execute-write-sql"], 403) + ) + + try: + data, _ = await _json_or_form_payload(request) + except QueryValidationError as ex: + return _block_framing(_error([ex.message], ex.status)) + if not isinstance(data, dict): + return _block_framing(_error(["JSON must be a dictionary"], 400)) + invalid_keys = set(data) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + sql = data.get("sql") or "" + if not isinstance(sql, str): + return _block_framing(_error(["sql must be a string"], 400)) + return _block_framing( + Response.json( + await _execute_write_analysis_data(self.ds, db, sql, request.actor) + ) + ) + + class QueryListView(BaseView): name = "query-list" diff --git a/docs/json_api.rst b/docs/json_api.rst index f44a39fe..2f581661 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -528,6 +528,7 @@ Creating saved queries ``POST //-/queries/-/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. .. _ExecuteWriteView: +.. _ExecuteWriteAnalyzeView: Executing write SQL ~~~~~~~~~~~~~~~~~~~ @@ -536,6 +537,8 @@ Executing write SQL ``POST //-/execute-write`` executes writable SQL. This requires ``execute-write-sql`` for the database plus the relevant table-level write permissions. +``POST //-/execute-write/-/analyze`` accepts ``{"sql": "..."}`` and returns the derived parameters plus the write operations that SQL would need in order to execute. + .. _QueryDefinitionView: Getting a saved query definition diff --git a/tests/test_queries.py b/tests/test_queries.py index a6080958..6d2c0b25 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -719,7 +719,9 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'data-sql-template="insert"' in response.text assert 'data-sql-template="update"' in response.text assert 'data-sql-template="delete"' in response.text + assert 'data-analyze-url="/data/-/execute-write/-/analyze"' in response.text assert 'addEventListener("paste"' in response.text + assert "refreshExecuteWriteAnalysis" in response.text assert '
    Required permissioninsertupdateread
    ' in response.text assert '' in response.text assert "" in response.text @@ -737,6 +739,53 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'executeWriteSqlInput.value = "\\n\\n\\n";' in empty_response.text +@pytest.mark.asyncio +async def test_execute_write_analyze_endpoint_uses_sql_only(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_analyze", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/execute-write/-/analyze", + actor={"id": "root"}, + json={"sql": "insert into dogs (name) values (:name)"}, + ) + read_only_response = await ds.client.post( + "/data/-/execute-write/-/analyze", + actor={"id": "root"}, + json={"sql": "select * from dogs where name = :name"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["ok"] is True + assert data["parameters"] == ["name"] + assert data["analysis_error"] is None + assert data["execute_disabled"] is False + assert data["analysis_rows"] == [ + { + "operation": "insert", + "database": "data", + "table": "dogs", + "required_permission": "insert-row", + "source": None, + "allowed": True, + } + ] + assert "params" not in data + + assert read_only_response.status_code == 200 + read_only_data = read_only_response.json() + assert read_only_data["ok"] is False + assert read_only_data["parameters"] == ["name"] + assert read_only_data["analysis_error"] == ( + "Use /-/query for read-only SQL; this endpoint only executes writes" + ) + assert read_only_data["execute_disabled"] is True + + @pytest.mark.asyncio async def test_database_action_menu_links_to_execute_write_for_permitted_actor(): ds = Datasette( From de55a76d402a6326c60a5f4cd1a03c7476613f0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 12:33:57 -0700 Subject: [PATCH 700/767] Fix 500 error when accessing query page without ?sql= parameter (#2744) Closes #2743 --- datasette/templates/query.html | 4 ++-- datasette/views/database.py | 43 ++++++++++++++++++---------------- docs/changelog.rst | 7 ++++++ tests/plugins/my_plugin.py | 4 ++-- tests/test_html.py | 16 +++++++++++++ 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 8b405da5..5f85ac6b 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -46,14 +46,14 @@ {% if not hide_sql %} {% if editable and allow_execute_sql %}

    + >{% if query and query.sql %}{{ query.sql }}{% elif tables %}select * from {{ tables[0].name|escape_sqlite }}{% endif %}

    {% else %} 
    {% if query %}{{ query.sql }}{% endif %}
    {% endif %} {% else %} {% if not canned_query %} {% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 0cf93832..8e4ea85a 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -577,7 +577,7 @@ class QueryView(View): named_parameters = [] if canned_query and canned_query.get("params"): named_parameters = canned_query["params"] - if not named_parameters: + if not named_parameters and sql: named_parameters = derive_named_parameters(sql) named_parameter_values = { named_parameter: params.get(named_parameter) or "" @@ -602,7 +602,7 @@ class QueryView(View): params_for_query = params - if not canned_query_write: + if sql and not canned_query_write: try: if not canned_query: # For regular queries we only allow SELECT, plus other rules @@ -646,6 +646,8 @@ class QueryView(View): # Handle formats from plugins if format_ == "csv": + if not sql: + raise DatasetteError("?sql= is required", status=400) async def fetch_data_for_csv(request, _next=None): results = await db.execute(sql, params, truncate=True) @@ -771,25 +773,26 @@ class QueryView(View): # - No magic parameters, so no :_ in the SQL string edit_sql_url = None is_validated_sql = False - try: - validate_sql_select(sql) - is_validated_sql = True - except InvalidSql: - pass - if allow_execute_sql and is_validated_sql and ":_" not in sql: - edit_sql_url = ( - datasette.urls.database(database) - + "/-/query" - + "?" - + urlencode( - { - **{ - "sql": sql, - }, - **named_parameter_values, - } + if sql: + try: + validate_sql_select(sql) + is_validated_sql = True + except InvalidSql: + pass + if allow_execute_sql and is_validated_sql and ":_" not in sql: + edit_sql_url = ( + datasette.urls.database(database) + + "/-/query" + + "?" + + urlencode( + { + **{ + "sql": sql, + }, + **named_parameter_values, + } + ) ) - ) async def query_actions(): query_actions = [] diff --git a/docs/changelog.rst b/docs/changelog.rst index 329b4769..dfb2a736 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v1_0_unreleased: + +Unreleased +---------- + +- Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) + .. _v1_0_a30: 1.0a30 (2026-05-24) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 4e401c07..f682e8b9 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -387,8 +387,8 @@ def view_actions(datasette, database, view, actor): @hookimpl def query_actions(datasette, database, query_name, sql): - # Don't explain an explain - if sql.lower().startswith("explain"): + # Don't explain an explain (or a missing query) + if not sql or sql.lower().startswith("explain"): return return [ { diff --git a/tests/test_html.py b/tests/test_html.py index efc1040d..d20796c9 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -241,6 +241,22 @@ def test_query_page_truncates(): ] +@pytest.mark.asyncio +async def test_query_page_with_no_sql(ds_client): + # https://github.com/simonw/datasette/issues/2743 + response = await ds_client.get("/fixtures/-/query") + assert response.status_code == 200 + assert '

    +

    + {% set parameter_names = [] %} + {% set parameter_values = {} %} + {% set sql_parameters_allow_expand = false %} + {% include "_sql_parameters.html" %}

    @@ -90,5 +95,11 @@ {% endif %} {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} + {% endblock %} diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 5037d006..9b522f66 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -75,61 +75,8 @@ color: #b00020; font-weight: 700; } -form.sql .execute-write-parameter-row textarea[data-parameter-control] { - border: 1px solid #ccc; - border-radius: 3px; - box-sizing: border-box; - display: block; - font-family: Helvetica, sans-serif; - font-size: 1em; - min-height: 7rem; - padding: 9px 4px; - width: 100%; -} -form.sql .execute-write-parameter-row { - align-items: start; - column-gap: 0.6rem; - display: grid; - grid-template-columns: minmax(8rem, 11rem) minmax(16rem, 1fr) auto; - margin: 0 0 0.65rem; - max-width: 52rem; -} -form.sql .execute-write-parameter-row label { - overflow-wrap: anywhere; - padding-top: 0.55rem; - width: auto; -} -form.sql .execute-write-parameter-row input[data-parameter-control] { - box-sizing: border-box; - width: 100%; -} -form.sql.core button.execute-write-parameter-toggle[type=button] { - font-size: 0.72rem; - height: 1.8rem; - line-height: 1; - margin: 0.25rem 0 0; - padding: 0.25rem 0.45rem; -} -@media (max-width: 480px) { - form.sql .execute-write-parameter-row { - grid-template-columns: 1fr; - row-gap: 0.25rem; - } - form.sql .execute-write-parameter-row label { - padding-top: 0; - } - form.sql.core button.execute-write-parameter-toggle[type=button] { - justify-self: start; - margin-top: 0; - } -} -form.sql .execute-write-editor { - max-width: 52rem; -} -form.sql .execute-write-editor textarea#sql-editor { - width: 100%; -} +{% include "_sql_parameter_styles.html" %} {% endblock %} {% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %} @@ -168,16 +115,11 @@ form.sql .execute-write-editor textarea#sql-editor { {% endif %} -

    +

    -
    - {% if parameter_names %} -

    Parameters

    - {% for parameter in parameter_names %} -

    - {% endfor %} - {% endif %} -
    + {% set sql_parameters_section_id = "execute-write-parameters-section" %} + {% set sql_parameters_allow_expand = true %} + {% include "_sql_parameters.html" %}

    Query operations

    @@ -222,128 +164,15 @@ if (executeWriteSqlInput && !executeWriteSqlInput.value) { {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 7c251e2c..3bcc7178 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -14,6 +14,7 @@ {% endif %} {% include "_codemirror.html" %} +{% include "_sql_parameter_styles.html" %} {% endblock %} {% block body_class %}query db-{{ database|to_css_class }}{% if canned_query %} query-{{ canned_query|to_css_class }}{% endif %}{% endblock %} @@ -36,7 +37,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

    @@ -45,7 +46,7 @@ {% endif %} {% if not hide_sql %} {% if editable and allow_execute_sql %} -

    {% else %} 
    {% if query %}{{ query.sql }}{% endif %}
    @@ -57,12 +58,10 @@ > {% endif %} {% endif %} - {% if named_parameter_values %} -

    Query parameters

    - {% for name, value in named_parameter_values.items() %} -

    - {% endfor %} - {% endif %} + {% set parameter_names = named_parameter_values.keys()|list %} + {% set parameter_values = named_parameter_values %} + {% set sql_parameters_allow_expand = false %} + {% include "_sql_parameters.html" %}

    {% if not hide_sql %}{% endif %} @@ -97,5 +96,11 @@ {% endif %} {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} + {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index e4eaee30..278f7e8c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1061,7 +1061,7 @@ class ExecuteWriteAnalyzeView(BaseView): name = "execute-write-analyze" has_json_alternate = False - async def post(self, request): + async def get(self, request): db = await self.ds.resolve_database(request) if not await self.ds.allowed( action="execute-write-sql", @@ -1072,13 +1072,7 @@ class ExecuteWriteAnalyzeView(BaseView): _error(["Permission denied: need execute-write-sql"], 403) ) - try: - data, _ = await _json_or_form_payload(request) - except QueryValidationError as ex: - return _block_framing(_error([ex.message], ex.status)) - if not isinstance(data, dict): - return _block_framing(_error(["JSON must be a dictionary"], 400)) - invalid_keys = set(data) - {"sql"} + invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( _error( @@ -1086,9 +1080,7 @@ class ExecuteWriteAnalyzeView(BaseView): 400, ) ) - sql = data.get("sql") or "" - if not isinstance(sql, str): - return _block_framing(_error(["sql must be a string"], 400)) + sql = request.args.get("sql") or "" return _block_framing( Response.json( await _execute_write_analysis_data(self.ds, db, sql, request.actor) @@ -1096,6 +1088,34 @@ class ExecuteWriteAnalyzeView(BaseView): ) +class QueryParametersView(BaseView): + name = "query-parameters" + has_json_alternate = False + + async def get(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + + invalid_keys = set(request.args) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + try: + parameters = _derived_query_parameters(request.args.get("sql") or "") + except QueryValidationError as ex: + return _block_framing(_error([ex.message], ex.status)) + return _block_framing(Response.json({"ok": True, "parameters": parameters})) + + class QueryListView(BaseView): name = "query-list" diff --git a/docs/json_api.rst b/docs/json_api.rst index 2f581661..91ed5306 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -527,17 +527,20 @@ Creating saved queries ``POST //-/queries/-/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. +.. _QueryParametersView: .. _ExecuteWriteView: .. _ExecuteWriteAnalyzeView: Executing write SQL ~~~~~~~~~~~~~~~~~~~ +``GET //-/query/-/parameters?sql=...`` returns the named parameters used by a SQL query. This requires ``execute-sql`` for the database. + ``GET //-/execute-write`` displays a form for executing writable SQL. A ``?sql=`` query string pre-populates the form without executing it. ``POST //-/execute-write`` executes writable SQL. This requires ``execute-write-sql`` for the database plus the relevant table-level write permissions. -``POST //-/execute-write/-/analyze`` accepts ``{"sql": "..."}`` and returns the derived parameters plus the write operations that SQL would need in order to execute. +``GET //-/execute-write/-/analyze?sql=...`` returns the derived parameters plus the write operations that SQL would need in order to execute. .. _QueryDefinitionView: diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index a9d22036..ae2c74e0 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -200,7 +200,10 @@ def test_error_in_on_success_message_sql(canned_write_client): def test_custom_params(canned_write_client): response = canned_write_client.get("/data/update_name?extra=foo") - assert '' in response.text + assert ( + '' + in response.text + ) def test_canned_query_pages_no_vary_header(canned_write_client): diff --git a/tests/test_html.py b/tests/test_html.py index e5f00e17..b49391a6 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -326,17 +326,29 @@ async def test_query_parameter_form_fields(ds_client): response = await ds_client.get("/fixtures/-/query?sql=select+:name") assert response.status_code == 200 assert ( - ' ' + ' ' in response.text ) + assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'id="sql-parameters-section"' in response.text + assert "setupSqlParameterRefresh" in response.text response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") assert response2.status_code == 200 assert ( - ' ' + ' ' in response2.text ) +@pytest.mark.asyncio +async def test_database_page_sql_parameter_refresh_markup(ds_client): + response = await ds_client.get("/fixtures") + assert response.status_code == 200 + assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'id="sql-parameters-section"' in response.text + assert "setupSqlParameterRefresh" in response.text + + @pytest.mark.asyncio async def test_row_html_simple_primary_key(ds_client): response = await ds_client.get("/fixtures/simple_primary_key/1") diff --git a/tests/test_queries.py b/tests/test_queries.py index 6d2c0b25..23820cf3 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -721,7 +721,7 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'data-sql-template="delete"' in response.text assert 'data-analyze-url="/data/-/execute-write/-/analyze"' in response.text assert 'addEventListener("paste"' in response.text - assert "refreshExecuteWriteAnalysis" in response.text + assert "setupSqlParameterRefresh" in response.text assert '

    Required permissioninsert
    ' in response.text assert '' in response.text assert "" in response.text @@ -747,15 +747,15 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): await db.execute_write("create table dogs (id integer primary key, name text)") await ds.invoke_startup() - response = await ds.client.post( + response = await ds.client.get( "/data/-/execute-write/-/analyze", actor={"id": "root"}, - json={"sql": "insert into dogs (name) values (:name)"}, + params={"sql": "insert into dogs (name) values (:name)"}, ) - read_only_response = await ds.client.post( + read_only_response = await ds.client.get( "/data/-/execute-write/-/analyze", actor={"id": "root"}, - json={"sql": "select * from dogs where name = :name"}, + params={"sql": "select * from dogs where name = :name"}, ) assert response.status_code == 200 @@ -786,6 +786,44 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): assert read_only_data["execute_disabled"] is True +@pytest.mark.asyncio +async def test_query_parameters_endpoint_uses_get_sql_only(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_parameters", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.get( + "/data/-/query/-/parameters", + actor={"id": "root"}, + params={ + "sql": "select * from dogs where name = :name and id = :id", + }, + ) + permission_denied_response = await ds.client.get( + "/data/-/query/-/parameters", + actor={"id": "not-root"}, + params={"sql": "select * from dogs where name = :name"}, + ) + magic_parameter_response = await ds.client.get( + "/data/-/query/-/parameters", + actor={"id": "root"}, + params={"sql": "select :_actor_id"}, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": True, "parameters": ["name", "id"]} + assert permission_denied_response.status_code == 403 + assert permission_denied_response.json()["errors"] == [ + "Permission denied: need execute-sql" + ] + assert magic_parameter_response.status_code == 400 + assert magic_parameter_response.json()["errors"] == [ + "Magic parameters are not allowed" + ] + + @pytest.mark.asyncio async def test_database_action_menu_links_to_execute_write_for_permitted_actor(): ds = Datasette( From 4208ded249b28f8b0918ce80d289bfc88f9e8921 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 12:46:21 -0700 Subject: [PATCH 702/767] No execute-write on immutable databases Refs https://github.com/simonw/datasette/issues/2742#issuecomment-4536690161 --- datasette/default_database_actions.py | 2 ++ datasette/views/database.py | 7 ++++ tests/test_queries.py | 46 +++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/datasette/default_database_actions.py b/datasette/default_database_actions.py index 78055392..e0cb3cdf 100644 --- a/datasette/default_database_actions.py +++ b/datasette/default_database_actions.py @@ -5,6 +5,8 @@ from datasette.resources import DatabaseResource @hookimpl def database_actions(datasette, actor, database, request): async def inner(): + if not datasette.get_database(database).is_mutable: + return [] if not await datasette.allowed( action="execute-write-sql", resource=DatabaseResource(database), diff --git a/datasette/views/database.py b/datasette/views/database.py index 278f7e8c..de02cd0f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -964,6 +964,13 @@ class ExecuteWriteView(BaseView): resource=DatabaseResource(db.name), actor=request.actor, ) + if not db.is_mutable: + return _block_framing( + _error( + ["Cannot execute write SQL because this database is immutable."], + 403, + ) + ) return await self._render_form( request, db, diff --git a/tests/test_queries.py b/tests/test_queries.py index 23820cf3..c31d7205 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -858,6 +858,52 @@ async def test_database_action_menu_links_to_execute_write_for_permitted_actor() assert "Execute write SQL" in writer_response.text +@pytest.mark.asyncio +async def test_database_action_menu_hides_execute_write_for_immutable_database(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + db = ds.add_memory_database("execute_write_menu_immutable", name="data") + db.is_mutable = False + await ds.invoke_startup() + + response = await ds.client.get("/data", actor={"id": "writer"}) + + assert response.status_code == 200 + assert "Execute write SQL" not in response.text + assert 'href="/data/-/execute-write"' not in response.text + + +@pytest.mark.asyncio +async def test_execute_write_get_rejects_immutable_database(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_get_immutable", name="data") + db.is_mutable = False + await ds.invoke_startup() + + response = await ds.client.get( + "/data/-/execute-write?sql=insert+into+dogs+(name)+values+('Cleo')", + actor={"id": "root"}, + ) + + assert response.status_code == 403 + assert response.json()["errors"] == [ + "Cannot execute write SQL because this database is immutable." + ] + + @pytest.mark.asyncio async def test_execute_write_post_requires_database_and_table_permissions(): ds = Datasette( From 8ab8999ba97e0ec1d113ee8d3954d6431f39fa28 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 12:55:36 -0700 Subject: [PATCH 703/767] Big visual improvement to /-/queries pages Including /db/-/queries Refs https://github.com/simonw/datasette/issues/2735#issuecomment-4536860239 --- datasette/templates/query_list.html | 226 ++++++++++++++++++++++++---- datasette/views/database.py | 12 +- tests/test_queries.py | 25 ++- 3 files changed, 229 insertions(+), 34 deletions(-) diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index af974550..dbd607ab 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -2,6 +2,155 @@ {% block title %}{% if database %}{{ database }}: {% endif %}queries{% endblock %} +{% block extra_head %} +{{- super() -}} + +{% endblock %} + {% block body_class %}query-list{% if database %} db-{{ database|to_css_class }}{% endif %}{% endblock %} {% block crumbs %} @@ -10,49 +159,66 @@ {% block content %} -

    Queries

    +
    - -

    +

    Queries

    + + +

    -

    - - - - -

    +
    +
    + Mode + + + +
    +
    + Publication + + + +
    +
    {% if queries %} -
      - {% for query in queries %} -
    • - {% if show_database %} - {{ query.database }}: - {% endif %} - {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} - {% if query.is_write %}Writable{% endif %} - {% if query.is_published %}Published{% endif %} -
      • - {% endfor %} -
    +
    Required permissioninsert
    + + + {% if show_database %}{% endif %} + + + + + + + {% for query in queries %} + + {% if show_database %} + + {% endif %} + + + + + {% endfor %} + +
    DatabaseQueryModePublication
    {{ query.database }} + {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} + {% if query.description %}

    {{ query.description }}

    {% endif %} +     		{% if query.is_write %}Writable{% else %}Read-only{% endif %}{% if query.is_published %}Published{% else %}Unpublished{% endif %}
    {% else %}

    No queries found.

    {% endif %} {% if next_url %} -

    Next page

    + {% endif %} +
+ {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index de02cd0f..3c660bc7 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -487,9 +487,9 @@ def _as_optional_bool(value, name): raise QueryValidationError("{} must be 0 or 1".format(name)) -def _query_list_limit(value): +def _query_list_limit(value, default=50): if value in (None, ""): - return 50 + return default try: return min(max(1, int(value)), 1000) except ValueError as ex: @@ -1136,7 +1136,10 @@ class QueryListView(BaseView): database = await self.database_name(request) format_ = request.url_vars.get("format") or "html" try: - limit = _query_list_limit(request.args.get("_size")) + limit = _query_list_limit( + request.args.get("_size"), + default=20 if format_ == "html" else 50, + ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") is_published = _as_optional_bool( request.args.get("is_published"), "is_published" @@ -1175,6 +1178,9 @@ class QueryListView(BaseView): data = { "ok": True, "database": database, + "database_color": ( + self.ds.get_database(database).color if database is not None else None + ), "queries": page["queries"], "next": page["next"], "next_url": next_url, diff --git a/tests/test_queries.py b/tests/test_queries.py index c31d7205..b7416ac7 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -451,12 +451,34 @@ async def test_query_list_search_filter_and_html(): assert html_response.status_code == 200 assert "Demo query 02" in html_response.text assert "Demo query 01" not in html_response.text + assert 'class="query-list-results"' in html_response.text + assert "Mode" in html_response.text + assert 'type="radio" name="is_published" value="1"' in html_response.text assert json_response.json()["queries"][0]["name"] == "demo_query_02" assert [query["name"] for query in filtered_response.json()["queries"]] == [ "private_query" ] +@pytest.mark.asyncio +async def test_query_list_html_defaults_to_twenty_and_shows_pagination(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_html_pagination", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 25) + + response = await ds.client.get("/data/-/queries", actor={"id": "root"}) + json_response = await ds.client.get("/data/-/queries.json", actor={"id": "root"}) + + assert response.status_code == 200 + assert response.text.count('aria-label="Query pagination"') == 1 + assert "Demo query 20" in response.text + assert "Demo query 21" not in response.text + assert 'href="/data/-/queries?_next=' in response.text + assert len(json_response.json()["queries"]) == 25 + + @pytest.mark.asyncio async def test_global_query_list_api_and_html(): ds = Datasette(memory=True) @@ -519,7 +541,8 @@ async def test_global_query_list_api_and_html(): ("beta", "beta_first"), ] assert html_response.status_code == 200 - assert 'href="/beta">beta:' in html_response.text + assert 'Database' in html_response.text + assert 'class="query-list-database" href="/beta">beta' in html_response.text assert "Beta first" in html_response.text assert "Alpha first" not in html_response.text From f1dd86ebfb01644fead19f9f007b9b76f863d72e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 14:05:26 -0700 Subject: [PATCH 704/767] Tweak URL designs of new endpoints --- datasette/app.py | 6 +++--- datasette/templates/database.html | 2 +- datasette/templates/execute_write.html | 2 +- datasette/templates/query.html | 2 +- datasette/templates/query_create.html | 2 +- docs/json_api.rst | 6 +++--- queries-plan.md | 4 ++-- tests/test_html.py | 4 ++-- tests/test_queries.py | 22 +++++++++++----------- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 90e41521..232aa0cf 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2745,11 +2745,11 @@ class Datasette: ) add_route( QueryInsertView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/-/insert$", + r"/(?P[^\/\.]+)/-/queries/insert$", ) add_route( ExecuteWriteAnalyzeView.as_view(self), - r"/(?P[^\/\.]+)/-/execute-write/-/analyze$", + r"/(?P[^\/\.]+)/-/execute-write/analyze$", ) add_route( ExecuteWriteView.as_view(self), @@ -2761,7 +2761,7 @@ class Datasette: ) add_route( QueryParametersView.as_view(self), - r"/(?P[^\/\.]+)/-/query/-/parameters$", + r"/(?P[^\/\.]+)/-/query/parameters$", ) add_route( wrap_view(QueryView, self), diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 0c9ec94c..62f9c620 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -26,7 +26,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if allow_execute_sql %} -
+

Custom SQL query

{% set parameter_names = [] %} diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 9b522f66..46f58c3b 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -95,7 +95,7 @@

{{ execution_message }}{% for link in execution_links %} {{ link.label }}{% endfor %}

{% endif %} - + {% if write_template_tables %}
diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 3bcc7178..f74d21f1 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -37,7 +37,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index fb2599d2..3c027def 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -17,7 +17,7 @@

Create query

- +


diff --git a/docs/json_api.rst b/docs/json_api.rst index 91ed5306..dd54c459 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -525,7 +525,7 @@ Creating saved queries in the UI Creating saved queries ~~~~~~~~~~~~~~~~~~~~~~ -``POST //-/queries/-/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. +``POST //-/queries/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. .. _QueryParametersView: .. _ExecuteWriteView: @@ -534,13 +534,13 @@ Creating saved queries Executing write SQL ~~~~~~~~~~~~~~~~~~~ -``GET //-/query/-/parameters?sql=...`` returns the named parameters used by a SQL query. This requires ``execute-sql`` for the database. +``GET //-/query/parameters?sql=...`` returns the named parameters used by a SQL query. This requires ``execute-sql`` for the database. ``GET //-/execute-write`` displays a form for executing writable SQL. A ``?sql=`` query string pre-populates the form without executing it. ``POST //-/execute-write`` executes writable SQL. This requires ``execute-write-sql`` for the database plus the relevant table-level write permissions. -``GET //-/execute-write/-/analyze?sql=...`` returns the derived parameters plus the write operations that SQL would need in order to execute. +``GET //-/execute-write/analyze?sql=...`` returns the derived parameters plus the write operations that SQL would need in order to execute. .. _QueryDefinitionView: diff --git a/queries-plan.md b/queries-plan.md index a708e887..72427df2 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -211,7 +211,7 @@ JSON endpoints should follow Datasette's existing write API style: use `POST` pl Endpoints: - `GET /-/queries` and `GET /{database}/-/queries` show searchable HTML query browsers. `GET /-/queries.json` lists query definitions across every database the actor can view; `GET /{database}/-/queries.json` scopes that list to one database. Both JSON endpoints use cursor pagination with `_next` and `_size`. -- `POST /{database}/-/queries/-/insert` creates a query. +- `POST /{database}/-/queries/insert` creates a query. - `GET /{database}/{query}/-/definition` returns one query definition without executing it. - `POST /{database}/{query}/-/update` updates one query. - `POST /{database}/{query}/-/delete` deletes one query. @@ -388,7 +388,7 @@ The read methods should reconstruct the existing dictionary shape used by query On `/{database}/-/query`, if the actor has both `execute-sql` and `insert-query`, show a save control for valid read-only SQL. That page already executes read-only arbitrary SQL, so the first UI can stay read-only even though the JSON API can accept writable SQL after `Database.analyze_sql()` validation. -The save form should call `POST /{database}/-/queries/-/insert` and default to `is_published=false`. +The save form should call `POST /{database}/-/queries/insert` and default to `is_published=false`. If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. diff --git a/tests/test_html.py b/tests/test_html.py index b49391a6..8cda6dba 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -329,7 +329,7 @@ async def test_query_parameter_form_fields(ds_client): ' ' in response.text ) - assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text assert 'id="sql-parameters-section"' in response.text assert "setupSqlParameterRefresh" in response.text response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") @@ -344,7 +344,7 @@ async def test_query_parameter_form_fields(ds_client): async def test_database_page_sql_parameter_refresh_markup(ds_client): response = await ds_client.get("/fixtures") assert response.status_code == 200 - assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text assert 'id="sql-parameters-section"' in response.text assert "setupSqlParameterRefresh" in response.text diff --git a/tests/test_queries.py b/tests/test_queries.py index b7416ac7..57920584 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -356,7 +356,7 @@ async def test_query_insert_api_creates_read_only_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={ "query": { @@ -568,7 +568,7 @@ async def test_query_insert_api_publish_requires_publish_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "writer"}, json={"query": {"name": "public", "sql": "select 1", "is_published": True}}, ) @@ -586,7 +586,7 @@ async def test_query_insert_api_creates_writable_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={ "query": { @@ -603,7 +603,7 @@ async def test_query_insert_api_creates_writable_query(): assert query["parameters"] == ["name"] bad_response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={ "query": { @@ -671,7 +671,7 @@ async def test_query_insert_api_rejects_magic_parameters(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={"query": {"name": "magic", "sql": "select :_actor_id"}}, ) @@ -742,7 +742,7 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'data-sql-template="insert"' in response.text assert 'data-sql-template="update"' in response.text assert 'data-sql-template="delete"' in response.text - assert 'data-analyze-url="/data/-/execute-write/-/analyze"' in response.text + assert 'data-analyze-url="/data/-/execute-write/analyze"' in response.text assert 'addEventListener("paste"' in response.text assert "setupSqlParameterRefresh" in response.text assert '' in response.text @@ -771,12 +771,12 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): await ds.invoke_startup() response = await ds.client.get( - "/data/-/execute-write/-/analyze", + "/data/-/execute-write/analyze", actor={"id": "root"}, params={"sql": "insert into dogs (name) values (:name)"}, ) read_only_response = await ds.client.get( - "/data/-/execute-write/-/analyze", + "/data/-/execute-write/analyze", actor={"id": "root"}, params={"sql": "select * from dogs where name = :name"}, ) @@ -818,19 +818,19 @@ async def test_query_parameters_endpoint_uses_get_sql_only(): await ds.invoke_startup() response = await ds.client.get( - "/data/-/query/-/parameters", + "/data/-/query/parameters", actor={"id": "root"}, params={ "sql": "select * from dogs where name = :name and id = :id", }, ) permission_denied_response = await ds.client.get( - "/data/-/query/-/parameters", + "/data/-/query/parameters", actor={"id": "not-root"}, params={"sql": "select * from dogs where name = :name"}, ) magic_parameter_response = await ds.client.get( - "/data/-/query/-/parameters", + "/data/-/query/parameters", actor={"id": "root"}, params={"sql": "select :_actor_id"}, ) From 4a1a4d7807fb99203b9053b6d270b265df61f0af Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 11:59:49 -0700 Subject: [PATCH 705/767] Query is_trusted and is_private properties Refs https://github.com/simonw/datasette/issues/2735#issuecomment-4547270516 Diff explanation: https://gist.github.com/simonw/1e4de6c4b041a51968eb273ee96dec1f --- datasette/app.py | 39 ++-- datasette/default_actions.py | 7 - datasette/default_permissions/defaults.py | 100 +++++---- datasette/templates/query_create.html | 4 +- datasette/templates/query_list.html | 65 +++++- datasette/utils/internal_db.py | 3 +- datasette/views/database.py | 79 ++++--- docs/authentication.rst | 10 - docs/internals.rst | 3 +- queries-plan.md | 84 ++++---- tests/test_queries.py | 245 ++++++++++++++++++---- 11 files changed, 421 insertions(+), 218 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 232aa0cf..3329ee7e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -618,7 +618,8 @@ class Datasette: fragment=query_config.get("fragment"), parameters=query_config.get("params"), is_write=bool(query_config.get("write")), - is_published=bool(query_config.get("is_published")), + is_private=bool(query_config.get("is_private")), + is_trusted=bool(query_config.get("is_trusted", True)), source="config", on_success_message=query_config.get("on_success_message"), on_success_message_sql=query_config.get("on_success_message_sql"), @@ -1084,7 +1085,8 @@ class Datasette: "parameters": parameters, "is_write": is_write, "write": is_write, - "is_published": bool(row["is_published"]), + "is_private": bool(row["is_private"]), + "is_trusted": bool(row["is_trusted"]), "source": row["source"], "owner_id": row["owner_id"], "on_success_message": options.get("on_success_message"), @@ -1119,7 +1121,8 @@ class Datasette: fragment=None, parameters=None, is_write=False, - is_published=False, + is_private=False, + is_trusted=False, source="plugin", owner_id=None, on_success_message=None, @@ -1144,8 +1147,8 @@ class Datasette: sql_statement = """ INSERT INTO queries ( database_name, name, sql, title, description, description_html, - options, parameters, is_write, is_published, source, owner_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + options, parameters, is_write, is_private, is_trusted, source, owner_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ if replace: sql_statement += """ @@ -1157,7 +1160,8 @@ class Datasette: options = excluded.options, parameters = excluded.parameters, is_write = excluded.is_write, - is_published = excluded.is_published, + is_private = excluded.is_private, + is_trusted = excluded.is_trusted, source = excluded.source, owner_id = excluded.owner_id, updated_at = CURRENT_TIMESTAMP @@ -1174,7 +1178,8 @@ class Datasette: options_json, parameters_json, int(bool(is_write)), - int(bool(is_published)), + int(bool(is_private)), + int(bool(is_trusted)), source, owner_id, ], @@ -1193,7 +1198,8 @@ class Datasette: fragment=UNCHANGED, parameters=UNCHANGED, is_write=UNCHANGED, - is_published=UNCHANGED, + is_private=UNCHANGED, + is_trusted=UNCHANGED, source=UNCHANGED, owner_id=UNCHANGED, on_success_message=UNCHANGED, @@ -1209,7 +1215,8 @@ class Datasette: "description_html": description_html, "parameters": parameters, "is_write": is_write, - "is_published": is_published, + "is_private": is_private, + "is_trusted": is_trusted, "source": source, "owner_id": owner_id, } @@ -1227,7 +1234,7 @@ class Datasette: for field, value in fields.items(): if value is UNCHANGED: continue - if field in {"is_write", "is_published"}: + if field in {"is_write", "is_private", "is_trusted"}: value = int(bool(value)) elif field == "parameters": value = json.dumps(list(value or [])) @@ -1300,7 +1307,8 @@ class Datasette: cursor=None, q=None, is_write=None, - is_published=None, + is_private=None, + is_trusted=None, source=None, owner_id=None, include_private=False, @@ -1372,9 +1380,12 @@ class Datasette: if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) - if is_published is not None: - where_clauses.append("q.is_published = :query_is_published") - params["query_is_published"] = int(bool(is_published)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) if source is not None: where_clauses.append("q.source = :query_source") params["query_source"] = source diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 6787b80e..6a1f77b8 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -68,13 +68,6 @@ def register_actions(): resource_class=DatabaseResource, also_requires="execute-sql", ), - Action( - name="publish-query", - abbr="pq", - description="Publish saved queries for actors without execute-sql", - resource_class=DatabaseResource, - also_requires="insert-query", - ), # Table-level actions (child-level) Action( name="view-table", diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 58deea01..dfd8d3e9 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -26,6 +26,32 @@ DEFAULT_ALLOW_ACTIONS = frozenset( ) +def _configured_query_restriction_selects(datasette: "Datasette") -> tuple[list[str], dict]: + selects = [] + params = {} + for index, (database_name, db_config) in enumerate( + ((datasette.config or {}).get("databases") or {}).items() + ): + for query_name, query_config in (db_config.get("queries") or {}).items(): + if isinstance(query_config, dict) and query_config.get("is_private"): + continue + parent_param = f"query_config_parent_{index}_{len(selects)}" + child_param = f"query_config_child_{index}_{len(selects)}" + selects.append( + f""" + SELECT :{parent_param} AS parent, :{child_param} AS child + WHERE NOT EXISTS ( + SELECT 1 FROM queries + WHERE database_name = :{parent_param} + AND name = :{child_param} + ) + """ + ) + params[parent_param] = database_name + params[child_param] = query_name + return selects, params + + @hookimpl(specname="permission_resources_sql") async def default_allow_sql_check( datasette: "Datasette", @@ -93,61 +119,45 @@ async def default_query_permissions_sql( if action != "view-query": return None - execute_sql = await datasette.allowed_resources_sql( - action="execute-sql", actor=actor - ) - sql = execute_sql.sql - params = {} - for key, value in execute_sql.params.items(): - new_key = f"query_execute_sql_{key}" - sql = sql.replace(f":{key}", f":{new_key}") - params[new_key] = value - - trusted_writable_sql = "" + params = {"query_owner_id": actor_id} + rule_sqls = [] if not datasette.default_deny: - trusted_writable_sql = """ - UNION ALL + rule_sqls.append( + """ SELECT database_name AS parent, name AS child, 1 AS allow, - 'trusted writable query' AS reason + 'non-private query' AS reason FROM queries - WHERE is_write = 1 - AND source IN ('config', 'plugin') - """ + WHERE is_private = 0 + """ + ) - user_writable_sql = "" if actor_id is not None: - params["query_owner_id"] = actor_id - user_writable_sql = """ - UNION ALL + rule_sqls.append( + """ SELECT database_name AS parent, name AS child, 1 AS allow, 'query owner' AS reason FROM queries - WHERE is_write = 1 - AND source = 'user' - AND owner_id = :query_owner_id + WHERE owner_id = :query_owner_id + """ + ) + + config_restriction_selects, config_restriction_params = ( + _configured_query_restriction_selects(datasette) + ) + + restriction_sqls = [ """ + SELECT database_name AS parent, name AS child + FROM queries + WHERE is_private = 0 + OR owner_id = :query_owner_id + """ + ] + restriction_sqls.extend(config_restriction_selects) + params.update(config_restriction_params) return PermissionSQL( - sql=f""" - WITH execute_sql_allowed AS ( - {sql} - ) - SELECT database_name AS parent, name AS child, 1 AS allow, - 'published query' AS reason - FROM queries - WHERE is_write = 0 - AND is_published = 1 - UNION ALL - SELECT q.database_name AS parent, q.name AS child, 1 AS allow, - 'execute-sql allows query' AS reason - FROM queries q - JOIN execute_sql_allowed es - ON es.parent = q.database_name - AND es.child IS NULL - WHERE q.is_write = 0 - AND q.is_published = 0 - {trusted_writable_sql} - {user_writable_sql} - """, + sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, + restriction_sql="\nUNION ALL\n".join(restriction_sqls), params=params, ) diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index 3c027def..686d971e 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -27,9 +27,7 @@

- {% if can_publish %} -

- {% endif %} +

{% if sql and analysis_is_write %}

Execute write SQL

{% endif %} diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index dbd607ab..25259b3d 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -73,7 +73,7 @@ border-collapse: collapse; font-size: 0.9rem; margin: 0.25rem 0 1rem; - min-width: 36rem; + min-width: 42rem; width: 100%; } .query-list-results th, @@ -100,6 +100,16 @@ font-size: 0.78rem; margin: 0.15rem 0 0; } +.query-list-owner { + color: #39445a; + font-family: var(--font-monospace, monospace); + white-space: nowrap; +} +.query-list-flags { + display: flex; + flex-wrap: wrap; + gap: 0.3rem; +} .query-list-pill { background-color: #eef1f5; border: 1px solid #d7dde5; @@ -116,15 +126,36 @@ background-color: #fff4db; border-color: #e2b64e; } -.query-list-pill-published { +.query-list-pill-public { background-color: #e7f5ec; border-color: #9ecfab; color: #267a3e; } -.query-list-pill-unpublished { +.query-list-pill-private { background-color: #f7edf0; border-color: #dbb8c1; } +.query-list-pill-trusted { + background-color: #e7f5ec; + border-color: #9ecfab; + color: #267a3e; +} +.query-list-empty { + color: #6b7280; +} +.query-list-footnotes { + border-top: 1px solid #d7dde5; + color: #4f5b6d; + font-size: 0.82rem; + margin: 0.35rem 0 1rem; + padding-top: 0.55rem; +} +.query-list-footnotes p { + margin: 0.25rem 0; +} +.query-list-footnotes .query-list-pill { + margin-right: 0.35rem; +} .query-list-pagination a { border: 1px solid #007bff; border-radius: 0.25rem; @@ -177,10 +208,10 @@
- Publication - - - + Visibility + + +
@@ -191,8 +222,8 @@ {% if show_database %}{% endif %} - - + + @@ -205,12 +236,24 @@ {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} {% if query.description %}

{{ query.description }}

{% endif %} - - + + {% endfor %}
DatabaseQueryModePublicationOwnerFlags
{% if query.is_write %}Writable{% else %}Read-only{% endif %}{% if query.is_published %}Published{% else %}Unpublished{% endif %}{% if query.owner_id is not none %}{{ query.owner_id }}{% else %}-{% endif %} + + {% if query.is_write %}Writable{% else %}Read-only{% endif %} + {% if query.is_private %}Private{% endif %} + {% if query.is_trusted %}Trusted{% endif %} + +
+ {% if show_private_note or show_trusted_note %} +
+ {% if show_private_note %}

PrivateOnly the owning actor can view this query.

{% endif %} + {% if show_trusted_note %}

TrustedExecution skips the usual SQL and write permission checks after view-query allows access.

{% endif %} +
+ {% endif %} {% else %}

No queries found.

{% endif %} diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 9c693b0a..bf172667 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -123,7 +123,8 @@ async def initialize_metadata_tables(db): options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/datasette/views/database.py b/datasette/views/database.py index 3c660bc7..91e9c350 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -428,7 +428,7 @@ _query_fields = { "fragment", "parameters", "params", - "is_published", + "is_private", "on_success_message", "on_success_message_sql", "on_success_redirect", @@ -571,7 +571,7 @@ async def _check_query_name(db, name, *, existing=False): raise QueryValidationError("Query name conflicts with a table or view") -async def _analyze_user_query(datasette, db, sql, *, actor, is_published): +async def _analyze_user_query(datasette, db, sql, *, actor): if not sql or not isinstance(sql, str): raise QueryValidationError("SQL is required") derived = _derived_query_parameters(sql) @@ -583,8 +583,6 @@ async def _analyze_user_query(datasette, db, sql, *, actor, is_published): is_write = _analysis_is_write(analysis) if is_write: - if is_published: - raise QueryValidationError("Writable queries cannot be published") try: await datasette.ensure_query_write_permissions( db.name, sql, actor=actor, analysis=analysis @@ -680,6 +678,26 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis +async def _ensure_stored_query_execution_permissions(datasette, db, query, actor): + if query.get("is_trusted"): + return + if query.get("write"): + await datasette.ensure_permission( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=actor, + ) + await datasette.ensure_query_write_permissions( + db.name, query["sql"], actor=actor + ) + else: + await datasette.ensure_permission( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=actor, + ) + + async def _execute_write_analysis_data(datasette, db, sql, actor): parameter_names = [] analysis_rows = [] @@ -752,7 +770,7 @@ async def _inserted_row_url(datasette, db, analysis, cursor): def _apply_query_data_types(data): typed = dict(data) - for key in ("hide_sql", "is_published"): + for key in ("hide_sql", "is_private"): if key in typed: typed[key] = _as_bool(typed[key]) return typed @@ -769,20 +787,12 @@ async def _prepare_query_create(datasette, request, db, data): if await datasette.get_query(db.name, name) is not None: raise QueryValidationError("Query already exists") - is_published = _as_bool(data.get("is_published")) is_write, derived, analysis = await _analyze_user_query( datasette, db, data.get("sql"), actor=request.actor, - is_published=is_published, ) - if is_published and not await datasette.allowed( - action="publish-query", - resource=DatabaseResource(db.name), - actor=request.actor, - ): - raise QueryValidationError("Permission denied: need publish-query", status=403) if not is_write and any(data.get(field) for field in _query_write_fields): raise QueryValidationError("Writable query fields require writable SQL") @@ -800,7 +810,8 @@ async def _prepare_query_create(datasette, request, db, data): "fragment": data.get("fragment"), "parameters": parameters, "is_write": is_write, - "is_published": is_published, + "is_private": _as_bool(data.get("is_private", True)), + "is_trusted": False, "source": "user", "owner_id": _actor_id(request.actor), "on_success_message": data.get("on_success_message"), @@ -819,7 +830,6 @@ async def _prepare_query_update(datasette, request, db, existing, update): update = _apply_query_data_types(update) sql = update.get("sql", existing["sql"]) - is_published = update.get("is_published", existing["is_published"]) query_is_write = existing["is_write"] derived = _derived_query_parameters(sql) parameters = None @@ -830,19 +840,7 @@ async def _prepare_query_update(datasette, request, db, existing, update): db, sql, actor=request.actor, - is_published=is_published, ) - elif is_published and query_is_write: - raise QueryValidationError("Writable queries cannot be published") - if is_published and not existing["is_published"]: - if not await datasette.allowed( - action="publish-query", - resource=DatabaseResource(db.name), - actor=request.actor, - ): - raise QueryValidationError( - "Permission denied: need publish-query", status=403 - ) if "parameters" in update or "params" in update: parameters = _coerce_query_parameters( @@ -864,7 +862,7 @@ async def _prepare_query_update(datasette, request, db, existing, update): "fragment": update.get("fragment"), "parameters": parameters, "is_write": query_is_write, - "is_published": is_published, + "is_private": update.get("is_private"), "on_success_message": update.get("on_success_message"), "on_success_message_sql": update.get("on_success_message_sql"), "on_success_redirect": update.get("on_success_redirect"), @@ -1141,8 +1139,8 @@ class QueryListView(BaseView): default=20 if format_ == "html" else 50, ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") - is_published = _as_optional_bool( - request.args.get("is_published"), "is_published" + is_private = _as_optional_bool( + request.args.get("is_private"), "is_private" ) except QueryValidationError as ex: return _error([ex.message], ex.status) @@ -1154,7 +1152,7 @@ class QueryListView(BaseView): cursor=request.args.get("_next"), q=request.args.get("q") or None, is_write=is_write, - is_published=is_published, + is_private=is_private, source=request.args.get("source") or None, owner_id=request.args.get("owner_id") or None, include_private=True, @@ -1186,12 +1184,14 @@ class QueryListView(BaseView): "next_url": next_url, "has_more": page["has_more"], "limit": page["limit"], + "show_private_note": any(query["is_private"] for query in page["queries"]), + "show_trusted_note": any(query["is_trusted"] for query in page["queries"]), "query_list_path": query_list_path, "show_database": database is None, "filters": { "q": request.args.get("q") or "", "is_write": request.args.get("is_write") or "", - "is_published": request.args.get("is_published") or "", + "is_private": request.args.get("is_private") or "", "source": request.args.get("source") or "", "owner_id": request.args.get("owner_id") or "", }, @@ -1255,11 +1255,6 @@ class QueryCreateView(BaseView): "database_color": db.color, "sql": sql, "parameter_names": parameter_names, - "can_publish": await self.ds.allowed( - action="publish-query", - resource=DatabaseResource(db.name), - actor=request.actor, - ), "analysis_error": analysis_error, "analysis_rows": analysis_rows, "analysis_is_write": bool( @@ -1435,9 +1430,9 @@ class QueryView(View): ): raise Forbidden("You do not have permission to view this query") - if canned_query.get("write") and canned_query.get("source") == "user": - await datasette.ensure_query_write_permissions( - db.name, canned_query["sql"], actor=request.actor + if canned_query.get("write"): + await _ensure_stored_query_execution_permissions( + datasette, db, canned_query, request.actor ) # If database is immutable, return an error @@ -1558,6 +1553,10 @@ class QueryView(View): ) if not visible: raise Forbidden("You do not have permission to view this query") + if not canned_query_write: + await _ensure_stored_query_execution_permissions( + datasette, db, canned_query, request.actor + ) else: await datasette.ensure_permission( diff --git a/docs/authentication.rst b/docs/authentication.rst index b6a4cb7e..6e835c8d 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1299,16 +1299,6 @@ insert-query Actor is allowed to create saved queries in a database. -``resource`` - ``datasette.resources.DatabaseResource(database)`` - ``database`` is the name of the database (string) - -.. _actions_publish_query: - -publish-query -------------- - -Actor is allowed to publish a saved read-only query so actors without ``execute-sql`` can run it. - ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) diff --git a/docs/internals.rst b/docs/internals.rst index b5da7cbf..c76de487 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2158,7 +2158,8 @@ The internal database schema is as follows: options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/queries-plan.md b/queries-plan.md index 72427df2..f4b8049c 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -13,9 +13,9 @@ Terminology change: these are now "queries", not "canned queries". Legacy code a - Internal table name: `queries`. - Query definitions should use real columns, not a JSON blob for all options. - Query parameter names live in a `parameters` text column as a JSON array. No default values for parameters in this pass. -- No `queries_database_is_published_idx` index. -- User-created queries require `execute-sql` and `insert-query` on the database. Writable queries additionally require matching table write permissions discovered by `Database.analyze_sql()`. -- `publish-query` is the permission for creating or updating a query so users without `execute-sql` can execute it. +- No separate index is needed for the privacy/trust flags yet. +- User-created queries require `execute-sql` and `insert-query` on the database. They default to private, and writable queries additionally require matching table write permissions discovered by `Database.analyze_sql()`. +- Configured queries default to trusted, which means actors who can view them can execute them without also holding `execute-sql` or the relevant write permissions. Config can opt out with `is_trusted: false`. - Add `update-query` and `delete-query`, so administrators can manage queries created by other users. - Remove the old `canned_queries()` hook from core. If we want compatibility later, build a separate `datasette-old-canned-queries` plugin. - Writable user-created queries can be supported using `Database.analyze_sql()`, provided we fail closed when analysis cannot prove the required permissions. @@ -45,7 +45,8 @@ CREATE TABLE IF NOT EXISTS queries ( options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -64,11 +65,12 @@ Column notes: - Less common presentation and writable-query behavior lives in `options`, stored as a JSON object. That covers `hide_sql`, `fragment`, `on_success_message`, `on_success_message_sql`, `on_success_redirect`, `on_error_message`, and `on_error_redirect`. - `parameters` is a JSON array of parameter names, stored as text. This preserves explicit parameter order, but does not support labels or default values. - Existing writable query behavior gets `is_write` as a column. Success/error messages, success/error redirects, and `on_success_message_sql` are stored in `options`. -- `is_published` only applies to read-only queries. A writable query can still be public through explicit `view-query` permissions, but the "publish for users without execute-sql" shortcut should be read-only. +- `is_private` means the query is only visible to its owning actor. This is enforced as a permission restriction, so broader `view-query` grants do not expose private rows. +- `is_trusted` means execution skips the usual `execute-sql` or write-permission checks after `view-query` has allowed access. - `source` distinguishes `user`, `config`, and `plugin` rows. - `owner_id` is the actor id for user-created rows. It is `NULL` for config/plugin rows. -No separate index is needed on `(database_name, name)` because the primary key already creates one. Do not add a `queries_database_is_published_idx` index for now. +No separate index is needed on `(database_name, name)` because the primary key already creates one. `QueryResource.resources_sql()` can become: @@ -104,7 +106,6 @@ Remove the old `canned_queries()` hookspec and all core calls to it. If compatib Add core actions: - `insert-query`, database-level, for creating queries in a database. -- `publish-query`, database-level, for marking read-only queries as executable by actors who lack `execute-sql`. - `update-query`, query-level, for modifying existing query definitions. - `delete-query`, query-level, for deleting existing query definitions. @@ -114,17 +115,11 @@ User-created query creation requires: - `insert-query` on `DatabaseResource(database)` - If analysis shows the query is writable, the table-level write permissions described in the writable query section. -Setting `is_published=1` requires: - -- `publish-query` on `DatabaseResource(database)` -- The query must be read-only according to `Database.analyze_sql()`. - Updating an existing query requires: - `update-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. - If the SQL changes, also require `execute-sql` on the database. - If the changed SQL is writable, also require the table-level write permissions described in the writable query section. -- If `is_published` changes from `0` to `1`, also require `publish-query` on the database. Deleting an existing query requires: @@ -133,18 +128,18 @@ Deleting an existing query requires: Default owner permissions: - For `source='user' AND owner_id = actor.id`, grant `update-query` and `delete-query`. -- Do not automatically grant execution if the user no longer has the execution permission described below. +- For `source='user' AND owner_id = actor.id`, grant `view-query`. If the query is private, restriction SQL ensures no other actor sees it through a broader grant. ## Executing queries Default execution rule for read-only queries: -- If `is_published=0`, the actor needs `execute-sql` on the database. -- If `is_published=1`, the actor can execute the query without `execute-sql`. +- If `is_trusted=0`, the actor needs `execute-sql` on the database. +- If `is_trusted=1`, the actor can execute the query without `execute-sql`, provided `view-query` allows access. Default execution rule for user-created writable queries: -- `is_published` must be `0`. +- `is_trusted` must be `0`. - The actor must have `view-query`. - The actor must currently have every write permission required by fresh `Database.analyze_sql()` results for the query SQL. @@ -152,14 +147,14 @@ Implementation: - Remove `view-query` from the broad `DEFAULT_ALLOW_ACTIONS` set. - Replace it with query-aware default `view-query` permission SQL. -- For `is_published=1 AND is_write=0`, emit a child-level `view-query` allow. -- For `is_published=0 AND is_write=0`, emit child-level `view-query` allows for queries whose parent database is in the actor's `execute-sql` allowed resources. -- For `is_write=1 AND source='user'`, emit `view-query` only for the owner or actors with explicit `view-query` permission, then have `QueryView` perform the fresh analysis/table-permission check before execution. -- For trusted writable queries, preserve current behavior by emitting child-level `view-query` allows for `is_write=1 AND source IN ('config', 'plugin')` when Datasette is not running with `--default-deny`. +- Emit default `view-query` allows for non-private rows when Datasette is not running with `--default-deny`. +- Emit default `view-query` allows for the owning actor. +- Use `restriction_sql` to limit private rows to their owner even when broader `view-query` permissions exist. +- Have `QueryView` perform the fresh `execute-sql` or table-permission check before execution unless the row has `is_trusted=1`. -For read-only queries this keeps `QueryView` simple: it checks `view-query` for the query resource, and the default permission hook encodes the relationship with `execute-sql`. User-created writable queries need one additional runtime permission check because their required table permissions are derived from fresh SQL analysis. +For read-only queries this keeps `QueryView` explicit: it checks `view-query` for the query resource, then checks `execute-sql` unless the row is trusted. User-created writable queries need one additional runtime permission check because their required table permissions are derived from fresh SQL analysis. -Explicit deny rules should still be able to block a published query. +Explicit deny rules should still be able to block a query, and `--default-deny` still blocks trusted queries unless something grants `view-query`. ## Writable queries @@ -180,7 +175,7 @@ Validation flow for user-created queries: 1. Derive named parameters from the SQL and pass harmless placeholder values into `db.analyze_sql()` so SQLite can prepare statements with bindings. 2. If analysis raises a SQLite error, reject the query. 3. If every table access is `read`, treat the query as read-only and require `execute-sql` plus `insert-query`/`update-query` as described above. -4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `is_published=0`. +4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `is_trusted=0`. 5. Reject writable user-created queries that access a database other than the database they are being saved against, until `analyze_sql()` can reliably map attached SQLite schemas back to Datasette database names. 6. For every write access returned by analysis, require the corresponding permission on `TableResource(access.database, access.table)`: - `insert` -> `insert-row` @@ -200,7 +195,7 @@ Fail closed cases for user-created writable queries: - Analysis reports any write operation that cannot be mapped to a Datasette table resource. - Analysis reports writes outside the target database. - The actor lacks any required table write permission. -- `is_published=1` is requested. +- `is_trusted=1` is requested through the user-facing API. This gives us writable user-created queries without letting `execute-sql` alone become a path to create arbitrary write endpoints. @@ -225,7 +220,7 @@ Create request: "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers", "description": "Highest revenue customers", - "is_published": false, + "is_private": true, "parameters": ["region"] } } @@ -242,7 +237,8 @@ Successful create returns `201` and the created query definition: "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers", "description": "Highest revenue customers", - "is_published": false, + "is_private": true, + "is_trusted": false, "parameters": ["region"] } } @@ -254,7 +250,7 @@ Update request, imitating `RowUpdateView`: { "update": { "title": "Top customers by revenue", - "is_published": true + "is_private": false }, "return": true } @@ -270,7 +266,8 @@ Successful update returns `{"ok": true}` by default. With `"return": true`, retu "name": "top_customers", "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers by revenue", - "is_published": true + "is_private": false, + "is_trusted": false } } ``` @@ -317,7 +314,8 @@ await datasette.add_query( fragment=None, parameters=None, is_write=False, - is_published=False, + is_private=False, + is_trusted=False, source="plugin", owner_id=None, on_success_message=None, @@ -340,7 +338,8 @@ await datasette.update_query( fragment=UNCHANGED, parameters=UNCHANGED, is_write=UNCHANGED, - is_published=UNCHANGED, + is_private=UNCHANGED, + is_trusted=UNCHANGED, source=UNCHANGED, owner_id=UNCHANGED, on_success_message=UNCHANGED, @@ -360,7 +359,8 @@ await datasette.list_queries( cursor=None, q=None, is_write=None, - is_published=None, + is_private=None, + is_trusted=None, source=None, owner_id=None, ) @@ -382,15 +382,13 @@ For column-backed fields, `None` should write SQL `NULL`. For option fields, `No Implementation detail: build the `UPDATE` statement dynamically from fields whose value is not `UNCHANGED`, validate non-nullable fields before writing, and update `updated_at` whenever at least one field changes. -The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `is_published`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. +The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `is_private`, `is_trusted`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. ## Query page save UI On `/{database}/-/query`, if the actor has both `execute-sql` and `insert-query`, show a save control for valid read-only SQL. That page already executes read-only arbitrary SQL, so the first UI can stay read-only even though the JSON API can accept writable SQL after `Database.analyze_sql()` validation. -The save form should call `POST /{database}/-/queries/insert` and default to `is_published=false`. - -If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. +The save form should call `POST /{database}/-/queries/insert` and default to `is_private=true`. On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. The global `/-/queries` page reuses the same interface and shows the database for each query. @@ -403,7 +401,7 @@ This page should require `execute-sql` and `insert-query` to access. It should p - Read-only - Writable -Read-only mode can share the same fields as the arbitrary SQL save flow: name, title, description, parameters, and optional published status if the actor has `publish-query`. +Read-only mode can share the same fields as the arbitrary SQL save flow: name, title, description, parameters, and privacy status. Writable mode should always run `Database.analyze_sql()` and show an analysis panel before saving: @@ -413,7 +411,7 @@ Writable mode should always run `Database.analyze_sql()` and show an analysis pa - whether the actor has that permission - source, when the operation comes from a trigger or view -The Save button should be disabled until analysis succeeds and every required table write permission is allowed. Writable mode should not show a publish control, because user-created writable queries cannot be published. +The Save button should be disabled until analysis succeeds and every required table write permission is allowed. The existing edit-SQL flow from query pages can continue to point back to arbitrary SQL. A later enhancement can add "update this query" when the actor owns it or has `update-query`. @@ -427,14 +425,16 @@ The existing edit-SQL flow from query pages can continue to point back to arbitr - `QueryResource.resources_sql()` returns rows from `queries`. - Database page and `/-/jump` list queries from the internal DB. - `view-query` is no longer globally default-allowed; default query permissions come from the query-aware hook. -- Unpublished read-only query requires `execute-sql` to execute. -- Published read-only query can be executed without `execute-sql`. -- Setting `is_published=true` requires `publish-query`. +- Private query is only visible to its owner, even when a broader `view-query` rule applies. +- Non-trusted read-only query requires `execute-sql` to execute. +- Trusted read-only query can be executed without `execute-sql` after `view-query` passes. +- Config queries default to trusted and can opt out with `is_trusted: false`. +- User API rejects client-supplied `is_trusted`. - User-created query requires both `execute-sql` and `insert-query`. - User-created writable query creation uses `Database.analyze_sql()` and requires matching `insert-row`, `update-row`, and/or `delete-row` permissions for every reported write access. - `/{database}/-/queries/-/create` provides the writable-query authoring UI with an analysis panel and disabled save until all required write permissions pass. - User-created writable query execution re-runs `Database.analyze_sql()` and re-checks table write permissions. -- User-created writable query cannot be published. +- User-created writable query cannot be trusted through the user API. - Query update uses `POST /{database}/{query}/-/update` with an `{"update": {...}}` body. - Query delete uses `POST /{database}/{query}/-/delete`. - There are no `PATCH` or HTTP `DELETE` routes for query management. diff --git a/tests/test_queries.py b/tests/test_queries.py index 57920584..c97b5733 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -15,7 +15,6 @@ async def add_numbered_queries(ds, database, count): "select {} as query_number".format(i), title="Demo query {:02d}".format(i), description="Seeded demo query number {:02d}".format(i), - is_published=True, source="user", owner_id="root", ) @@ -44,7 +43,8 @@ async def test_queries_internal_table_schema(): "options", "parameters", "is_write", - "is_published", + "is_private", + "is_trusted", "source", "owner_id", "created_at", @@ -67,7 +67,7 @@ async def test_add_get_and_remove_query(): hide_sql=True, fragment="chart", parameters=["region"], - is_published=True, + is_trusted=True, source="user", owner_id="alice", ) @@ -100,7 +100,8 @@ async def test_add_get_and_remove_query(): "parameters": ["region"], "is_write": False, "write": False, - "is_published": True, + "is_private": False, + "is_trusted": True, "source": "user", "owner_id": "alice", "on_success_message": None, @@ -161,7 +162,8 @@ async def test_update_query_only_updates_provided_fields(): assert query["params"] == [] assert query["on_success_redirect"] is None assert query["sql"] == "select 1" - assert query["is_published"] is False + assert query["is_private"] is False + assert query["is_trusted"] is False options_row = ( await ds.get_internal_database().execute( """ @@ -208,7 +210,8 @@ async def test_config_queries_imported_to_internal_table(): "parameters": ["name"], "is_write": False, "write": False, - "is_published": False, + "is_private": False, + "is_trusted": True, "source": "config", "owner_id": None, "on_success_message": None, @@ -232,30 +235,171 @@ async def test_query_resources_come_from_internal_table(): @pytest.mark.asyncio -async def test_unpublished_query_requires_execute_sql_but_published_does_not(): - ds = Datasette(memory=True, settings={"default_allow_sql": False}) +async def test_default_deny_blocks_view_query_even_for_trusted_query(): + ds = Datasette(memory=True, default_deny=True) ds.add_memory_database("query_permissions", name="data") await ds.invoke_startup() - await ds.add_query("data", "unpublished", "select 1", is_published=False) - await ds.add_query("data", "published", "select 1", is_published=True) + await ds.add_query("data", "trusted", "select 1", is_trusted=True) assert not await ds.allowed( - action="execute-sql", - resource=DatabaseResource("data"), + action="view-query", + resource=QueryResource("data", "trusted"), actor=None, ) + + +@pytest.mark.asyncio +async def test_private_query_restriction_blocks_broad_view_query_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-query": {"id": "*"}, + } + } + } + }, + ) + ds.add_memory_database("private_query_permissions", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "shared_report", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "alice"}, + ) assert not await ds.allowed( action="view-query", - resource=QueryResource("data", "unpublished"), - actor=None, + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, ) assert await ds.allowed( action="view-query", - resource=QueryResource("data", "published"), - actor=None, + resource=QueryResource("data", "shared_report"), + actor={"id": "bob"}, ) +@pytest.mark.asyncio +async def test_config_query_restriction_does_not_override_private_internal_query(): + ds = Datasette(memory=True, default_deny=True) + ds.add_memory_database("private_query_with_config_name", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + ds.config = { + "databases": { + "data": { + "permissions": {"view-query": {"id": "*"}}, + "queries": {"private_report": {"sql": "select 2"}}, + } + } + } + + assert not await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, + ) + + +@pytest.mark.asyncio +async def test_untrusted_shared_query_execution_requires_execute_sql(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "viewer"}, + "view-query": {"id": "viewer"}, + } + } + } + }, + ) + ds.add_memory_database("untrusted_query_execution", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "shared_report", + "select 1 as one", + is_private=False, + is_trusted=False, + source="user", + owner_id="alice", + ) + + denied = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) + assert denied.status_code == 403 + + ds.config["databases"]["data"]["permissions"]["execute-sql"] = {"id": "viewer"} + allowed = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) + assert allowed.status_code == 200 + assert allowed.json()["rows"] == [{"one": 1}] + + +@pytest.mark.asyncio +async def test_config_queries_are_trusted_by_default_but_can_opt_out(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-query": {"id": "viewer"}, + }, + "queries": { + "trusted_report": {"sql": "select 1 as one"}, + "untrusted_report": { + "sql": "select 2 as two", + "is_trusted": False, + }, + }, + } + } + }, + ) + ds.add_memory_database("trusted_query_config", name="data") + await ds.invoke_startup() + + trusted = await ds.client.get("/data/trusted_report.json", actor={"id": "viewer"}) + untrusted = await ds.client.get( + "/data/untrusted_report.json", actor={"id": "viewer"} + ) + + assert trusted.status_code == 200 + assert trusted.json()["rows"] == [{"one": 1}] + assert untrusted.status_code == 403 + + @pytest.mark.asyncio async def test_database_page_query_preview_is_limited(): ds = Datasette(memory=True) @@ -281,7 +425,6 @@ async def test_query_actions_are_registered(): assert ds.get_action("execute-write-sql").resource_class is DatabaseResource assert ds.get_action("insert-query").resource_class is DatabaseResource - assert ds.get_action("publish-query").resource_class is DatabaseResource assert ds.get_action("update-query").resource_class is QueryResource assert ds.get_action("delete-query").resource_class is QueryResource @@ -430,21 +573,33 @@ async def test_query_list_search_filter_and_html(): "private_query", "select 'private'", title="Private query", - is_published=False, + is_private=True, source="user", owner_id="root", ) + await ds.add_query( + "data", + "trusted_query", + "select 'trusted'", + title="Trusted query", + is_trusted=True, + source="config", + ) html_response = await ds.client.get( "/data/-/queries?q=02", actor={"id": "root"}, ) + flags_response = await ds.client.get( + "/data/-/queries", + actor={"id": "root"}, + ) json_response = await ds.client.get( "/data/-/queries.json?q=02", actor={"id": "root"}, ) filtered_response = await ds.client.get( - "/data/-/queries.json?is_published=0", + "/data/-/queries.json?is_private=1", actor={"id": "root"}, ) @@ -453,7 +608,22 @@ async def test_query_list_search_filter_and_html(): assert "Demo query 01" not in html_response.text assert 'class="query-list-results"' in html_response.text assert "Mode" in html_response.text - assert 'type="radio" name="is_published" value="1"' in html_response.text + assert 'type="radio" name="is_private" value="1"' in html_response.text + assert "Only the owning actor can view this query." not in html_response.text + assert ( + "Execution skips the usual SQL and write permission checks" + not in html_response.text + ) + assert flags_response.status_code == 200 + assert 'Owner' in flags_response.text + assert 'Flags' in flags_response.text + assert 'Mode' not in flags_response.text + assert 'class="query-list-owner">root' in flags_response.text + assert 'class="query-list-pill">Read-only' in flags_response.text + assert 'class="query-list-pill query-list-pill-private">Private' in flags_response.text + assert 'class="query-list-pill query-list-pill-trusted">Trusted' in flags_response.text + assert "Only the owning actor can view this query." in flags_response.text + assert "Execution skips the usual SQL and write permission checks" in flags_response.text assert json_response.json()["queries"][0]["name"] == "demo_query_02" assert [query["name"] for query in filtered_response.json()["queries"]] == [ "private_query" @@ -491,7 +661,6 @@ async def test_global_query_list_api_and_html(): "alpha_first", "select 1", title="Alpha first", - is_published=True, source="user", owner_id="root", ) @@ -500,7 +669,6 @@ async def test_global_query_list_api_and_html(): "alpha_second", "select 2", title="Alpha second", - is_published=True, source="user", owner_id="root", ) @@ -509,7 +677,6 @@ async def test_global_query_list_api_and_html(): "beta_first", "select 3", title="Beta first", - is_published=True, source="user", owner_id="root", ) @@ -548,7 +715,7 @@ async def test_global_query_list_api_and_html(): @pytest.mark.asyncio -async def test_query_insert_api_publish_requires_publish_query(): +async def test_query_insert_api_rejects_is_trusted(): ds = Datasette( memory=True, default_deny=True, @@ -564,17 +731,17 @@ async def test_query_insert_api_publish_requires_publish_query(): } }, ) - ds.add_memory_database("query_publish_api", name="data") + ds.add_memory_database("query_trusted_api", name="data") await ds.invoke_startup() response = await ds.client.post( "/data/-/queries/insert", actor={"id": "writer"}, - json={"query": {"name": "public", "sql": "select 1", "is_published": True}}, + json={"query": {"name": "trusted", "sql": "select 1", "is_trusted": True}}, ) - assert response.status_code == 403 - assert response.json()["errors"] == ["Permission denied: need publish-query"] + assert response.status_code == 400 + assert response.json()["errors"] == ["Invalid keys: is_trusted"] @pytest.mark.asyncio @@ -599,24 +766,10 @@ async def test_query_insert_api_creates_writable_query(): assert response.status_code == 201 query = response.json()["query"] assert query["is_write"] is True - assert query["is_published"] is False + assert query["is_private"] is True + assert query["is_trusted"] is False assert query["parameters"] == ["name"] - bad_response = await ds.client.post( - "/data/-/queries/insert", - actor={"id": "root"}, - json={ - "query": { - "name": "published_insert", - "sql": "insert into dogs (name) values (:name)", - "is_published": True, - } - }, - ) - - assert bad_response.status_code == 400 - assert bad_response.json()["errors"] == ["Writable queries cannot be published"] - @pytest.mark.asyncio async def test_query_update_and_delete_api(): @@ -1103,6 +1256,10 @@ async def test_user_writable_query_execution_rechecks_table_permissions(): config={ "databases": { "data": { + "permissions": { + "view-database": {"id": ["alice", "bob"]}, + "execute-write-sql": {"id": ["alice", "bob"]}, + }, "tables": { "dogs": { "permissions": { From 1cd162e9da48b924c289ec9343e9d801b51a89f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 12:07:30 -0700 Subject: [PATCH 706/767] Removed some no-longer-necessary code, simplified view-query is back in the default allow actions now. We have other mechanisms that work for controlling visibility, and the fact that queries default to running with the permissions of the actor makes this safe. --- datasette/default_permissions/defaults.py | 55 +++-------------------- tests/test_permissions.py | 9 +++- tests/test_queries.py | 39 ++++++++++++++++ 3 files changed, 51 insertions(+), 52 deletions(-) diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index dfd8d3e9..ed0a6d66 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -21,37 +21,12 @@ DEFAULT_ALLOW_ACTIONS = frozenset( "view-database", "view-database-download", "view-table", + "view-query", "execute-sql", } ) -def _configured_query_restriction_selects(datasette: "Datasette") -> tuple[list[str], dict]: - selects = [] - params = {} - for index, (database_name, db_config) in enumerate( - ((datasette.config or {}).get("databases") or {}).items() - ): - for query_name, query_config in (db_config.get("queries") or {}).items(): - if isinstance(query_config, dict) and query_config.get("is_private"): - continue - parent_param = f"query_config_parent_{index}_{len(selects)}" - child_param = f"query_config_child_{index}_{len(selects)}" - selects.append( - f""" - SELECT :{parent_param} AS parent, :{child_param} AS child - WHERE NOT EXISTS ( - SELECT 1 FROM queries - WHERE database_name = :{parent_param} - AND name = :{child_param} - ) - """ - ) - params[parent_param] = database_name - params[child_param] = query_name - return selects, params - - @hookimpl(specname="permission_resources_sql") async def default_allow_sql_check( datasette: "Datasette", @@ -121,16 +96,6 @@ async def default_query_permissions_sql( params = {"query_owner_id": actor_id} rule_sqls = [] - if not datasette.default_deny: - rule_sqls.append( - """ - SELECT database_name AS parent, name AS child, 1 AS allow, - 'non-private query' AS reason - FROM queries - WHERE is_private = 0 - """ - ) - if actor_id is not None: rule_sqls.append( """ @@ -141,23 +106,13 @@ async def default_query_permissions_sql( """ ) - config_restriction_selects, config_restriction_params = ( - _configured_query_restriction_selects(datasette) - ) - - restriction_sqls = [ - """ + return PermissionSQL( + sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, + restriction_sql=""" SELECT database_name AS parent, name AS child FROM queries WHERE is_private = 0 OR owner_id = :query_owner_id - """ - ] - restriction_sqls.extend(config_restriction_selects) - params.update(config_restriction_params) - - return PermissionSQL( - sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, - restriction_sql="\nUNION ALL\n".join(restriction_sqls), + """, params=params, ) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 22f294bb..4f342d8f 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -937,16 +937,20 @@ async def test_permissions_in_config( updated_config = copy.deepcopy(previous_config) updated_config.update(config) perms_ds.config = updated_config + await perms_ds.apply_queries_config() try: # Convert old-style resource to Resource object - from datasette.resources import DatabaseResource, TableResource + from datasette.resources import DatabaseResource, QueryResource, TableResource resource_obj = None if resource: if isinstance(resource, str): resource_obj = DatabaseResource(database=resource) elif isinstance(resource, tuple) and len(resource) == 2: - resource_obj = TableResource(database=resource[0], table=resource[1]) + if action == "view-query": + resource_obj = QueryResource(database=resource[0], query=resource[1]) + else: + resource_obj = TableResource(database=resource[0], table=resource[1]) result = await perms_ds.allowed( action=action, resource=resource_obj, actor=actor @@ -956,6 +960,7 @@ async def test_permissions_in_config( assert result == expected_result finally: perms_ds.config = previous_config + await perms_ds.apply_queries_config() @pytest.mark.asyncio diff --git a/tests/test_queries.py b/tests/test_queries.py index c97b5733..dde57dea 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -248,6 +248,45 @@ async def test_default_deny_blocks_view_query_even_for_trusted_query(): ) +@pytest.mark.asyncio +async def test_view_query_default_allow_still_respects_private_restriction(): + ds = Datasette(memory=True) + ds.add_memory_database("default_view_query_permissions", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "shared_report", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "shared_report"), + actor=None, + ) + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, + ) + + @pytest.mark.asyncio async def test_private_query_restriction_blocks_broad_view_query_permission(): ds = Datasette( From 1ac4265ffd295ea62008b13b3e37af96f5450be4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 12:12:59 -0700 Subject: [PATCH 707/767] Require permissions for untrusted stored query execution, refs #2735 --- datasette/views/database.py | 7 +++---- docs/authentication.rst | 2 +- queries-plan.md | 8 +++----- tests/test_queries.py | 12 ++++++++++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 91e9c350..bd939d87 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1430,10 +1430,9 @@ class QueryView(View): ): raise Forbidden("You do not have permission to view this query") - if canned_query.get("write"): - await _ensure_stored_query_execution_permissions( - datasette, db, canned_query, request.actor - ) + await _ensure_stored_query_execution_permissions( + datasette, db, canned_query, request.actor + ) # If database is immutable, return an error if not db.is_mutable: diff --git a/docs/authentication.rst b/docs/authentication.rst index 6e835c8d..453aaa19 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1285,7 +1285,7 @@ Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.i view-query ---------- -Actor is allowed to view (and execute) a saved query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size - this includes executing :ref:`canned_queries_writable`. +Actor is allowed to view a saved query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size. Executing an untrusted saved query also requires ``execute-sql`` or the relevant write permissions; trusted saved queries can execute with ``view-query`` alone. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) diff --git a/queries-plan.md b/queries-plan.md index f4b8049c..da6b7c92 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -25,7 +25,7 @@ Terminology change: these are now "queries", not "canned queries". Legacy code a - Query definitions currently come from `datasette.yaml` or the `canned_queries()` plugin hook. - `Datasette.get_canned_queries(database_name, actor)` calls that hook every time it needs query definitions. - `QueryResource.resources_sql()` currently enumerates databases and calls the hook for each one, because permissions and `/-/jump` need query resources. -- Query pages execute if the actor has `view-query` for `QueryResource(database, query)`. +- Query pages are visible if the actor has `view-query` for `QueryResource(database, query)`. Executing an untrusted stored query also checks `execute-sql` or the relevant write permissions. - Arbitrary SQL executes if the actor has `execute-sql` for `DatabaseResource(database)`. The main performance and architecture win is making query resource enumeration a direct SQL query against the internal database. @@ -145,9 +145,7 @@ Default execution rule for user-created writable queries: Implementation: -- Remove `view-query` from the broad `DEFAULT_ALLOW_ACTIONS` set. -- Replace it with query-aware default `view-query` permission SQL. -- Emit default `view-query` allows for non-private rows when Datasette is not running with `--default-deny`. +- Keep `view-query` in the broad `DEFAULT_ALLOW_ACTIONS` set, so saved queries remain visible by default in all-public Datasette. - Emit default `view-query` allows for the owning actor. - Use `restriction_sql` to limit private rows to their owner even when broader `view-query` permissions exist. - Have `QueryView` perform the fresh `execute-sql` or table-permission check before execution unless the row has `is_trusted=1`. @@ -424,7 +422,7 @@ The existing edit-SQL flow from query pages can continue to point back to arbitr - The old `canned_queries()` hook is no longer called by core. - `QueryResource.resources_sql()` returns rows from `queries`. - Database page and `/-/jump` list queries from the internal DB. -- `view-query` is no longer globally default-allowed; default query permissions come from the query-aware hook. +- `view-query` remains globally default-allowed, with `restriction_sql` narrowing private queries to their owner. - Private query is only visible to its owner, even when a broader `view-query` rule applies. - Non-trusted read-only query requires `execute-sql` to execute. - Trusted read-only query can be executed without `execute-sql` after `view-query` passes. diff --git a/tests/test_queries.py b/tests/test_queries.py index dde57dea..997f8b39 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -395,8 +395,16 @@ async def test_untrusted_shared_query_execution_requires_execute_sql(): owner_id="alice", ) - denied = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) - assert denied.status_code == 403 + denied_get = await ds.client.get( + "/data/shared_report.json", actor={"id": "viewer"} + ) + denied_post = await ds.client.post( + "/data/shared_report", + actor={"id": "viewer"}, + data={}, + ) + assert denied_get.status_code == 403 + assert denied_post.status_code == 403 ds.config["databases"]["data"]["permissions"]["execute-sql"] = {"id": "viewer"} allowed = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) From 866852eff603c219b8bf7d13f2a69b5ff032fa67 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 12:46:18 -0700 Subject: [PATCH 708/767] Clarifying comments --- datasette/default_permissions/defaults.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index ed0a6d66..32ad4ef1 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -80,6 +80,7 @@ async def default_query_permissions_sql( if action in {"update-query", "delete-query"}: if actor_id is None: return None + # Query owner can update/delete query return PermissionSQL( sql=""" SELECT database_name AS parent, name AS child, 1 AS allow, @@ -97,15 +98,15 @@ async def default_query_permissions_sql( params = {"query_owner_id": actor_id} rule_sqls = [] if actor_id is not None: - rule_sqls.append( - """ + # Query owner can view-query + rule_sqls.append(""" SELECT database_name AS parent, name AS child, 1 AS allow, 'query owner' AS reason FROM queries WHERE owner_id = :query_owner_id - """ - ) + """) + # restriction_sql enforces private queries ONLY visible to owner return PermissionSQL( sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, restriction_sql=""" From 71c76e38534378cbce8576771238a788feccf3ad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:08:19 -0700 Subject: [PATCH 709/767] Better faceting on /-/queries Ref https://github.com/simonw/datasette/pull/2741#issuecomment-4548321815 --- datasette/app.py | 69 +++++++++++++++++ datasette/templates/query_list.html | 94 +++++++++++++---------- datasette/views/database.py | 99 +++++++++++++++++++++++- tests/test_permissions.py | 8 +- tests/test_queries.py | 115 +++++++++++++++++++++++++--- 5 files changed, 330 insertions(+), 55 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 3329ee7e..1acdfcd8 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1298,6 +1298,75 @@ class Datasette: ) return self._query_row_to_dict(rows.first()) + async def count_queries( + self, + database=None, + *, + actor=None, + q=None, + is_write=None, + is_private=None, + is_trusted=None, + source=None, + owner_id=None, + ): + allowed_sql, allowed_params = await self.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + ) + params = dict(allowed_params) + where_clauses = [] + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + row = ( + await self.get_internal_database().execute( + """ + SELECT count(*) AS count + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + """.format( + allowed_sql=allowed_sql, + where=" AND ".join(where_clauses) or "1 = 1", + ), + params, + ) + ).first() + return row["count"] + async def list_queries( self, database=None, diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index 25259b3d..fa4859b1 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -9,7 +9,7 @@ max-width: 64rem; } .query-list-filters { - margin: 0.5rem 0 1rem; + margin: 0.5rem 0 0.75rem; } .query-list-search { align-items: center; @@ -32,43 +32,63 @@ line-height: 1.1; padding: 0.35rem 0.65rem; } -.query-list-filter-groups { +.query-list-facets { align-items: flex-start; display: flex; flex-wrap: wrap; - gap: 0.8rem 1.4rem; + gap: 1rem 1.6rem; + margin: 0 0 1rem; } -.query-list-filter-group { - border: 0; +.query-list-facet { + margin: 0; +} +.query-list-facet h2 { + font-size: 0.9rem; + line-height: 1.2; + margin: 0 0 0.35rem; +} +.query-list-facet ul { display: flex; flex-wrap: wrap; gap: 0.35rem; margin: 0; - min-width: 0; padding: 0; + list-style: none; } -.query-list-filter-group legend { - font-weight: 700; - margin: 0 0.45rem 0 0; - padding: 0; -} -.query-list-filter-group label { +.query-list-facet-link, +.query-list-facet-link:link, +.query-list-facet-link:visited, +.query-list-facet-link:hover, +.query-list-facet-link:focus, +.query-list-facet-link:active { align-items: center; border: 1px solid #c8d1dc; border-radius: 0.25rem; - cursor: pointer; + color: #39445a; display: inline-flex; font-size: 0.82rem; - gap: 0.3rem; + gap: 0.4rem; line-height: 1.1; padding: 0.35rem 0.55rem; + text-decoration: none; } -.query-list-filter-group input { - margin: 0; +.query-list-facet-link:hover { + border-color: #7ca5c8; + color: #1f5d85; } -.query-list-filter-group input:checked + span { +.query-list-facet-link-active { + background-color: #edf6fb; + border-color: #6d9fc0; font-weight: 700; } +.query-list-facet-disabled { + color: #7b8794; + cursor: default; +} +.query-list-facet-count { + color: #4f5b6d; + font-variant-numeric: tabular-nums; +} .query-list-results { border-collapse: collapse; font-size: 0.9rem; @@ -169,15 +189,6 @@ .query-list-search input[type=search] { max-width: none; } - .query-list-filter-group { - display: block; - } - .query-list-filter-group legend { - margin-bottom: 0.3rem; - } - .query-list-filter-group label { - margin: 0 0.25rem 0.35rem 0; - } } {% endblock %} @@ -198,24 +209,27 @@

+ {% if filters.is_write %}{% endif %} + {% if filters.is_private %}{% endif %} + {% if filters.source %}{% endif %} + {% if filters.owner_id %}{% endif %}

-
-
- Mode - - - -
-
- Visibility - - - -
-
+ + {% if queries %}
diff --git a/datasette/views/database.py b/datasette/views/database.py index bd939d87..2e77d36b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1121,6 +1121,21 @@ class QueryParametersView(BaseView): return _block_framing(Response.json({"ok": True, "parameters": parameters})) +def _query_list_url(path, query_string, *, set_args=None, remove_args=None): + set_args = set_args or {} + remove_args = set(remove_args or ()) + skip = set(set_args) | remove_args | {"_next"} + pairs = [ + (key, value) + for key, value in parse_qsl(query_string, keep_blank_values=True) + if key not in skip + ] + for key, value in set_args.items(): + if value not in (None, ""): + pairs.append((key, value)) + return path + (("?" + urlencode(pairs)) if pairs else "") + + class QueryListView(BaseView): name = "query-list" @@ -1139,9 +1154,7 @@ class QueryListView(BaseView): default=20 if format_ == "html" else 50, ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") - is_private = _as_optional_bool( - request.args.get("is_private"), "is_private" - ) + is_private = _as_optional_bool(request.args.get("is_private"), "is_private") except QueryValidationError as ex: return _error([ex.message], ex.status) @@ -1173,6 +1186,80 @@ class QueryListView(BaseView): urlencode(pairs), ) + current_filters = { + "actor": request.actor, + "q": request.args.get("q") or None, + "is_write": is_write, + "is_private": is_private, + "source": request.args.get("source") or None, + "owner_id": request.args.get("owner_id") or None, + } + + async def facet_count(field, value): + if current_filters[field] is not None and current_filters[field] != value: + return 0 + filters = dict(current_filters) + filters[field] = value + return await self.ds.count_queries(database, **filters) + + def facet_href(field, value): + if current_filters[field] == value: + return _query_list_url( + query_list_path, + request.query_string, + remove_args=[field], + ) + if current_filters[field] is not None: + return None + return _query_list_url( + query_list_path, + request.query_string, + set_args={field: str(int(value))}, + ) + + async def facet_item(label, field, value): + count = await facet_count(field, value) + active = current_filters[field] == value + if not active and not count: + return None + return { + "label": label, + "count": count, + "href": facet_href(field, value) if active or count else None, + "active": active, + } + + async def facet_items(items): + return [ + item + for item in [ + await facet_item(label, field, value) + for label, field, value in items + ] + if item is not None + ] + + facets = [ + { + "title": "Mode", + "items": await facet_items( + [ + ("Read-only", "is_write", False), + ("Writable", "is_write", True), + ] + ), + }, + { + "title": "Visibility", + "items": await facet_items( + [ + ("Not private", "is_private", False), + ("Private", "is_private", True), + ] + ), + }, + ] + data = { "ok": True, "database": database, @@ -1188,6 +1275,7 @@ class QueryListView(BaseView): "show_trusted_note": any(query["is_trusted"] for query in page["queries"]), "query_list_path": query_list_path, "show_database": database is None, + "facets": facets, "filters": { "q": request.args.get("q") or "", "is_write": request.args.get("is_write") or "", @@ -1715,6 +1803,9 @@ class QueryView(View): } ) metadata = await datasette.get_database_metadata(database) + if canned_query: + metadata = dict(canned_query) + metadata.pop("source", None) renderers = {} for key, (_, can_render) in datasette.renderers.items(): @@ -1865,7 +1956,7 @@ class QueryView(View): ) ), show_hide_hidden=markupsafe.Markup(show_hide_hidden), - metadata=canned_query or metadata, + metadata=metadata, alternate_url_json=alternate_url_json, select_templates=[ f"{'*' if template_name == template.name else ''}{template_name}" diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 4f342d8f..eb6cee9f 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -948,9 +948,13 @@ async def test_permissions_in_config( resource_obj = DatabaseResource(database=resource) elif isinstance(resource, tuple) and len(resource) == 2: if action == "view-query": - resource_obj = QueryResource(database=resource[0], query=resource[1]) + resource_obj = QueryResource( + database=resource[0], query=resource[1] + ) else: - resource_obj = TableResource(database=resource[0], table=resource[1]) + resource_obj = TableResource( + database=resource[0], table=resource[1] + ) result = await perms_ds.allowed( action=action, resource=resource_obj, actor=actor diff --git a/tests/test_queries.py b/tests/test_queries.py index 997f8b39..36f7107a 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -395,9 +395,7 @@ async def test_untrusted_shared_query_execution_requires_execute_sql(): owner_id="alice", ) - denied_get = await ds.client.get( - "/data/shared_report.json", actor={"id": "viewer"} - ) + denied_get = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) denied_post = await ds.client.post( "/data/shared_report", actor={"id": "viewer"}, @@ -608,6 +606,27 @@ async def test_query_list_and_definition_api(): assert definition_response.json()["query"]["title"] == "Demo query 01" +@pytest.mark.asyncio +async def test_query_page_does_not_show_internal_source(): + ds = Datasette(memory=True) + ds.add_memory_database("query_page_source", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "stored_report", + "select 1 as one", + title="Stored report", + source="user", + owner_id="root", + ) + + response = await ds.client.get("/data/stored_report", actor={"id": "root"}) + + assert response.status_code == 200 + assert "Stored report" in response.text + assert "Data source:" not in response.text + + @pytest.mark.asyncio async def test_query_list_search_filter_and_html(): ds = Datasette(memory=True) @@ -632,6 +651,15 @@ async def test_query_list_search_filter_and_html(): is_trusted=True, source="config", ) + await ds.add_query( + "data", + "writable_query", + "insert into dogs (name) values (:name)", + title="Writable query", + is_write=True, + source="user", + owner_id="root", + ) html_response = await ds.client.get( "/data/-/queries?q=02", @@ -649,13 +677,21 @@ async def test_query_list_search_filter_and_html(): "/data/-/queries.json?is_private=1", actor={"id": "root"}, ) + filtered_write_response = await ds.client.get( + "/data/-/queries?is_write=1", + actor={"id": "root"}, + ) + filtered_private_response = await ds.client.get( + "/data/-/queries?is_private=1", + actor={"id": "root"}, + ) assert html_response.status_code == 200 assert "Demo query 02" in html_response.text assert "Demo query 01" not in html_response.text assert 'class="query-list-results"' in html_response.text - assert "Mode" in html_response.text - assert 'type="radio" name="is_private" value="1"' in html_response.text + assert 'class="query-list-facets"' in html_response.text + assert 'type="radio"' not in html_response.text assert "Only the owning actor can view this query." not in html_response.text assert ( "Execution skips the usual SQL and write permission checks" @@ -667,14 +703,75 @@ async def test_query_list_search_filter_and_html(): assert '' not in flags_response.text assert 'class="query-list-owner">root' in flags_response.text assert 'class="query-list-pill">Read-only' in flags_response.text - assert 'class="query-list-pill query-list-pill-private">Private' in flags_response.text - assert 'class="query-list-pill query-list-pill-trusted">Trusted' in flags_response.text + assert ( + 'class="query-list-pill query-list-pill-write">Writable' + in flags_response.text + ) + assert ( + 'class="query-list-pill query-list-pill-private">Private' + in flags_response.text + ) + assert ( + 'class="query-list-pill query-list-pill-trusted">Trusted' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_write=0">Read-only5' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_write=1">Writable1' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_private=0">Not private5' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_private=1">Private1' + in flags_response.text + ) assert "Only the owning actor can view this query." in flags_response.text - assert "Execution skips the usual SQL and write permission checks" in flags_response.text + assert ( + "Execution skips the usual SQL and write permission checks" + in flags_response.text + ) assert json_response.json()["queries"][0]["name"] == "demo_query_02" assert [query["name"] for query in filtered_response.json()["queries"]] == [ "private_query" ] + assert "Writable query" in filtered_write_response.text + assert "Demo query 01" not in filtered_write_response.text + assert ( + 'query-list-facet-link query-list-facet-link-active" href="/data/-/queries"' + in filtered_write_response.text + ) + assert ( + 'Read-only0' + not in filtered_write_response.text + ) + assert ( + 'href="/data/-/queries?is_write=1&is_private=0">Not private1' + in filtered_write_response.text + ) + assert ( + 'Private0' + not in filtered_write_response.text + ) + assert "Private query" in filtered_private_response.text + assert "Demo query 01" not in filtered_private_response.text + assert ( + 'href="/data/-/queries?is_private=1&is_write=0">Read-only1' + in filtered_private_response.text + ) + assert ( + 'Writable0' + not in filtered_private_response.text + ) + assert ( + 'Not private0' + not in filtered_private_response.text + ) @pytest.mark.asyncio @@ -1313,7 +1410,7 @@ async def test_user_writable_query_execution_rechecks_table_permissions(): "insert-row": {"id": "alice"}, } } - } + }, } } }, From 0fcaa5792ba73143661515af0088d7e5d968e96c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:12:07 -0700 Subject: [PATCH 710/767] Style query operations on create query Made it consistent with the SQL write page. --- .../_execute_write_analysis_styles.html | 37 +++++++++++++++++++ datasette/templates/execute_write.html | 36 +----------------- datasette/templates/query_create.html | 19 +++++----- tests/test_queries.py | 6 ++- 4 files changed, 52 insertions(+), 46 deletions(-) create mode 100644 datasette/templates/_execute_write_analysis_styles.html diff --git a/datasette/templates/_execute_write_analysis_styles.html b/datasette/templates/_execute_write_analysis_styles.html new file mode 100644 index 00000000..f20e67b2 --- /dev/null +++ b/datasette/templates/_execute_write_analysis_styles.html @@ -0,0 +1,37 @@ + diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 46f58c3b..414d4af7 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -40,42 +40,8 @@ border-radius: 0.25rem; min-width: 13rem; } -.execute-write-analysis { - border-collapse: collapse; - font-size: 0.9rem; - margin: 0.25rem 0 1rem; - min-width: 44rem; -} -.execute-write-analysis th, -.execute-write-analysis td { - border-bottom: 1px solid #d7dde5; - padding: 0.45rem 0.7rem; - text-align: left; - vertical-align: top; -} -.execute-write-analysis th { - background-color: #edf6fb; - border-top: 1px solid #d7dde5; - color: #39445a; - font-weight: 700; -} -.execute-write-analysis tbody tr:nth-child(even) { - background-color: rgba(39, 104, 144, 0.05); -} -.execute-write-analysis code { - background: transparent; - font-size: 0.9em; - white-space: nowrap; -} -.execute-write-analysis-allowed { - color: #267a3e; - font-weight: 700; -} -.execute-write-analysis-denied { - color: #b00020; - font-weight: 700; -} +{% include "_execute_write_analysis_styles.html" %} {% include "_sql_parameter_styles.html" %} {% endblock %} diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index 686d971e..2d8a9122 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -5,6 +5,7 @@ {% block extra_head %} {{- super() -}} {% include "_codemirror.html" %} +{% include "_execute_write_analysis_styles.html" %} {% endblock %} {% block body_class %}query-create db-{{ database|to_css_class }}{% endblock %} @@ -32,30 +33,28 @@

Execute write SQL

{% endif %} -

Analysis

+

Query operations

{% if analysis_error %}

{{ analysis_error }}

{% elif analysis_rows %} -
Mode
+
- + - {% for row in analysis_rows %} - - - - - - + + + + + {% endfor %} diff --git a/tests/test_queries.py b/tests/test_queries.py index 36f7107a..c27c23da 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -998,7 +998,11 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): assert "Create query" in create_response.text assert "Read-only" in create_response.text assert "Writable" in create_response.text - assert "required permission" in create_response.text + assert "

Query operations

" in create_response.text + assert '
Operation Database Tablerequired permissionRequired permission AllowedSource
{{ row.operation }}{{ row.database }}{{ row.table }}{{ row.required_permission }}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}{{ row.source or "" }}{{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% endif %}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}
' in create_response.text + assert '' in create_response.text + assert '' not in create_response.text + assert "" in create_response.text assert query_response.status_code == 200 assert "Save query" in query_response.text assert "/data/-/queries/-/create?sql=select+%2A+from+dogs" in query_response.text From 70b23ff4a55528083512fab96aa50725f415cbe4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:47:24 -0700 Subject: [PATCH 711/767] Tweaked save query link --- datasette/templates/query.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index f74d21f1..1900bd31 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -66,7 +66,7 @@ {% if not hide_sql %}{% endif %} {{ show_hide_hidden }} - {% if save_query_url %}Save query{% endif %} + {% if save_query_url %}Save this query{% endif %} {% if canned_query and edit_sql_url %}Edit SQL{% endif %}

From eb7c25c57cf914629c08eaa477d0709b0f41efeb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:48:40 -0700 Subject: [PATCH 712/767] Major redesign of create saved query UI https://github.com/simonw/datasette/pull/2741#issuecomment-4548707129 --- datasette/app.py | 6 +- datasette/static/app.css | 4 + .../_execute_write_analysis_scripts.html | 111 +++++++ .../_execute_write_analysis_styles.html | 4 + .../templates/_sql_parameter_scripts.html | 17 +- datasette/templates/execute_write.html | 88 +----- datasette/templates/query_create.html | 296 +++++++++++++++--- datasette/views/database.py | 181 ++++++++--- tests/test_queries.py | 170 +++++++++- 9 files changed, 705 insertions(+), 172 deletions(-) create mode 100644 datasette/templates/_execute_write_analysis_scripts.html diff --git a/datasette/app.py b/datasette/app.py index 1acdfcd8..8936b099 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -50,7 +50,7 @@ from .views.database import ( ExecuteWriteView, TableCreateView, QueryView, - QueryCreateView, + QueryCreateAnalyzeView, QueryDeleteView, QueryDefinitionView, GlobalQueryListView, @@ -2820,8 +2820,8 @@ class Datasette: r"/(?P[^\/\.]+)/-/queries(\.(?Pjson))?$", ) add_route( - QueryCreateView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/-/create$", + QueryCreateAnalyzeView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/analyze$", ) add_route( QueryInsertView.as_view(self), diff --git a/datasette/static/app.css b/datasette/static/app.css index c21d0dc4..4f4db133 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1414,6 +1414,10 @@ svg.dropdown-menu-icon { position: relative; top: 1px; } +.save-query { + display: inline-block; + margin-left: 0.45em; +} .blob-download { display: block; diff --git a/datasette/templates/_execute_write_analysis_scripts.html b/datasette/templates/_execute_write_analysis_scripts.html new file mode 100644 index 00000000..a19bae13 --- /dev/null +++ b/datasette/templates/_execute_write_analysis_scripts.html @@ -0,0 +1,111 @@ + diff --git a/datasette/templates/_execute_write_analysis_styles.html b/datasette/templates/_execute_write_analysis_styles.html index f20e67b2..165cfe9f 100644 --- a/datasette/templates/_execute_write_analysis_styles.html +++ b/datasette/templates/_execute_write_analysis_styles.html @@ -34,4 +34,8 @@ color: #b00020; font-weight: 700; } +.execute-write-analysis-na { + color: #687386; + font-style: italic; +} diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html index 68e46069..159a141c 100644 --- a/datasette/templates/_sql_parameter_scripts.html +++ b/datasette/templates/_sql_parameter_scripts.html @@ -215,9 +215,10 @@ window.datasetteSqlParameters = (() => { if (!form) { return null; } + const shouldRenderParameters = options.renderParameters !== false; const section = options.section || form.querySelector("[data-sql-parameters-section]"); - if (!section) { + if (shouldRenderParameters && !section) { return null; } const manager = { @@ -225,12 +226,16 @@ window.datasetteSqlParameters = (() => { section, allowExpand: options.allowExpand === undefined - ? section.dataset.allowExpand === "1" + ? section + ? section.dataset.allowExpand === "1" + : false : options.allowExpand, parameterState: new Map(), }; - bindParameterControls(manager); - syncParameterState(manager); + if (section) { + bindParameterControls(manager); + syncParameterState(manager); + } const url = options.url || form.dataset.parametersUrl; let refreshTimer = null; @@ -254,7 +259,9 @@ window.datasetteSqlParameters = (() => { if (!response.ok) { throw new Error((data.errors || [response.statusText]).join("; ")); } - renderParameters(manager, data.parameters || []); + if (shouldRenderParameters) { + renderParameters(manager, data.parameters || []); + } if (options.onData) { options.onData(data, manager); } diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 414d4af7..7a627a7a 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -131,6 +131,7 @@ if (executeWriteSqlInput && !executeWriteSqlInput.value) { {% include "_codemirror_foot.html" %} {% include "_sql_parameter_scripts.html" %} +{% include "_execute_write_analysis_scripts.html" %} + + {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 2e77d36b..aafcf40b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -551,6 +551,17 @@ def _wants_json(request, is_json, data): ) +def _query_create_form_error_message(message): + return { + "Query name is required": "URL is required", + "Invalid query name": "Invalid URL", + "Query name conflicts with a table or view": ( + "URL conflicts with an existing table or view" + ), + "Query already exists": "A query already exists at that URL", + }.get(message, message) + + async def _json_or_form_payload(request): content_type = request.headers.get("content-type", "") if content_type.startswith("application/json"): @@ -731,6 +742,54 @@ async def _execute_write_analysis_data(datasette, db, sql, actor): } +async def _query_create_analysis_data(datasette, db, sql, actor): + has_sql = bool(sql and sql.strip()) + parameter_names = [] + analysis_rows = [] + analysis_error = None + if has_sql: + try: + parameter_names = _derived_query_parameters(sql) + params = {parameter: "" for parameter in parameter_names} + analysis = await db.analyze_sql(sql, params) + analysis_rows = await _analysis_rows_with_permissions( + datasette, analysis, actor + ) + except (QueryValidationError, sqlite3.DatabaseError) as ex: + analysis_error = getattr(ex, "message", str(ex)) + return { + "ok": analysis_error is None, + "parameters": parameter_names, + "analysis_error": analysis_error, + "analysis_rows": analysis_rows, + "has_sql": has_sql, + "analysis_is_write": bool( + analysis_rows and any(row["required_permission"] for row in analysis_rows) + ), + "save_disabled": bool( + (not has_sql) + or analysis_error + or any(row["allowed"] is False for row in analysis_rows) + ), + } + + +async def _query_create_form_context( + datasette, request, db, *, sql="", name="", title="", description="", is_private=True +): + analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor) + return { + "database": db.name, + "database_color": db.color, + "sql": sql, + "name": name, + "title": title, + "description": description, + "is_private": is_private, + **analysis_data, + } + + async def _inserted_row_url(datasette, db, analysis, cursor): if cursor.rowcount != 1: return None @@ -1307,6 +1366,35 @@ class QueryCreateView(BaseView): name = "query-create" has_json_alternate = False + async def _render_form( + self, + request, + db, + *, + sql="", + name="", + title="", + description="", + is_private=True, + status=200, + ): + response = await self.render( + ["query_create.html"], + request, + await _query_create_form_context( + self.ds, + request, + db, + sql=sql, + name=name, + title=title, + description=description, + is_private=is_private, + ), + ) + response.status = status + return response + async def get(self, request): db = await self.ds.resolve_database(request) await self.ds.ensure_permission( @@ -1320,46 +1408,61 @@ class QueryCreateView(BaseView): actor=request.actor, ) - sql = request.args.get("sql") or "" - analysis_error = None - analysis_rows = [] - parameter_names = [] - if sql: - try: - parameter_names = _derived_query_parameters(sql) - params = {parameter: "" for parameter in parameter_names} - analysis = await db.analyze_sql(sql, params) - analysis_rows = await _analysis_rows_with_permissions( - self.ds, analysis, request.actor - ) - except (QueryValidationError, sqlite3.DatabaseError) as ex: - analysis_error = getattr(ex, "message", str(ex)) + return await self._render_form(request, db, sql=request.args.get("sql") or "") - return await self.render( - ["query_create.html"], - request, - { - "database": db.name, - "database_color": db.color, - "sql": sql, - "parameter_names": parameter_names, - "analysis_error": analysis_error, - "analysis_rows": analysis_rows, - "analysis_is_write": bool( - analysis_rows - and any(row["required_permission"] for row in analysis_rows) - ), - "save_disabled": bool( - analysis_error - or any(row["allowed"] is False for row in analysis_rows) - ), - }, + +class QueryCreateAnalyzeView(BaseView): + name = "query-create-analyze" + has_json_alternate = False + + async def get(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + if not await self.ds.allowed( + action="insert-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need insert-query"], 403)) + + invalid_keys = set(request.args) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + sql = request.args.get("sql") or "" + return _block_framing( + Response.json( + await _query_create_analysis_data(self.ds, db, sql, request.actor) + ) ) -class QueryInsertView(BaseView): +class QueryInsertView(QueryCreateView): name = "query-insert" + async def _error_response(self, request, db, query_data, message, status): + message = _query_create_form_error_message(message) + self.ds.add_message(request, message, self.ds.ERROR) + return await self._render_form( + request, + db, + sql=query_data.get("sql") or "", + name=query_data.get("name") or "", + title=query_data.get("title") or "", + description=query_data.get("description") or "", + is_private=_as_bool(query_data.get("is_private", True)), + status=status, + ) + async def post(self, request): db = await self.ds.resolve_database(request) if not await self.ds.allowed( @@ -1375,6 +1478,8 @@ class QueryInsertView(BaseView): ): return _error(["Permission denied: need insert-query"], 403) + is_json = False + query_data = {} try: data, is_json = await _json_or_form_payload(request) if not isinstance(data, dict): @@ -1384,6 +1489,10 @@ class QueryInsertView(BaseView): raise QueryValidationError("JSON must contain a query dictionary") prepared = await _prepare_query_create(self.ds, request, db, query_data) except QueryValidationError as ex: + if not is_json and isinstance(query_data, dict): + return await self._error_response( + request, db, query_data, ex.message, ex.status + ) return _error([ex.message], ex.status) prepared.pop("analysis") @@ -1391,6 +1500,8 @@ class QueryInsertView(BaseView): try: await self.ds.add_query(db.name, name, replace=False, **prepared) except sqlite3.IntegrityError as ex: + if not is_json and isinstance(query_data, dict): + return await self._error_response(request, db, query_data, str(ex), 400) return _error([str(ex)], 400) query = await self.ds.get_query(db.name, name) @@ -1896,7 +2007,7 @@ class QueryView(View): ): save_query_url = ( datasette.urls.database(database) - + "/-/queries/-/create?" + + "/-/queries/insert?" + urlencode({"sql": sql}) ) diff --git a/tests/test_queries.py b/tests/test_queries.py index c27c23da..32cdfae3 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -986,6 +986,14 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): await ds.invoke_startup() create_response = await ds.client.get( + "/data/-/queries/insert?sql=select+*+from+dogs", + actor={"id": "root"}, + ) + blank_create_response = await ds.client.get( + "/data/-/queries/insert", + actor={"id": "root"}, + ) + old_create_response = await ds.client.get( "/data/-/queries/-/create?sql=select+*+from+dogs", actor={"id": "root"}, ) @@ -996,16 +1004,171 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): assert create_response.status_code == 200 assert "Create query" in create_response.text - assert "Read-only" in create_response.text assert "Writable" in create_response.text + assert 'type="radio"' not in create_response.text + assert 'name="parameters"' not in create_response.text + assert 'id="query-parameters"' not in create_response.text + assert 'class="query-create-field"' in create_response.text + assert '' not in create_response.text + assert '' in create_response.text + assert '' in create_response.text + assert '/data/' in create_response.text + assert ( + '' + in create_response.text + ) + assert 'function slugify(value)' in create_response.text + assert 'data-analyze-url="/data/-/queries/analyze"' in create_response.text + assert "setupSqlParameterRefresh" in create_response.text + assert "renderParameters: false" in create_response.text + assert "datasetteSqlAnalysis.renderAnalysis" in create_response.text + assert "data-query-create-submit" in create_response.text + assert "data-query-create-writable" in create_response.text + assert ( + "Queries marked private can only be seen by you, their creator." + in create_response.text + ) assert "

Query operations

" in create_response.text assert '
Required permissionSourceread
' in create_response.text assert '' in create_response.text assert '' not in create_response.text assert "" in create_response.text + assert ( + create_response.text.count( + '' + ) + == 2 + ) + assert create_response.text.index('value="Save query"') < create_response.text.index( + "

Query operations

" + ) + assert blank_create_response.status_code == 200 + assert ( + '
Required permissionSourcereadn/a
' in response.text assert '' in response.text assert "" in response.text From 5dca2dc9beea96c52e6a9c806df66c9a1f2f7874 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:54:47 -0700 Subject: [PATCH 713/767] Show query count on database page --- datasette/templates/database.html | 2 +- datasette/views/database.py | 18 +++++++++++++++++- tests/test_queries.py | 11 ++++++----- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 62f9c620..371f6a22 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -59,7 +59,7 @@ {% endfor %} {% if queries_more %} -

View all queries

+

View {{ "{:,}".format(queries_count) }} quer{% if queries_count == 1 %}y{% else %}ies{% endif %}

{% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index feb38619..d40d69d1 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -102,6 +102,11 @@ class DatabaseView(View): ) canned_queries = queries_page["queries"] queries_more = queries_page["has_more"] + queries_count = ( + await datasette.count_queries(database, actor=request.actor) + if queries_more + else len(canned_queries) + ) async def database_actions(): links = [] @@ -134,6 +139,7 @@ class DatabaseView(View): "views": sql_views, "queries": canned_queries, "queries_more": queries_more, + "queries_count": queries_count, "allow_execute_sql": allow_execute_sql, "table_columns": ( await _table_columns(datasette, database) if allow_execute_sql else {} @@ -168,6 +174,7 @@ class DatabaseView(View): views=sql_views, queries=canned_queries, queries_more=queries_more, + queries_count=queries_count, allow_execute_sql=allow_execute_sql, table_columns=( await _table_columns(datasette, database) @@ -219,6 +226,7 @@ class DatabaseContext(Context): queries_more: bool = field( metadata={"help": "Boolean indicating if more saved queries are available"} ) + queries_count: int = field(metadata={"help": "Count of visible saved queries"}) allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) @@ -775,7 +783,15 @@ async def _query_create_analysis_data(datasette, db, sql, actor): async def _query_create_form_context( - datasette, request, db, *, sql="", name="", title="", description="", is_private=True + datasette, + request, + db, + *, + sql="", + name="", + title="", + description="", + is_private=True, ): analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor) return { diff --git a/tests/test_queries.py b/tests/test_queries.py index 32cdfae3..09b41645 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -458,9 +458,10 @@ async def test_database_page_query_preview_is_limited(): assert html_response.status_code == 200 assert "Demo query 05" in html_response.text assert "Demo query 06" not in html_response.text - assert 'href="/data/-/queries"' in html_response.text + assert 'View 25 queries' in html_response.text assert len(json_response.json()["queries"]) == 5 assert json_response.json()["queries_more"] is True + assert json_response.json()["queries_count"] == 25 @pytest.mark.asyncio @@ -1017,7 +1018,7 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): '' in create_response.text ) - assert 'function slugify(value)' in create_response.text + assert "function slugify(value)" in create_response.text assert 'data-analyze-url="/data/-/queries/analyze"' in create_response.text assert "setupSqlParameterRefresh" in create_response.text assert "renderParameters: false" in create_response.text @@ -1039,9 +1040,9 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): ) == 2 ) - assert create_response.text.index('value="Save query"') < create_response.text.index( - "

Query operations

" - ) + assert create_response.text.index( + 'value="Save query"' + ) < create_response.text.index("

Query operations

") assert blank_create_response.status_code == 200 assert ( '
Required permissioninsert
' in create_response.text assert '' in create_response.text @@ -1053,6 +1067,12 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): "

Analysis will show each affected table and required permission.

" not in blank_create_response.text ) + assert "Enter SQL to analyze this query." in blank_create_response.text + assert write_create_response.status_code == 200 + assert ( + 'This query updates data in the database.' + in write_create_response.text + ) assert query_response.status_code == 200 assert "Save this query" in query_response.text assert "/data/-/queries/insert?sql=select+%2A+from+dogs" in query_response.text From 024b9117725bbed17396a5a4b3f48663c23337f5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:09:53 -0700 Subject: [PATCH 715/767] Clarifying comment https://github.com/simonw/datasette/pull/2741/changes#r3306856046 --- datasette/default_permissions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index a9f2d8bd..6cd46f04 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -26,6 +26,7 @@ from .restrictions import ( from .root import root_user_permissions_sql as root_user_permissions_sql from .config import config_permissions_sql as config_permissions_sql from .defaults import ( + # Avoid "datasette.default_permissions" does not explicitly export attribute default_allow_sql_check as default_allow_sql_check, default_action_permissions_sql as default_action_permissions_sql, default_query_permissions_sql as default_query_permissions_sql, From ac6ee097dd06050188d44c6d4b17a98a12c7b481 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:10:48 -0700 Subject: [PATCH 716/767] Disallow update/delete of private queries If a user does not own a private query they cannot update or delete it either, even if they have global update-query. https://github.com/simonw/datasette/pull/2741/changes#r3306417463 --- datasette/default_permissions/defaults.py | 33 ++++----- tests/test_queries.py | 81 +++++++++++++++++++++++ 2 files changed, 95 insertions(+), 19 deletions(-) diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 32ad4ef1..5bc74425 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -77,36 +77,31 @@ async def default_query_permissions_sql( ) -> Optional[PermissionSQL]: actor_id = actor.get("id") if isinstance(actor, dict) else None - if action in {"update-query", "delete-query"}: - if actor_id is None: - return None - # Query owner can update/delete query - return PermissionSQL( - sql=""" - SELECT database_name AS parent, name AS child, 1 AS allow, - 'query owner' AS reason - FROM queries - WHERE source = 'user' - AND owner_id = :query_owner_id - """, - params={"query_owner_id": actor_id}, - ) - - if action != "view-query": + if action not in {"view-query", "update-query", "delete-query"}: return None params = {"query_owner_id": actor_id} rule_sqls = [] if actor_id is not None: - # Query owner can view-query - rule_sqls.append(""" + if action in {"update-query", "delete-query"}: + # Query owner can update/delete query + rule_sqls.append(""" + SELECT database_name AS parent, name AS child, 1 AS allow, + 'query owner' AS reason + FROM queries + WHERE source = 'user' + AND owner_id = :query_owner_id + """) + else: + # Query owner can view-query + rule_sqls.append(""" SELECT database_name AS parent, name AS child, 1 AS allow, 'query owner' AS reason FROM queries WHERE owner_id = :query_owner_id """) - # restriction_sql enforces private queries ONLY visible to owner + # restriction_sql enforces private queries ONLY visible/mutable by owner return PermissionSQL( sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, restriction_sql=""" diff --git a/tests/test_queries.py b/tests/test_queries.py index f888dda0..26a0748c 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1581,6 +1581,87 @@ async def test_query_owner_gets_update_delete_and_writable_view_defaults(): ) +@pytest.mark.asyncio +async def test_private_query_restricts_broad_update_delete_permissions(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "update-query": {"id": "bob"}, + "delete-query": {"id": "bob"}, + }, + }, + }, + }, + ) + ds.add_memory_database("query_broad_update_delete", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "alice_private", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "alice_public", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + for action in ("update-query", "delete-query"): + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), + actor={"id": "bob"}, + ) + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_public"), + actor={"id": "bob"}, + ) + + private_update_response = await ds.client.post( + "/data/alice_private/-/update", + actor={"id": "bob"}, + json={"update": {"title": "Nope"}}, + ) + private_delete_response = await ds.client.post( + "/data/alice_private/-/delete", + actor={"id": "bob"}, + json={}, + ) + public_update_response = await ds.client.post( + "/data/alice_public/-/update", + actor={"id": "bob"}, + json={"update": {"title": "Bob can edit public queries"}}, + ) + public_delete_response = await ds.client.post( + "/data/alice_public/-/delete", + actor={"id": "bob"}, + json={}, + ) + + assert private_update_response.status_code == 403 + assert private_delete_response.status_code == 403 + assert public_update_response.status_code == 200 + assert public_delete_response.status_code == 200 + assert await ds.get_query("data", "alice_private") is not None + assert await ds.get_query("data", "alice_public") is None + + @pytest.mark.asyncio async def test_user_writable_query_execution_rechecks_table_permissions(): ds = Datasette( From 180a6a86fd77ac43f6cf3bfb7d7f9150003da419 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:16:10 -0700 Subject: [PATCH 717/767] Remove queries-plan.md We do not need this any more. It can live forever in Git history. --- queries-plan.md | 446 ------------------------------------------------ 1 file changed, 446 deletions(-) delete mode 100644 queries-plan.md diff --git a/queries-plan.md b/queries-plan.md deleted file mode 100644 index da6b7c92..00000000 --- a/queries-plan.md +++ /dev/null @@ -1,446 +0,0 @@ -# Queries in the internal database - -Plan for . - -## Goal - -Move named query definitions into Datasette's internal database, so hundreds or thousands of queries can be listed, searched, permission-filtered, managed, and executed efficiently. - -Terminology change: these are now "queries", not "canned queries". Legacy code and documentation can mention the old name only when describing compatibility or migration. - -## Decisions so far - -- Internal table name: `queries`. -- Query definitions should use real columns, not a JSON blob for all options. -- Query parameter names live in a `parameters` text column as a JSON array. No default values for parameters in this pass. -- No separate index is needed for the privacy/trust flags yet. -- User-created queries require `execute-sql` and `insert-query` on the database. They default to private, and writable queries additionally require matching table write permissions discovered by `Database.analyze_sql()`. -- Configured queries default to trusted, which means actors who can view them can execute them without also holding `execute-sql` or the relevant write permissions. Config can opt out with `is_trusted: false`. -- Add `update-query` and `delete-query`, so administrators can manage queries created by other users. -- Remove the old `canned_queries()` hook from core. If we want compatibility later, build a separate `datasette-old-canned-queries` plugin. -- Writable user-created queries can be supported using `Database.analyze_sql()`, provided we fail closed when analysis cannot prove the required permissions. - -## Current shape - -- Query definitions currently come from `datasette.yaml` or the `canned_queries()` plugin hook. -- `Datasette.get_canned_queries(database_name, actor)` calls that hook every time it needs query definitions. -- `QueryResource.resources_sql()` currently enumerates databases and calls the hook for each one, because permissions and `/-/jump` need query resources. -- Query pages are visible if the actor has `view-query` for `QueryResource(database, query)`. Executing an untrusted stored query also checks `execute-sql` or the relevant write permissions. -- Arbitrary SQL executes if the actor has `execute-sql` for `DatabaseResource(database)`. - -The main performance and architecture win is making query resource enumeration a direct SQL query against the internal database. - -## Proposed internal schema - -Start with one `queries` table. - -```sql -CREATE TABLE IF NOT EXISTS queries ( - database_name TEXT NOT NULL, - name TEXT NOT NULL, - sql TEXT NOT NULL, - title TEXT, - description TEXT, - description_html TEXT, - options TEXT NOT NULL DEFAULT '{}', - parameters TEXT NOT NULL DEFAULT '[]', - is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), - is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), - source TEXT NOT NULL DEFAULT 'user', - owner_id TEXT, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (database_name, name) -); - -CREATE INDEX IF NOT EXISTS queries_owner_idx - ON queries(owner_id); -``` - -Column notes: - -- `database_name`, `name`, and `sql` are the routing and execution core. -- Display fields become columns: `title`, `description`, and `description_html`. -- Less common presentation and writable-query behavior lives in `options`, stored as a JSON object. That covers `hide_sql`, `fragment`, `on_success_message`, `on_success_message_sql`, `on_success_redirect`, `on_error_message`, and `on_error_redirect`. -- `parameters` is a JSON array of parameter names, stored as text. This preserves explicit parameter order, but does not support labels or default values. -- Existing writable query behavior gets `is_write` as a column. Success/error messages, success/error redirects, and `on_success_message_sql` are stored in `options`. -- `is_private` means the query is only visible to its owning actor. This is enforced as a permission restriction, so broader `view-query` grants do not expose private rows. -- `is_trusted` means execution skips the usual `execute-sql` or write-permission checks after `view-query` has allowed access. -- `source` distinguishes `user`, `config`, and `plugin` rows. -- `owner_id` is the actor id for user-created rows. It is `NULL` for config/plugin rows. - -No separate index is needed on `(database_name, name)` because the primary key already creates one. - -`QueryResource.resources_sql()` can become: - -```sql -SELECT q.database_name AS parent, q.name AS child -FROM queries q -JOIN catalog_databases cd ON cd.database_name = q.database_name -``` - -The join keeps persisted queries for detached databases from appearing as live resources. - -## Config and plugin migration - -`datasette.yaml` can continue to support `databases: {db}: queries:` blocks, but core should import them directly into the internal `queries` tables at startup: - -1. Ensure the internal schema exists. -2. Delete previous `source='config'` rows. -3. Read configured query blocks for each live database. -4. Normalize string definitions to `{"sql": ...}`. -5. Insert rows into `queries`, storing explicit `params` as JSON in `parameters`. - -Plugins should move to: - -```python -await datasette.add_query(...) -await datasette.remove_query(...) -``` - -Remove the old `canned_queries()` hookspec and all core calls to it. If compatibility is needed, build `datasette-old-canned-queries` later as a plugin that restores the hook and imports old hook results using `datasette.add_query()`. - -## Permission model - -Add core actions: - -- `insert-query`, database-level, for creating queries in a database. -- `update-query`, query-level, for modifying existing query definitions. -- `delete-query`, query-level, for deleting existing query definitions. - -User-created query creation requires: - -- `execute-sql` on `DatabaseResource(database)` -- `insert-query` on `DatabaseResource(database)` -- If analysis shows the query is writable, the table-level write permissions described in the writable query section. - -Updating an existing query requires: - -- `update-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. -- If the SQL changes, also require `execute-sql` on the database. -- If the changed SQL is writable, also require the table-level write permissions described in the writable query section. - -Deleting an existing query requires: - -- `delete-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. - -Default owner permissions: - -- For `source='user' AND owner_id = actor.id`, grant `update-query` and `delete-query`. -- For `source='user' AND owner_id = actor.id`, grant `view-query`. If the query is private, restriction SQL ensures no other actor sees it through a broader grant. - -## Executing queries - -Default execution rule for read-only queries: - -- If `is_trusted=0`, the actor needs `execute-sql` on the database. -- If `is_trusted=1`, the actor can execute the query without `execute-sql`, provided `view-query` allows access. - -Default execution rule for user-created writable queries: - -- `is_trusted` must be `0`. -- The actor must have `view-query`. -- The actor must currently have every write permission required by fresh `Database.analyze_sql()` results for the query SQL. - -Implementation: - -- Keep `view-query` in the broad `DEFAULT_ALLOW_ACTIONS` set, so saved queries remain visible by default in all-public Datasette. -- Emit default `view-query` allows for the owning actor. -- Use `restriction_sql` to limit private rows to their owner even when broader `view-query` permissions exist. -- Have `QueryView` perform the fresh `execute-sql` or table-permission check before execution unless the row has `is_trusted=1`. - -For read-only queries this keeps `QueryView` explicit: it checks `view-query` for the query resource, then checks `execute-sql` unless the row is trusted. User-created writable queries need one additional runtime permission check because their required table permissions are derived from fresh SQL analysis. - -Explicit deny rules should still be able to block a query, and `--default-deny` still blocks trusted queries unless something grants `view-query`. - -## Writable queries - -Writable user-created queries should be in scope, guarded by `Database.analyze_sql()`. - -The secure rule: a user can create, update, or execute a writable user-created query only if they currently have the corresponding write permissions for every table the SQL can affect. - -`Database.analyze_sql(sql, params=None)` runs the SQL through SQLite's authorizer on an isolated connection and returns a `SQLAnalysis` object containing `SQLTableAccess` rows: - -- `operation`: `read`, `insert`, `update`, or `delete` -- `database`: Datasette database name for `main`, or SQLite schema name where no Datasette mapping exists -- `table`: affected table or view -- `columns`: read/updated columns where SQLite reports them -- `source`: trigger/view/CTE source when SQLite reports one - -Validation flow for user-created queries: - -1. Derive named parameters from the SQL and pass harmless placeholder values into `db.analyze_sql()` so SQLite can prepare statements with bindings. -2. If analysis raises a SQLite error, reject the query. -3. If every table access is `read`, treat the query as read-only and require `execute-sql` plus `insert-query`/`update-query` as described above. -4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `is_trusted=0`. -5. Reject writable user-created queries that access a database other than the database they are being saved against, until `analyze_sql()` can reliably map attached SQLite schemas back to Datasette database names. -6. For every write access returned by analysis, require the corresponding permission on `TableResource(access.database, access.table)`: - - `insert` -> `insert-row` - - `update` -> `update-row` - - `delete` -> `delete-row` -7. Include write accesses reported from triggers and views, since those are real side effects. -8. Re-run the same analysis and permission checks when SQL changes through `update_query()` or `POST .../-/update`. -9. Re-run analysis before executing user-created writable queries, so schema or trigger changes cannot leave a previously saved query with stale permission assumptions. - -The user-facing API should not trust a submitted `is_write` value. It should derive `is_write` from analysis. - -Trusted configuration and plugin code can still call `datasette.add_query(..., is_write=True, ...)`. Those are treated as deployment/admin-authored queries. They keep the existing execution model: they require `view-query`, and the default `view-query` hook should preserve current default-open behavior for trusted writable queries while still respecting `--default-deny`. - -Fail closed cases for user-created writable queries: - -- Analysis fails. -- Analysis reports any write operation that cannot be mapped to a Datasette table resource. -- Analysis reports writes outside the target database. -- The actor lacks any required table write permission. -- `is_trusted=1` is requested through the user-facing API. - -This gives us writable user-created queries without letting `execute-sql` alone become a path to create arbitrary write endpoints. - -## HTTP API sketch - -JSON endpoints should follow Datasette's existing write API style: use `POST` plus action paths such as `/-/insert`, `/-/update`, and `/-/delete`, not HTTP `PATCH` or `DELETE`. - -Endpoints: - -- `GET /-/queries` and `GET /{database}/-/queries` show searchable HTML query browsers. `GET /-/queries.json` lists query definitions across every database the actor can view; `GET /{database}/-/queries.json` scopes that list to one database. Both JSON endpoints use cursor pagination with `_next` and `_size`. -- `POST /{database}/-/queries/insert` creates a query. -- `GET /{database}/{query}/-/definition` returns one query definition without executing it. -- `POST /{database}/{query}/-/update` updates one query. -- `POST /{database}/{query}/-/delete` deletes one query. - -Create request: - -```json -{ - "query": { - "name": "top_customers", - "sql": "select * from customers order by revenue desc limit 20", - "title": "Top customers", - "description": "Highest revenue customers", - "is_private": true, - "parameters": ["region"] - } -} -``` - -Successful create returns `201` and the created query definition: - -```json -{ - "ok": true, - "query": { - "database": "fixtures", - "name": "top_customers", - "sql": "select * from customers order by revenue desc limit 20", - "title": "Top customers", - "description": "Highest revenue customers", - "is_private": true, - "is_trusted": false, - "parameters": ["region"] - } -} -``` - -Update request, imitating `RowUpdateView`: - -```json -{ - "update": { - "title": "Top customers by revenue", - "is_private": false - }, - "return": true -} -``` - -Successful update returns `{"ok": true}` by default. With `"return": true`, return the updated query definition: - -```json -{ - "ok": true, - "query": { - "database": "fixtures", - "name": "top_customers", - "sql": "select * from customers order by revenue desc limit 20", - "title": "Top customers by revenue", - "is_private": false, - "is_trusted": false - } -} -``` - -Delete request: - -```http -POST /{database}/{query}/-/delete -Content-Type: application/json -``` - -Successful delete returns: - -```json -{ - "ok": true -} -``` - -Validation: - -- Update bodies must be dictionaries containing an `update` dictionary, with optional `return`; invalid keys return `{"ok": false, "errors": [...]}`. -- Validate route-safe query names. -- Reject names that collide with a table or view in the same database, since table routes currently win over query routes. -- Analyze user-created SQL with `Database.analyze_sql()`. -- Use `validate_sql_select(sql)` as the read-only fast path when analysis shows only reads, but do not require it for writable queries that pass analysis and permission checks. -- Reject magic parameters such as `:_actor_id`, `:_cookie_*`, and `:_header_*` for user-created queries. -- Reject client-supplied `is_write`; derive it from analysis. -- Reject writable-only success/error fields for read-only queries. - -## Python API sketch - -Add methods on `Datasette`: - -```python -await datasette.add_query( - database, - name, - sql, - title=None, - description=None, - description_html=None, - hide_sql=False, - fragment=None, - parameters=None, - is_write=False, - is_private=False, - is_trusted=False, - source="plugin", - owner_id=None, - on_success_message=None, - on_success_message_sql=None, - on_success_redirect=None, - on_error_message=None, - on_error_redirect=None, - replace=True, -) - -await datasette.update_query( - database, - name, - *, - sql=UNCHANGED, - title=UNCHANGED, - description=UNCHANGED, - description_html=UNCHANGED, - hide_sql=UNCHANGED, - fragment=UNCHANGED, - parameters=UNCHANGED, - is_write=UNCHANGED, - is_private=UNCHANGED, - is_trusted=UNCHANGED, - source=UNCHANGED, - owner_id=UNCHANGED, - on_success_message=UNCHANGED, - on_success_message_sql=UNCHANGED, - on_success_redirect=UNCHANGED, - on_error_message=UNCHANGED, - on_error_redirect=UNCHANGED, -) - -await datasette.remove_query(database, name, source=None) - -await datasette.get_query(database, name) -await datasette.list_queries( - database, - actor=None, - limit=50, - cursor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, -) -``` - -`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. Passing `database=None` lists visible queries across all live databases, still filtered through `view-query` permission SQL. - -`update_query()` should use an internal sentinel default such as `UNCHANGED = object()` so callers can distinguish "leave this column alone" from "set this column to `NULL`": - -```python -await datasette.update_query( - "fixtures", - "top_customers", - on_success_redirect=None, -) -``` - -For column-backed fields, `None` should write SQL `NULL`. For option fields, `None` should remove that key from the JSON object so `get_query()` returns `None`; omitting the field should leave the existing option unchanged. - -Implementation detail: build the `UPDATE` statement dynamically from fields whose value is not `UNCHANGED`, validate non-nullable fields before writing, and update `updated_at` whenever at least one field changes. - -The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `is_private`, `is_trusted`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. - -## Query page save UI - -On `/{database}/-/query`, if the actor has both `execute-sql` and `insert-query`, show a save control for valid read-only SQL. That page already executes read-only arbitrary SQL, so the first UI can stay read-only even though the JSON API can accept writable SQL after `Database.analyze_sql()` validation. - -The save form should call `POST /{database}/-/queries/insert` and default to `is_private=true`. - -On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. The global `/-/queries` page reuses the same interface and shows the database for each query. - -## Dedicated create query UI - -Add `/{database}/-/queries/-/create` for the fuller query authoring flow, including writable queries. - -This page should require `execute-sql` and `insert-query` to access. It should provide a SQL editor and a mode control: - -- Read-only -- Writable - -Read-only mode can share the same fields as the arbitrary SQL save flow: name, title, description, parameters, and privacy status. - -Writable mode should always run `Database.analyze_sql()` and show an analysis panel before saving: - -- detected operation -- database and table -- required permission -- whether the actor has that permission -- source, when the operation comes from a trigger or view - -The Save button should be disabled until analysis succeeds and every required table write permission is allowed. - -The existing edit-SQL flow from query pages can continue to point back to arbitrary SQL. A later enhancement can add "update this query" when the actor owns it or has `update-query`. - -## Test plan - -- Internal schema creates `queries`. -- Query parameters are stored in the `queries.parameters` text column as a JSON array of names. -- Config `queries:` blocks import into internal tables. -- Legacy string query definitions normalize to SQL rows. -- The old `canned_queries()` hook is no longer called by core. -- `QueryResource.resources_sql()` returns rows from `queries`. -- Database page and `/-/jump` list queries from the internal DB. -- `view-query` remains globally default-allowed, with `restriction_sql` narrowing private queries to their owner. -- Private query is only visible to its owner, even when a broader `view-query` rule applies. -- Non-trusted read-only query requires `execute-sql` to execute. -- Trusted read-only query can be executed without `execute-sql` after `view-query` passes. -- Config queries default to trusted and can opt out with `is_trusted: false`. -- User API rejects client-supplied `is_trusted`. -- User-created query requires both `execute-sql` and `insert-query`. -- User-created writable query creation uses `Database.analyze_sql()` and requires matching `insert-row`, `update-row`, and/or `delete-row` permissions for every reported write access. -- `/{database}/-/queries/-/create` provides the writable-query authoring UI with an analysis panel and disabled save until all required write permissions pass. -- User-created writable query execution re-runs `Database.analyze_sql()` and re-checks table write permissions. -- User-created writable query cannot be trusted through the user API. -- Query update uses `POST /{database}/{query}/-/update` with an `{"update": {...}}` body. -- Query delete uses `POST /{database}/{query}/-/delete`. -- There are no `PATCH` or HTTP `DELETE` routes for query management. -- `datasette.update_query(..., field=None)` writes `NULL` for column-backed fields and removes JSON keys for option fields, while omitted fields are left unchanged. -- Owner gets default `update-query` and `delete-query` for their own user-created rows. -- Admin can manage other users' queries with `update-query` and `delete-query`. -- User API rejects magic parameters. -- User API rejects writable queries if analysis fails, reports writes outside the target database, or reports writes the actor is not allowed to perform. -- Trusted config/plugin writable queries still execute through `view-query`. -- Trusted config/plugin writable queries are not default-allowed under `--default-deny`. -- Persisted internal DB does not expose queries for detached databases. From 24887004cffd52fe801ecd73da78e13b246ddede Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:51:57 -0700 Subject: [PATCH 718/767] Rename insert-query to store-query Also queries/insert to queries/store Refs https://github.com/simonw/datasette/pull/2741#issuecomment-4549103663 --- datasette/app.py | 6 ++--- datasette/default_actions.py | 6 ++--- datasette/templates/query_create.html | 2 +- datasette/views/database.py | 22 +++++++-------- docs/authentication.rst | 7 ++--- docs/json_api.rst | 5 ++-- tests/test_queries.py | 39 +++++++++++++++------------ 7 files changed, 47 insertions(+), 40 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8936b099..42a2d27d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -54,9 +54,9 @@ from .views.database import ( QueryDeleteView, QueryDefinitionView, GlobalQueryListView, - QueryInsertView, QueryListView, QueryParametersView, + QueryStoreView, QueryUpdateView, ) from .views.index import IndexView @@ -2824,8 +2824,8 @@ class Datasette: r"/(?P[^\/\.]+)/-/queries/analyze$", ) add_route( - QueryInsertView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/insert$", + QueryStoreView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/store$", ) add_route( ExecuteWriteAnalyzeView.as_view(self), diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 6a1f77b8..0f4c25fa 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -62,9 +62,9 @@ def register_actions(): resource_class=DatabaseResource, ), Action( - name="insert-query", - abbr="iq", - description="Create saved queries", + name="store-query", + abbr="sq", + description="Create stored queries", resource_class=DatabaseResource, also_requires="execute-sql", ), diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index cb14ada4..f5dadbff 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -156,7 +156,7 @@ form.sql .query-create-sql textarea#sql-editor {

Create query

-
+

{{ urls.database(database) }}/

diff --git a/datasette/views/database.py b/datasette/views/database.py index d40d69d1..900b94ba 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1419,7 +1419,7 @@ class QueryCreateView(BaseView): actor=request.actor, ) await self.ds.ensure_permission( - action="insert-query", + action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ) @@ -1440,11 +1440,11 @@ class QueryCreateAnalyzeView(BaseView): ): return _block_framing(_error(["Permission denied: need execute-sql"], 403)) if not await self.ds.allowed( - action="insert-query", + action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing(_error(["Permission denied: need insert-query"], 403)) + return _block_framing(_error(["Permission denied: need store-query"], 403)) invalid_keys = set(request.args) - {"sql"} if invalid_keys: @@ -1462,8 +1462,8 @@ class QueryCreateAnalyzeView(BaseView): ) -class QueryInsertView(QueryCreateView): - name = "query-insert" +class QueryStoreView(QueryCreateView): + name = "query-store" async def _error_response(self, request, db, query_data, message, status): message = _query_create_form_error_message(message) @@ -1488,11 +1488,11 @@ class QueryInsertView(QueryCreateView): ): return _error(["Permission denied: need execute-sql"], 403) if not await self.ds.allowed( - action="insert-query", + action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return _error(["Permission denied: need insert-query"], 403) + return _error(["Permission denied: need store-query"], 403) is_json = False query_data = {} @@ -1961,8 +1961,8 @@ class QueryView(View): resource=DatabaseResource(database=database), actor=request.actor, ) - allow_insert_query = await datasette.allowed( - action="insert-query", + allow_store_query = await datasette.allowed( + action="store-query", resource=DatabaseResource(database=database), actor=request.actor, ) @@ -2020,13 +2020,13 @@ class QueryView(View): if ( not canned_query and allow_execute_sql - and allow_insert_query + and allow_store_query and is_validated_sql and ":_" not in sql ): save_query_url = ( datasette.urls.database(database) - + "/-/queries/insert?" + + "/-/queries/store?" + urlencode({"sql": sql}) ) diff --git a/docs/authentication.rst b/docs/authentication.rst index 453aaa19..184fec5e 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1293,11 +1293,12 @@ Actor is allowed to view a saved query page, e.g. https://latest.datasette.io/fi ``query`` is the name of the query (string) .. _actions_insert_query: +.. _actions_store_query: -insert-query ------------- +store-query +----------- -Actor is allowed to create saved queries in a database. +Actor is allowed to create stored queries in a database. ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) diff --git a/docs/json_api.rst b/docs/json_api.rst index dd54c459..1a6c7021 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -518,14 +518,15 @@ Listing saved queries Creating saved queries in the UI ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``GET //-/queries/-/create`` provides a form for creating saved queries. +``GET //-/queries/store`` provides a form for creating stored queries. +.. _QueryStoreView: .. _QueryInsertView: Creating saved queries ~~~~~~~~~~~~~~~~~~~~~~ -``POST //-/queries/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. +``POST //-/queries/store`` creates a stored query. This requires ``execute-sql`` and ``store-query`` for the database. .. _QueryParametersView: .. _ExecuteWriteView: diff --git a/tests/test_queries.py b/tests/test_queries.py index 26a0748c..5d4da9bb 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -470,7 +470,7 @@ async def test_query_actions_are_registered(): await ds.invoke_startup() assert ds.get_action("execute-write-sql").resource_class is DatabaseResource - assert ds.get_action("insert-query").resource_class is DatabaseResource + assert ds.get_action("store-query").resource_class is DatabaseResource assert ds.get_action("update-query").resource_class is QueryResource assert ds.get_action("delete-query").resource_class is QueryResource @@ -537,15 +537,15 @@ async def test_analyze_write_query_rejects_writes_to_attached_databases(): @pytest.mark.asyncio -async def test_query_insert_api_creates_read_only_query(): +async def test_query_store_api_creates_read_only_query(): ds = Datasette(memory=True, default_deny=True) ds.root_enabled = True - db = ds.add_memory_database("query_insert_api", name="data") + db = ds.add_memory_database("query_store_api", name="data") await db.execute_write("create table dogs (id integer primary key, name text)") await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, json={ "query": { @@ -860,7 +860,7 @@ async def test_global_query_list_api_and_html(): @pytest.mark.asyncio -async def test_query_insert_api_rejects_is_trusted(): +async def test_query_store_api_rejects_is_trusted(): ds = Datasette( memory=True, default_deny=True, @@ -870,7 +870,7 @@ async def test_query_insert_api_rejects_is_trusted(): "permissions": { "view-database": {"id": "writer"}, "execute-sql": {"id": "writer"}, - "insert-query": {"id": "writer"}, + "store-query": {"id": "writer"}, } } } @@ -880,7 +880,7 @@ async def test_query_insert_api_rejects_is_trusted(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "writer"}, json={"query": {"name": "trusted", "sql": "select 1", "is_trusted": True}}, ) @@ -890,7 +890,7 @@ async def test_query_insert_api_rejects_is_trusted(): @pytest.mark.asyncio -async def test_query_insert_api_creates_writable_query(): +async def test_query_store_api_creates_writable_query(): ds = Datasette(memory=True, default_deny=True) ds.root_enabled = True db = ds.add_memory_database("query_write_api", name="data") @@ -898,7 +898,7 @@ async def test_query_insert_api_creates_writable_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, json={ "query": { @@ -962,14 +962,14 @@ async def test_query_update_and_delete_api(): @pytest.mark.asyncio -async def test_query_insert_api_rejects_magic_parameters(): +async def test_query_store_api_rejects_magic_parameters(): ds = Datasette(memory=True, default_deny=True) ds.root_enabled = True ds.add_memory_database("query_magic_api", name="data") await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, json={"query": {"name": "magic", "sql": "select :_actor_id"}}, ) @@ -987,15 +987,19 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): await ds.invoke_startup() create_response = await ds.client.get( - "/data/-/queries/insert?sql=select+*+from+dogs", + "/data/-/queries/store?sql=select+*+from+dogs", actor={"id": "root"}, ) write_create_response = await ds.client.get( - "/data/-/queries/insert?sql=insert+into+dogs+(name)+values+('Cleo')", + "/data/-/queries/store?sql=insert+into+dogs+(name)+values+('Cleo')", actor={"id": "root"}, ) blank_create_response = await ds.client.get( - "/data/-/queries/insert", + "/data/-/queries/store", + actor={"id": "root"}, + ) + old_insert_response = await ds.client.get( + "/data/-/queries/insert?sql=select+*+from+dogs", actor={"id": "root"}, ) old_create_response = await ds.client.get( @@ -1075,7 +1079,8 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): ) assert query_response.status_code == 200 assert "Save this query" in query_response.text - assert "/data/-/queries/insert?sql=select+%2A+from+dogs" in query_response.text + assert "/data/-/queries/store?sql=select+%2A+from+dogs" in query_response.text + assert old_insert_response.status_code == 404 assert old_create_response.status_code == 404 @@ -1153,7 +1158,7 @@ async def test_create_query_form_error_redisplays_form_with_values(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, data={ "name": "dogs", @@ -1176,7 +1181,7 @@ async def test_create_query_form_error_redisplays_form_with_values(): assert 'name="is_private" value="1" checked' in response.text public_response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, data={ "name": "dogs", From 0cadd071871ef0b33e4ce3a23e316a104b3137c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:53:31 -0700 Subject: [PATCH 719/767] No need to document QueryCreateAnalyzeView --- tests/test_docs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index 396ba1a2..0d0ef1e1 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -66,7 +66,14 @@ def documented_views(): if first_word.endswith("View"): view_labels.add(first_word) # We deliberately don't document these: - view_labels.update(("PatternPortfolioView", "AuthTokenView", "ApiExplorerView")) + view_labels.update( + ( + "PatternPortfolioView", + "AuthTokenView", + "ApiExplorerView", + "QueryCreateAnalyzeView", + ) + ) return view_labels From 4bf1c4b065fef64676abf5eabd04ff35e07188c5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:54:35 -0700 Subject: [PATCH 720/767] Rename canned queries to queries/stored queries in docs --- datasette/default_actions.py | 4 +- datasette/hookspecs.py | 4 +- datasette/resources.py | 2 +- datasette/views/database.py | 24 ++++----- datasette/views/table.py | 4 +- docs/authentication.rst | 16 +++--- docs/configuration.rst | 10 ++-- docs/custom_templates.rst | 8 +-- docs/internals.rst | 12 ++--- docs/introspection.rst | 2 +- docs/json_api.rst | 32 ++++++------ docs/pages.rst | 4 +- docs/plugin_hooks.rst | 16 +++--- docs/spatialite.rst | 2 +- docs/sql_queries.rst | 95 ++++++++++++++++++++++++++---------- tests/test_html.py | 6 +-- tests/test_permissions.py | 4 +- 17 files changed, 144 insertions(+), 101 deletions(-) diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 0f4c25fa..2f78570b 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -121,13 +121,13 @@ def register_actions(): Action( name="update-query", abbr="uq", - description="Update saved queries", + description="Update stored queries", resource_class=QueryResource, ), Action( name="delete-query", abbr="dq", - description="Delete saved queries", + description="Delete stored queries", resource_class=QueryResource, ), ) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index a4067eaa..22da02a4 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -174,7 +174,7 @@ def view_actions(datasette, actor, database, view, request): @hookspec def query_actions(datasette, actor, database, query_name, request, sql, params): - """Links for the query and canned query actions menu""" + """Links for the query and stored query actions menu""" @hookspec @@ -229,7 +229,7 @@ def top_query(datasette, request, database, sql): @hookspec def top_canned_query(datasette, request, database, query_name): - """HTML to include at the top of the canned query page""" + """HTML to include at the top of the stored query page""" @hookspec diff --git a/datasette/resources.py b/datasette/resources.py index 91a46d36..ee2e6d98 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -41,7 +41,7 @@ class TableResource(Resource): class QueryResource(Resource): - """A saved query in a database.""" + """A stored query in a database.""" name = "query" parent_class = DatabaseResource diff --git a/datasette/views/database.py b/datasette/views/database.py index 900b94ba..f30d3815 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -222,11 +222,11 @@ class DatabaseContext(Context): tables: list = field(metadata={"help": "List of table objects in the database"}) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) views: list = field(metadata={"help": "List of view objects in the database"}) - queries: list = field(metadata={"help": "List of canned query objects"}) + queries: list = field(metadata={"help": "List of stored query objects"}) queries_more: bool = field( - metadata={"help": "Boolean indicating if more saved queries are available"} + metadata={"help": "Boolean indicating if more stored queries are available"} ) - queries_count: int = field(metadata={"help": "Count of visible saved queries"}) + queries_count: int = field(metadata={"help": "Count of visible stored queries"}) allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) @@ -272,7 +272,7 @@ class QueryContext(Context): metadata={"help": "The SQL query object containing the `sql` string"} ) canned_query: str = field( - metadata={"help": "The name of the canned query if this is a canned query"} + metadata={"help": "The name of the stored query if this is a stored query"} ) private: bool = field( metadata={"help": "Boolean indicating if this is a private database"} @@ -282,11 +282,11 @@ class QueryContext(Context): # ) canned_query_write: bool = field( metadata={ - "help": "Boolean indicating if this is a canned query that allows writes" + "help": "Boolean indicating if this is a stored query that allows writes" } ) metadata: dict = field( - metadata={"help": "Metadata about the database or the canned query"} + metadata={"help": "Metadata about the database or the stored query"} ) db_is_immutable: bool = field( metadata={"help": "Boolean indicating if this database is immutable"} @@ -315,7 +315,7 @@ class QueryContext(Context): metadata={"help": "Dictionary of parameter names/values"} ) edit_sql_url: str = field( - metadata={"help": "URL to edit the SQL for a canned query"} + metadata={"help": "URL to edit the SQL for a stored query"} ) display_rows: list = field(metadata={"help": "List of result rows to display"}) columns: list = field(metadata={"help": "List of column names"}) @@ -1623,7 +1623,7 @@ class QueryView(View): db = await datasette.resolve_database(request) - # We must be a canned query + # We must be a stored query table_found = False try: await datasette.resolve_table(request) @@ -1742,14 +1742,14 @@ class QueryView(View): # Create lookup dict for quick access allowed_dict = {r.child: r for r in allowed_tables_page.resources} - # Are we a canned query? + # Are we a stored query? canned_query = None canned_query_write = False if "table" in request.url_vars: try: await datasette.resolve_table(request) except TableNotFound as table_not_found: - # Was this actually a canned query? + # Was this actually a stored query? canned_query = await datasette.get_canned_query( table_not_found.database_name, table_not_found.table, request.actor ) @@ -1759,7 +1759,7 @@ class QueryView(View): private = False if canned_query: - # Respect canned query permissions + # Respect stored query permissions visible, private = await datasette.check_visibility( request.actor, action="view-query", @@ -1823,7 +1823,7 @@ class QueryView(View): # For regular queries we only allow SELECT, plus other rules validate_sql_select(sql) else: - # Canned queries can run magic parameters + # Stored queries can run magic parameters params_for_query = MagicParameters(sql, params, request, datasette) await params_for_query.execute_params() results = await datasette.execute( diff --git a/datasette/views/table.py b/datasette/views/table.py index 7027bb10..7b1a5a82 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -963,11 +963,11 @@ async def table_view_traced(datasette, request): try: resolved = await datasette.resolve_table(request) except TableNotFound as not_found: - # Was this actually a canned query? + # Was this actually a stored query? canned_query = await datasette.get_canned_query( not_found.database_name, not_found.table, request.actor ) - # If this is a canned query, not a table, then dispatch to QueryView instead + # If this is a stored query, not a table, then dispatch to QueryView instead if canned_query: return await QueryView()(request, datasette) else: diff --git a/docs/authentication.rst b/docs/authentication.rst index 184fec5e..22db41d8 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -468,7 +468,7 @@ You can control the following: * Access to the entire Datasette instance * Access to specific databases * Access to specific tables and views -* Access to specific :ref:`canned_queries` +* Access to specific :ref:`queries ` If a user has permission to view a table they will be able to view that table, independent of if they have permission to view the database or instance that the table exists within. @@ -641,12 +641,12 @@ This works for SQL views as well - you can list their names in the ``"tables"`` .. _authentication_permissions_query: -Access to specific canned queries ---------------------------------- +Access to specific queries +-------------------------- -:ref:`canned_queries` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. +:ref:`Queries ` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. -To limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user`: +To limit access to the ``add_name`` query in your ``dogs.db`` database to just the :ref:`root user`: .. [[[cog config_example(cog, """ @@ -1285,7 +1285,7 @@ Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.i view-query ---------- -Actor is allowed to view a saved query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size. Executing an untrusted saved query also requires ``execute-sql`` or the relevant write permissions; trusted saved queries can execute with ``view-query`` alone. +Actor is allowed to view a stored query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size. Executing an untrusted stored query also requires ``execute-sql`` or the relevant write permissions; :ref:`trusted stored queries ` can execute with ``view-query`` alone. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) @@ -1308,7 +1308,7 @@ Actor is allowed to create stored queries in a database. update-query ------------ -Actor is allowed to update a saved query. +Actor is allowed to update a stored query. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) @@ -1320,7 +1320,7 @@ Actor is allowed to update a saved query. delete-query ------------ -Actor is allowed to delete a saved query. +Actor is allowed to delete a stored query. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) diff --git a/docs/configuration.rst b/docs/configuration.rst index 8c8c8a67..cf9590b8 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -87,6 +87,7 @@ This is equivalent to a ``datasette.yaml`` file containing the following: } .. [[[end]]] + .. _configuration_reference: ``datasette.yaml`` reference @@ -435,10 +436,10 @@ Here is a simple example: .. _configuration_reference_canned_queries: -Canned queries configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Queries configuration +~~~~~~~~~~~~~~~~~~~~~ -:ref:`Canned queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: +:ref:`Queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: .. [[[cog from metadata_doc import config_example, config_example @@ -483,7 +484,7 @@ Canned queries configuration } .. [[[end]]] -See the :ref:`canned queries documentation ` for more, including how to configure :ref:`writable canned queries `. +See the :ref:`queries documentation ` for more, including how to configure :ref:`writable queries `. .. _configuration_reference_css_js: @@ -1211,4 +1212,3 @@ For column types that accept additional configuration, use an object with ``type } } .. [[[end]]] - diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 8cc40f0f..c324fb79 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -29,7 +29,7 @@ The custom SQL template (``/dbname?sql=...``) gets this: -A canned query template (``/dbname/queryname``) gets this: +A stored query template (``/dbname/queryname``) gets this: .. code-block:: html @@ -193,8 +193,8 @@ The lookup rules Datasette uses are as follows:: query-mydatabase.html query.html - Canned query page (/mydatabase/canned-query): - query-mydatabase-canned-query.html + Stored query page (/mydatabase/query-name): + query-mydatabase-query-name.html query-mydatabase.html query.html @@ -230,7 +230,7 @@ will look something like this:: -This example is from the canned query page for a query called "tz" in the +This example is from the stored query page for a query called "tz" in the database called "mydb". The asterisk shows which template was selected - so in this case, Datasette found a template file called ``query-mydb-tz.html`` and used that - but if that template had not been found, it would have tried for diff --git a/docs/internals.rst b/docs/internals.rst index c76de487..084922f8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -725,7 +725,7 @@ The builder methods are: - ``allow_all(action)`` - allow an action across all databases and resources - ``allow_database(database, action)`` - allow an action on a specific database -- ``allow_resource(database, resource, action)`` - allow an action on a specific resource (table, SQL view or :ref:`canned query `) within a database +- ``allow_resource(database, resource, action)`` - allow an action on a specific resource (table, SQL view or :ref:`stored query `) within a database Each method returns the ``TokenRestrictions`` instance so calls can be chained. @@ -837,10 +837,10 @@ await .get_resource_metadata(self, database_name, resource_name) ``database_name`` - string The name of the database to query. ``resource_name`` - string - The name of the resource (table, view, or canned query) inside ``database_name`` to query. + The name of the resource (table, view, or stored query) inside ``database_name`` to query. Returns metadata keys and values for the specified "resource" as a dictionary. -A "resource" in this context can be a table, view, or canned query. +A "resource" in this context can be a table, view, or stored query. Internally queries the ``metadata_resources`` table inside the :ref:`internal database `. .. _datasette_get_column_metadata: @@ -851,7 +851,7 @@ await .get_column_metadata(self, database_name, resource_name, column_name) ``database_name`` - string The name of the database to query. ``resource_name`` - string - The name of the resource (table, view, or canned query) inside ``database_name`` to query. + The name of the resource (table, view, or stored query) inside ``database_name`` to query. ``column_name`` - string The name of the column inside ``resource_name`` to query. @@ -897,7 +897,7 @@ await .set_resource_metadata(self, database_name, resource_name, key, value) ``database_name`` - string The database the metadata entry belongs to. ``resource_name`` - string - The resource (table, view, or canned query) the metadata entry belongs to. + The resource (table, view, or stored query) the metadata entry belongs to. ``key`` - string The metadata entry key to insert (ex ``title``, ``description``, etc.) ``value`` - string @@ -915,7 +915,7 @@ await .set_column_metadata(self, database_name, resource_name, column_name, key, ``database_name`` - string The database the metadata entry belongs to. ``resource_name`` - string - The resource (table, view, or canned query) the metadata entry belongs to. + The resource (table, view, or stored query) the metadata entry belongs to. ``column-name`` - string The column the metadata entry belongs to. ``key`` - string diff --git a/docs/introspection.rst b/docs/introspection.rst index d2eb8efd..7702a4b5 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -149,7 +149,7 @@ Shows currently attached databases. `Databases example /-/queries.json`` returns saved query definitions for a specific database. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. +``GET /-/queries.json`` returns stored query definitions across every database that the actor can view. ``GET //-/queries.json`` returns stored query definitions for a specific database. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. .. _QueryCreateView: -Creating saved queries in the UI -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Creating stored queries in the UI +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``GET //-/queries/store`` provides a form for creating stored queries. .. _QueryStoreView: .. _QueryInsertView: -Creating saved queries -~~~~~~~~~~~~~~~~~~~~~~ +Creating stored queries +~~~~~~~~~~~~~~~~~~~~~~~ ``POST //-/queries/store`` creates a stored query. This requires ``execute-sql`` and ``store-query`` for the database. @@ -545,24 +545,24 @@ Executing write SQL .. _QueryDefinitionView: -Getting a saved query definition -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Getting a stored query definition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``GET ///-/definition`` returns a saved query definition without executing it. +``GET ///-/definition`` returns a stored query definition without executing it. .. _QueryUpdateView: -Updating saved queries -~~~~~~~~~~~~~~~~~~~~~~ +Updating stored queries +~~~~~~~~~~~~~~~~~~~~~~~ -``POST ///-/update`` updates a saved query using a JSON body with an ``"update"`` object. +``POST ///-/update`` updates a stored query using a JSON body with an ``"update"`` object. .. _QueryDeleteView: -Deleting saved queries -~~~~~~~~~~~~~~~~~~~~~~ +Deleting stored queries +~~~~~~~~~~~~~~~~~~~~~~~ -``POST ///-/delete`` deletes a saved query. +``POST ///-/delete`` deletes a stored query. .. _TableInsertView: diff --git a/docs/pages.rst b/docs/pages.rst index 34c851a5..e57c15e6 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -28,7 +28,7 @@ The index page can also be accessed at ``/-/``, useful for if the default index Database ======== -Each database has a page listing the tables, views and canned queries available for that database. If the :ref:`actions_execute_sql` permission is enabled (it's on by default) there will also be an interface for executing arbitrary SQL select queries against the data. +Each database has a page listing the tables, views and stored queries available for that database. If the :ref:`actions_execute_sql` permission is enabled (it's on by default) there will also be an interface for executing arbitrary SQL select queries against the data. Examples: @@ -68,7 +68,7 @@ This means you can link directly to a query by constructing the following URL: ``/database-name/-/query?sql=SELECT+*+FROM+table_name`` -Each configured :ref:`canned query ` has its own page, at ``/database-name/query-name``. Viewing this page will execute the query and display the results. +Each configured :ref:`stored query ` has its own page, at ``/database-name/query-name``. Viewing this page will execute the query and display the results. In both cases adding a ``.json`` extension to the URL will return the results as JSON. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index b2676b3e..264b473e 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -609,7 +609,7 @@ When a request is received, the ``"render"`` callback function is called with ze The SQL query that was executed. ``query_name`` - string or None - If this was the execution of a :ref:`canned query `, the name of that query. + If this was the execution of a :ref:`stored query `, the name of that query. ``database`` - string The name of the database. @@ -1212,7 +1212,7 @@ Examples: `datasette-saved-queries `__ @@ -1635,7 +1635,7 @@ register_magic_parameters(datasette) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. -:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`canned queries `. This plugin hook allows additional magic parameters to be defined by plugins. +:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`configured queries `. This plugin hook allows additional magic parameters to be defined by plugins. Magic parameters all take this format: ``_prefix_rest_of_parameter``. The prefix indicates which magic parameter function should be called - the rest of the parameter is passed as an argument to that function. @@ -1828,7 +1828,7 @@ jump_items_sql(datasette, actor, request) This hook allows plugins to add extra results to Datasette's ``/`` jump menu, which is powered by the ``/-/jump`` JSON endpoint. -Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be searched alongside Datasette's own databases, tables, views and canned query results. The hook can also be an ``async def`` function, or return an awaitable that resolves to one of these values. +Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be searched alongside Datasette's own databases, tables, views and stored query results. The hook can also be an ``async def`` function, or return an awaitable that resolves to one of these values. ``JumpSQL`` queries run against Datasette's internal database by default. To run a query against another database, pass its name as the optional ``database=`` argument. For example, ``JumpSQL(database="content", sql="...")`` runs against the ``content`` database. @@ -2004,7 +2004,7 @@ query_actions(datasette, actor, database, query_name, request, sql, params) The name of the database. ``query_name`` - string or None - The name of the canned query, or ``None`` if this is an arbitrary SQL query. + The name of the stored query, or ``None`` if this is an arbitrary SQL query. ``request`` - :ref:`internals_request` The current HTTP request. @@ -2015,7 +2015,7 @@ query_actions(datasette, actor, database, query_name, request, sql, params) ``params`` - dictionary The parameters passed to the SQL query, if any. -Populates a "Query actions" menu on the canned query and arbitrary SQL query pages. +Populates a "Query actions" menu on the stored query and arbitrary SQL query pages. This example adds a new query action linking to a page for explaining a query: @@ -2294,9 +2294,9 @@ top_canned_query(datasette, request, database, query_name) The name of the database. ``query_name`` - string - The name of the canned query. + The name of the stored query. -Returns HTML to be displayed at the top of the canned query page. +Returns HTML to be displayed at the top of the stored query page. .. _plugin_event_tracking: diff --git a/docs/spatialite.rst b/docs/spatialite.rst index c93c1e00..1999ab78 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -30,7 +30,7 @@ Warning The following steps are recommended: - Disable arbitrary SQL queries by untrusted users. See :ref:`authentication_permissions_execute_sql` for ways to do this. The easiest is to start Datasette with the ``datasette --setting default_allow_sql off`` option. - - Define :ref:`canned_queries` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. + - Define :ref:`queries ` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. The `Datasette SpatiaLite tutorial `__ includes detailed instructions for running SpatiaLite safely using these techniques diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 7c3cd4ac..d60656e3 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -68,10 +68,10 @@ You can also use the `sqlite-utils `__ tool .. _canned_queries: -Canned queries --------------- +Queries +------- -As an alternative to adding views to your database, you can define canned queries inside your ``datasette.yaml`` file. Here's an example: +As an alternative to adding views to your database, you can define named queries inside your ``datasette.yaml`` file. Here's an example: .. [[[cog from metadata_doc import config_example, config_example @@ -120,24 +120,67 @@ Then run Datasette like this:: datasette sf-trees.db -m metadata.json -Each canned query will be listed on the database index page, and will also get its own URL at:: +Each configured query will be listed on the database index page, and will also get its own URL at:: - /database-name/canned-query-name + /database-name/query-name For the above example, that URL would be:: /sf-trees/just_species -You can optionally include ``"title"`` and ``"description"`` keys to show a title and description on the canned query page. As with regular table metadata you can alternatively specify ``"description_html"`` to have your description rendered as HTML (rather than having HTML special characters escaped). +You can optionally include ``"title"`` and ``"description"`` keys to show a title and description on the query page. As with regular table metadata you can alternatively specify ``"description_html"`` to have your description rendered as HTML (rather than having HTML special characters escaped). + +.. _stored_queries: +.. _saved_queries: + +Stored queries +~~~~~~~~~~~~~~ + +Datasette stores both configured queries and user-created queries in the ``queries`` table in the :ref:`internal database `. Configured queries come from the ``queries`` section of ``datasette.yaml``. User-created stored queries can be created from the SQL query page by actors with the :ref:`actions_store_query` and :ref:`actions_execute_sql` permissions. Writable stored queries also require the permissions needed for the writes they perform. + +Stored queries created by users default to private. Private stored queries can only be viewed, updated or deleted by the actor that created them. Broad ``view-query``, ``update-query`` or ``delete-query`` permission grants still do not allow other actors to access another actor's private stored queries. + +Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. + +.. _trusted_stored_queries: +.. _trusted_saved_queries: + +Trusted stored queries +++++++++++++++++++++++ + +A trusted stored query can execute with ``view-query`` permission alone. It skips the additional ``execute-sql`` and write permission checks that are applied to untrusted stored queries. + +Trusted stored queries should only be used for SQL that has been reviewed by someone trusted to configure the Datasette instance. For that reason, trusted stored queries can only be added using configuration. Users cannot create trusted stored queries through the web interface or the stored query JSON API. + +Queries defined in ``datasette.yaml`` are trusted by default: + +.. code-block:: yaml + + databases: + mydatabase: + queries: + report: + sql: select * from report + +You can opt out of this behavior for a configured query using ``is_trusted: false``: + +.. code-block:: yaml + + databases: + mydatabase: + queries: + report: + sql: select * from report + is_trusted: false .. _canned_queries_named_parameters: -Canned query parameters -~~~~~~~~~~~~~~~~~~~~~~~ +Query parameters +~~~~~~~~~~~~~~~~ -Canned queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the canned query page or by adding them to the URL. This means canned queries can be used to create custom JSON APIs based on a carefully designed SQL statement. +Configured queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the query page or by adding them to the URL. This means configured queries can be used to create custom JSON APIs based on a carefully designed SQL statement. -Here's an example of a canned query with a named parameter: +Here's an example of a configured query with a named parameter: .. code-block:: sql @@ -147,7 +190,7 @@ Here's an example of a canned query with a named parameter: where neighborhood like '%' || :text || '%' order by neighborhood; -In the canned query configuration looks like this: +The query configuration looks like this: .. [[[cog @@ -204,7 +247,7 @@ In the canned query configuration looks like this: Note that we are using SQLite string concatenation here - the ``||`` operator - to add wildcard ``%`` characters to the string provided by the user. -You can try this canned query out here: +You can try this query out here: https://latest.datasette.io/fixtures/neighborhood_search?text=town In this example the ``:text`` named parameter is automatically extracted from the query using a regular expression. @@ -272,15 +315,15 @@ You can alternatively provide an explicit list of named parameters using the ``" .. _canned_queries_options: -Additional canned query options -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Additional query options +~~~~~~~~~~~~~~~~~~~~~~~~ -Additional options can be specified for canned queries in the YAML or JSON configuration. +Additional options can be specified for configured queries in the YAML or JSON configuration. hide_sql ++++++++ -Canned queries default to displaying their SQL query at the top of the page. If the query is extremely long you may want to hide it by default, with a "show" link that can be used to make it visible. +Configured queries default to displaying their SQL query at the top of the page. If the query is extremely long you may want to hide it by default, with a "show" link that can be used to make it visible. Add the ``"hide_sql": true`` option to hide the SQL query by default. @@ -289,7 +332,7 @@ fragment Some plugins, such as `datasette-vega `__, can be configured by including additional data in the fragment hash of the URL - the bit that comes after a ``#`` symbol. -You can set a default fragment hash that will be included in the link to the canned query from the database index page using the ``"fragment"`` key. +You can set a default fragment hash that will be included in the link to the query from the database index page using the ``"fragment"`` key. This example demonstrates both ``fragment`` and ``hide_sql``: @@ -348,12 +391,12 @@ This example demonstrates both ``fragment`` and ``hide_sql``: .. _canned_queries_writable: -Writable canned queries -~~~~~~~~~~~~~~~~~~~~~~~ +Writable queries +~~~~~~~~~~~~~~~~ -Canned queries by default are read-only. You can use the ``"write": true`` key to indicate that a canned query can write to the database. +Configured queries are read-only by default. You can use the ``"write": true`` key to indicate that a query can write to the database. -See :ref:`authentication_permissions_query` for details on how to add permission checks to canned queries, using the ``"allow"`` key. +See :ref:`authentication_permissions_query` for details on how to add permission checks to queries, using the ``"allow"`` key. .. [[[cog config_example(cog, { @@ -488,7 +531,7 @@ Magic parameters Named parameters that start with an underscore are special: they can be used to automatically add values created by Datasette that are not contained in the incoming form fields or query string. -These magic parameters are only supported for canned queries: to avoid security issues (such as queries that extract the user's private cookies) they are not available to SQL that is executed by the user as a custom SQL query. +These magic parameters are only supported for configured queries: to avoid security issues (such as queries that extract the user's private cookies) they are not available to SQL that is executed by the user as a custom SQL query. Available magic parameters are: @@ -580,12 +623,12 @@ Additional custom magic parameters can be added by plugins using the :ref:`plugi .. _canned_queries_json_api: -JSON API for writable canned queries -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +JSON API for writable queries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Writable canned queries can also be accessed using a JSON API. You can POST data to them using JSON, and you can request that their response is returned to you as JSON. +Writable queries can also be accessed using a JSON API. You can POST data to them using JSON, and you can request that their response is returned to you as JSON. -To submit JSON to a writable canned query, encode key/value parameters as a JSON document:: +To submit JSON to a writable query, encode key/value parameters as a JSON document:: POST /mydatabase/add_message diff --git a/tests/test_html.py b/tests/test_html.py index 9e460da1..8edb9f6e 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -154,7 +154,7 @@ async def test_database_page(ds_client): ("/fixtures/simple_view", "simple_view"), ] == sorted([(a["href"], a.text) for a in views_ul.find_all("a")]) - # And a list of canned queries + # And a list of stored queries queries_ul = soup.find("h2", string="Queries").find_next_sibling("ul") assert queries_ul is not None assert [ @@ -701,7 +701,7 @@ async def test_show_hide_sql_query(ds_client): @pytest.mark.asyncio async def test_canned_query_with_hide_has_no_hidden_sql(ds_client): - # For a canned query the show/hide should NOT have a hidden SQL field + # For a stored query the show/hide should NOT have a hidden SQL field # https://github.com/simonw/datasette/issues/1411 response = await ds_client.get("/fixtures/pragma_cache_size?_hide_sql=1") soup = Soup(response.content, "html.parser") @@ -1106,7 +1106,7 @@ async def test_trace_correctly_escaped(ds_client): "/fixtures/-/query?sql=select+*+from+facetable", "http://localhost/fixtures/-/query.json?sql=select+*+from+facetable", ), - # Canned query page + # Stored query page ( "/fixtures/neighborhood_search?text=town", "http://localhost/fixtures/neighborhood_search.json?text=town", diff --git a/tests/test_permissions.py b/tests/test_permissions.py index eb6cee9f..0e38c876 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -890,7 +890,7 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "t1"), expected_result=True, ), - # view-query on canned query, wrong actor + # view-query on stored query, wrong actor PermConfigTestCase( config={ "databases": { @@ -909,7 +909,7 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "q1"), expected_result=False, ), - # view-query on canned query, right actor + # view-query on stored query, right actor PermConfigTestCase( config={ "databases": { From b1029acc68626c2fddf7b678adc3339be0fce6e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:05:41 -0700 Subject: [PATCH 721/767] top_canned_query is now top_stored_query, closes #2747 --- datasette/hookspecs.py | 2 +- datasette/templates/query.html | 2 +- datasette/views/database.py | 8 ++++---- docs/changelog.rst | 1 + docs/plugin_hooks.rst | 4 ++-- tests/test_plugins.py | 10 ++++++---- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 22da02a4..dcd502af 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -228,7 +228,7 @@ def top_query(datasette, request, database, sql): @hookspec -def top_canned_query(datasette, request, database, query_name): +def top_stored_query(datasette, request, database, query_name): """HTML to include at the top of the stored query page""" diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 785b05af..3f03424a 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -33,7 +33,7 @@ {% set action_links, action_title = query_actions(), "Query actions" %} {% include "_action_menu.html" %} -{% if canned_query %}{{ top_canned_query() }}{% else %}{{ top_query() }}{% endif %} +{% if canned_query %}{{ top_stored_query() }}{% else %}{{ top_query() }}{% endif %} {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index f30d3815..def3c530 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -339,8 +339,8 @@ class QueryContext(Context): top_query: callable = field( metadata={"help": "Callable to render the top_query slot"} ) - top_canned_query: callable = field( - metadata={"help": "Callable to render the top_canned_query slot"} + top_stored_query: callable = field( + metadata={"help": "Callable to render the top_stored_query slot"} ) query_actions: callable = field( metadata={ @@ -2095,8 +2095,8 @@ class QueryView(View): top_query=make_slot_function( "top_query", datasette, request, database=database, sql=sql ), - top_canned_query=make_slot_function( - "top_canned_query", + top_stored_query=make_slot_function( + "top_stored_query", datasette, request, database=database, diff --git a/docs/changelog.rst b/docs/changelog.rst index dfb2a736..300ac02f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) +- The ``top_canned_query()`` plugin hook has been renamed to :ref:`top_stored_query() `. (:issue:`2747`) .. _v1_0_a30: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 264b473e..4737ca03 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -2279,9 +2279,9 @@ top_query(datasette, request, database, sql) Returns HTML to be displayed at the top of the query results page. -.. _plugin_hook_top_canned_query: +.. _plugin_hook_top_stored_query: -top_canned_query(datasette, request, database, query_name) +top_stored_query(datasette, request, database, query_name) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``datasette`` - :ref:`internals_datasette` diff --git a/tests/test_plugins.py b/tests/test_plugins.py index f7adbd66..32276437 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1486,8 +1486,10 @@ class SlotPlugin: return "Xtop_query:{}:{}:{}".format(database, sql, request.args["z"]) @hookimpl - def top_canned_query(self, request, database, query_name): - return "Xtop_query:{}:{}:{}".format(database, query_name, request.args["z"]) + def top_stored_query(self, request, database, query_name): + return "Xtop_stored_query:{}:{}:{}".format( + database, query_name, request.args["z"] + ) @pytest.mark.asyncio @@ -1548,12 +1550,12 @@ async def test_hook_top_query(ds_client): @pytest.mark.asyncio -async def test_hook_top_canned_query(ds_client): +async def test_hook_top_stored_query(ds_client): try: pm.register(SlotPlugin(), name="SlotPlugin") response = await ds_client.get("/fixtures/magic_parameters?z=xyz") assert response.status_code == 200 - assert "Xtop_query:fixtures:magic_parameters:xyz" in response.text + assert "Xtop_stored_query:fixtures:magic_parameters:xyz" in response.text finally: pm.unregister(name="SlotPlugin") From 2f73869c09962e320e5f40f4691df70618cd052e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:09:48 -0700 Subject: [PATCH 722/767] Document that canned_queries() has been removed --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 300ac02f..674ff5b3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) - The ``top_canned_query()`` plugin hook has been renamed to :ref:`top_stored_query() `. (:issue:`2747`) +- The ``canned_queries()`` plugin hook has been removed. Plugins can use the new ``datasette.add_query()``, ``datasette.update_query()`` and ``datasette.remove_query()`` methods to managed stored queries instead. .. _v1_0_a30: From 56b14f37d547e03ba902516ac9ae13ef52765f77 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:16:18 -0700 Subject: [PATCH 723/767] The stored queries do not live in that DB --- docs/authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 22db41d8..86df7f04 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1298,7 +1298,7 @@ Actor is allowed to view a stored query page, e.g. https://latest.datasette.io/f store-query ----------- -Actor is allowed to create stored queries in a database. +Actor is allowed to create stored queries against a database. ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) From 02a1468f1b3c8c14fb80037686b43de856e49c1f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:17:51 -0700 Subject: [PATCH 724/767] Renamed canned queries to queries / stored queries in docs And a few renames in code and YAML as well. --- .github/workflows/deploy-latest.yml | 33 +- datasette/app.py | 7 - datasette/facets.py | 2 +- datasette/static/app.css | 2 +- datasette/templates/query.html | 18 +- datasette/views/database.py | 92 +++--- datasette/views/table.py | 6 +- docs/authentication.rst | 10 +- docs/changelog.rst | 23 +- docs/configuration.rst | 6 +- docs/plugin_hooks.rst | 12 +- docs/spatialite.rst | 2 +- docs/sql_queries.rst | 12 +- docs/upgrade-1.0a20.md | 6 +- tests/test_canned_queries.py | 473 ---------------------------- tests/test_html.py | 12 +- tests/test_jump.py | 4 +- 17 files changed, 115 insertions(+), 605 deletions(-) delete mode 100644 tests/test_canned_queries.py diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 7d8dd37d..166d33d0 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -57,7 +57,7 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - - name: And the counters writable canned query demo + - name: And the counters writable stored query demo run: | cat > plugins/counters.py <This query cannot be executed because the database is immutable.

{% endif %} -

{{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

+

{{ metadata.title or database }}{% if stored_query and not metadata.title %}: {{ stored_query }}{% endif %}{% if private %} 🔒{% endif %}

{% set action_links, action_title = query_actions(), "Query actions" %} {% include "_action_menu.html" %} -{% if canned_query %}{{ top_stored_query() }}{% else %}{{ top_query() }}{% endif %} +{% if stored_query %}{{ top_stored_query() }}{% else %}{{ top_query() }}{% endif %} {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

@@ -52,7 +52,7 @@ 
{% if query %}{{ query.sql }}{% endif %}
{% endif %} {% else %} - {% if not canned_query %} + {% if not stored_query %} @@ -64,10 +64,10 @@ {% include "_sql_parameters.html" %}

{% if not hide_sql %}{% endif %} - + {{ show_hide_hidden }} {% if save_query_url %}Save this query{% endif %} - {% if canned_query and edit_sql_url %}Edit SQL{% endif %} + {% if stored_query and edit_sql_url %}Edit SQL{% endif %}

@@ -90,7 +90,7 @@
Required permission
{% else %} - {% if not canned_query_write and not error %} + {% if not stored_query_write and not error %}

0 results

{% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index def3c530..c36476f6 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -100,12 +100,12 @@ class DatabaseView(View): limit=5, include_private=True, ) - canned_queries = queries_page["queries"] + stored_queries = queries_page["queries"] queries_more = queries_page["has_more"] queries_count = ( await datasette.count_queries(database, actor=request.actor) if queries_more - else len(canned_queries) + else len(stored_queries) ) async def database_actions(): @@ -137,7 +137,7 @@ class DatabaseView(View): "tables": tables, "hidden_count": len([t for t in tables if t["hidden"]]), "views": sql_views, - "queries": canned_queries, + "queries": stored_queries, "queries_more": queries_more, "queries_count": queries_count, "allow_execute_sql": allow_execute_sql, @@ -172,7 +172,7 @@ class DatabaseView(View): tables=tables, hidden_count=len([t for t in tables if t["hidden"]]), views=sql_views, - queries=canned_queries, + queries=stored_queries, queries_more=queries_more, queries_count=queries_count, allow_execute_sql=allow_execute_sql, @@ -271,7 +271,7 @@ class QueryContext(Context): query: dict = field( metadata={"help": "The SQL query object containing the `sql` string"} ) - canned_query: str = field( + stored_query: str = field( metadata={"help": "The name of the stored query if this is a stored query"} ) private: bool = field( @@ -280,7 +280,7 @@ class QueryContext(Context): # urls: dict = field( # metadata={"help": "Object containing URL helpers like `database()`"} # ) - canned_query_write: bool = field( + stored_query_write: bool = field( metadata={ "help": "Boolean indicating if this is a stored query that allows writes" } @@ -1629,10 +1629,10 @@ class QueryView(View): await datasette.resolve_table(request) table_found = True except TableNotFound as table_not_found: - canned_query = await datasette.get_canned_query( - table_not_found.database_name, table_not_found.table, request.actor + stored_query = await datasette.get_query( + table_not_found.database_name, table_not_found.table ) - if canned_query is None: + if stored_query is None: raise if table_found: # That should not have happened @@ -1640,13 +1640,13 @@ class QueryView(View): if not await datasette.allowed( action="view-query", - resource=QueryResource(database=db.name, query=canned_query["name"]), + resource=QueryResource(database=db.name, query=stored_query["name"]), actor=request.actor, ): raise Forbidden("You do not have permission to view this query") await _ensure_stored_query_execution_permissions( - datasette, db, canned_query, request.actor + datasette, db, stored_query, request.actor ) # If database is immutable, return an error @@ -1674,19 +1674,19 @@ class QueryView(View): or params.get("_json") ) params_for_query = MagicParameters( - canned_query["sql"], params, request, datasette + stored_query["sql"], params, request, datasette ) await params_for_query.execute_params() ok = None redirect_url = None try: cursor = await db.execute_write( - canned_query["sql"], params_for_query, request=request + stored_query["sql"], params_for_query, request=request ) # success message can come from on_success_message or on_success_message_sql message = None message_type = datasette.INFO - on_success_message_sql = canned_query.get("on_success_message_sql") + on_success_message_sql = stored_query.get("on_success_message_sql") if on_success_message_sql: try: message_result = ( @@ -1698,18 +1698,18 @@ class QueryView(View): message = "Error running on_success_message_sql: {}".format(ex) message_type = datasette.ERROR if not message: - message = canned_query.get( + message = stored_query.get( "on_success_message" ) or "Query executed, {} row{} affected".format( cursor.rowcount, "" if cursor.rowcount == 1 else "s" ) - redirect_url = canned_query.get("on_success_redirect") + redirect_url = stored_query.get("on_success_redirect") ok = True except Exception as ex: - message = canned_query.get("on_error_message") or str(ex) + message = stored_query.get("on_error_message") or str(ex) message_type = datasette.ERROR - redirect_url = canned_query.get("on_error_redirect") + redirect_url = stored_query.get("on_error_redirect") ok = False if should_return_json: return Response.json( @@ -1743,33 +1743,33 @@ class QueryView(View): allowed_dict = {r.child: r for r in allowed_tables_page.resources} # Are we a stored query? - canned_query = None - canned_query_write = False + stored_query = None + stored_query_write = False if "table" in request.url_vars: try: await datasette.resolve_table(request) except TableNotFound as table_not_found: # Was this actually a stored query? - canned_query = await datasette.get_canned_query( - table_not_found.database_name, table_not_found.table, request.actor + stored_query = await datasette.get_query( + table_not_found.database_name, table_not_found.table ) - if canned_query is None: + if stored_query is None: raise - canned_query_write = bool(canned_query.get("write")) + stored_query_write = bool(stored_query.get("write")) private = False - if canned_query: + if stored_query: # Respect stored query permissions visible, private = await datasette.check_visibility( request.actor, action="view-query", - resource=QueryResource(database=database, query=canned_query["name"]), + resource=QueryResource(database=database, query=stored_query["name"]), ) if not visible: raise Forbidden("You do not have permission to view this query") - if not canned_query_write: + if not stored_query_write: await _ensure_stored_query_execution_permissions( - datasette, db, canned_query, request.actor + datasette, db, stored_query, request.actor ) else: @@ -1783,15 +1783,15 @@ class QueryView(View): params = {key: request.args.get(key) for key in request.args} sql = None - if canned_query: - sql = canned_query["sql"] + if stored_query: + sql = stored_query["sql"] elif "sql" in params: sql = params.pop("sql") # Extract any :named parameters named_parameters = [] - if canned_query and canned_query.get("params"): - named_parameters = canned_query["params"] + if stored_query and stored_query.get("params"): + named_parameters = stored_query["params"] if not named_parameters and sql: named_parameters = derive_named_parameters(sql) named_parameter_values = { @@ -1817,9 +1817,9 @@ class QueryView(View): params_for_query = params - if sql and not canned_query_write: + if sql and not stored_query_write: try: - if not canned_query: + if not stored_query: # For regular queries we only allow SELECT, plus other rules validate_sql_select(sql) else: @@ -1879,7 +1879,7 @@ class QueryView(View): columns=columns, rows=rows, sql=sql, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query["name"] if stored_query else None, database=database, table=None, request=request, @@ -1911,10 +1911,10 @@ class QueryView(View): elif format_ == "html": headers = {} templates = [f"query-{to_css_class(database)}.html", "query.html"] - if canned_query: + if stored_query: templates.insert( 0, - f"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html", + f"query-{to_css_class(database)}-{to_css_class(stored_query['name'])}.html", ) environment = datasette.get_jinja_environment(request) @@ -1932,8 +1932,8 @@ class QueryView(View): } ) metadata = await datasette.get_database_metadata(database) - if canned_query: - metadata = dict(canned_query) + if stored_query: + metadata = dict(stored_query) metadata.pop("source", None) renderers = {} @@ -1968,7 +1968,7 @@ class QueryView(View): ) show_hide_hidden = "" - if canned_query and canned_query.get("hide_sql"): + if stored_query and stored_query.get("hide_sql"): if bool(params.get("_show_sql")): show_hide_link = path_with_removed_args(request, {"_show_sql"}) show_hide_text = "hide" @@ -2018,7 +2018,7 @@ class QueryView(View): ) save_query_url = None if ( - not canned_query + not stored_query and allow_execute_sql and allow_store_query and is_validated_sql @@ -2036,7 +2036,7 @@ class QueryView(View): datasette=datasette, actor=request.actor, database=database, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query["name"] if stored_query else None, request=request, sql=sql, params=params, @@ -2056,15 +2056,15 @@ class QueryView(View): "sql": sql, "params": params, }, - canned_query=canned_query["name"] if canned_query else None, + stored_query=stored_query["name"] if stored_query else None, private=private, - canned_query_write=canned_query_write, + stored_query_write=stored_query_write, db_is_immutable=not db.is_mutable, error=query_error, hide_sql=hide_sql, show_hide_link=datasette.urls.path(show_hide_link), show_hide_text=show_hide_text, - editable=not canned_query, + editable=not stored_query, allow_execute_sql=allow_execute_sql, save_query_url=save_query_url, tables=await get_tables(datasette, request, db, allowed_dict), @@ -2100,7 +2100,7 @@ class QueryView(View): datasette, request, database=database, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query["name"] if stored_query else None, ), query_actions=query_actions, ), diff --git a/datasette/views/table.py b/datasette/views/table.py index 7b1a5a82..da69c6b5 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -964,11 +964,11 @@ async def table_view_traced(datasette, request): resolved = await datasette.resolve_table(request) except TableNotFound as not_found: # Was this actually a stored query? - canned_query = await datasette.get_canned_query( - not_found.database_name, not_found.table, request.actor + stored_query = await datasette.get_query( + not_found.database_name, not_found.table ) # If this is a stored query, not a table, then dispatch to QueryView instead - if canned_query: + if stored_query: return await QueryView()(request, datasette) else: raise diff --git a/docs/authentication.rst b/docs/authentication.rst index 86df7f04..cec47f97 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -121,7 +121,7 @@ This configuration will deny access to everyone except the user with ``id`` of ` How permissions are resolved ---------------------------- -Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. +Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified. @@ -468,7 +468,7 @@ You can control the following: * Access to the entire Datasette instance * Access to specific databases * Access to specific tables and views -* Access to specific :ref:`queries ` +* Access to specific :ref:`queries ` If a user has permission to view a table they will be able to view that table, independent of if they have permission to view the database or instance that the table exists within. @@ -496,7 +496,7 @@ Here's how to restrict access to your entire Datasette instance to just the ``"i title: My private Datasette instance allow: id: root - + .. tab:: datasette.json @@ -644,7 +644,7 @@ This works for SQL views as well - you can list their names in the ``"tables"`` Access to specific queries -------------------------- -:ref:`Queries ` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. +:ref:`Queries ` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. To limit access to the ``add_name`` query in your ``dogs.db`` database to just the :ref:`root user`: @@ -1020,7 +1020,7 @@ You can also restrict permissions such that they can only be used within specifi The resulting token will only be able to insert rows, and only to tables in the ``mydatabase`` database. -Finally, you can restrict permissions to individual resources - tables, SQL views and :ref:`named queries ` - within a specific database:: +Finally, you can restrict permissions to individual resources - tables, SQL views and :ref:`named queries ` - within a specific database:: datasette create-token root --resource mydatabase mytable insert-row diff --git a/docs/changelog.rst b/docs/changelog.rst index 674ff5b3..d15dec50 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,7 +11,8 @@ Unreleased - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) - The ``top_canned_query()`` plugin hook has been renamed to :ref:`top_stored_query() `. (:issue:`2747`) -- The ``canned_queries()`` plugin hook has been removed. Plugins can use the new ``datasette.add_query()``, ``datasette.update_query()`` and ``datasette.remove_query()`` methods to managed stored queries instead. +- The ``canned_queries()`` plugin hook has been removed. Plugins can use the new ``datasette.add_query()``, ``datasette.update_query()`` and ``datasette.remove_query()`` methods to manage stored queries instead. +- The ``datasette.get_canned_query()`` and ``datasette.get_canned_queries()`` methods have been removed. Plugins can use ``datasette.get_query()`` and ``datasette.list_queries()`` instead. .. _v1_0_a30: @@ -658,7 +659,7 @@ For more information and workarounds, read `the security advisory `` in a ``
`` element, to help with advanced CSS customization. (:issue:`1446`) - The :ref:`render_cell() ` plugin hook can now return an awaitable function. This means the hook can execute SQL queries. (:issue:`1425`) - :ref:`plugin_register_routes` plugin hook now accepts an optional ``datasette`` argument. (:issue:`1404`) -- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`) +- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`queries_options`. (:issue:`1422`) - New ``--cpu`` option for :ref:`datasette publish cloudrun `. (:issue:`1420`) - If `Rich `__ is installed in the same virtual environment as Datasette, it will be used to provide enhanced display of error tracebacks on the console. (:issue:`1416`) - ``datasette.utils`` :ref:`internals_utils_parse_metadata` function, used by the new `datasette-remote-metadata plugin `__, is now a documented API. (:issue:`1405`) @@ -1426,7 +1427,7 @@ See also `Datasette 0.50: The annotated release notes `__. -- Writable canned queries now expose a JSON API, see :ref:`canned_queries_json_api`. (:issue:`880`) +- Writable canned queries now expose a JSON API, see :ref:`queries_json_api`. (:issue:`880`) - New mechanism for defining page templates with custom path parameters - a template file called ``pages/about/{slug}.html`` will be used to render any requests to ``/about/something``. See :ref:`custom_pages_parameters`. (:issue:`944`) - ``register_output_renderer()`` render functions can now return a ``Response``. (:issue:`953`) - New ``--upgrade`` option for ``datasette install``. (:issue:`945`) @@ -1518,7 +1519,7 @@ Magic parameters for canned queries, a log out feature, improved plugin document Magic parameters for canned queries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Canned queries now support :ref:`canned_queries_magic_parameters`, which can be used to insert or select automatically generated values. For example:: +Canned queries now support :ref:`queries_magic_parameters`, which can be used to insert or select automatically generated values. For example:: insert into logs (user_id, timestamp) @@ -1549,7 +1550,7 @@ New plugin hooks - :ref:`plugin_hook_register_magic_parameters` can be used to define new types of magic canned query parameters. - :ref:`plugin_hook_startup` can run custom code when Datasette first starts up. `datasette-init `__ is a new plugin that uses this hook to create database tables and views on startup if they have not yet been created. (:issue:`834`) -- :ref:`plugin_hook_canned_queries` lets plugins provide additional canned queries beyond those defined in Datasette's metadata. See `datasette-saved-queries `__ for an example of this hook in action. (:issue:`852`) +- ``canned_queries()`` lets plugins provide additional canned queries beyond those defined in Datasette's metadata. See `datasette-saved-queries `__ for an example of this hook in action. (:issue:`852`) - :ref:`plugin_hook_forbidden` is a hook for customizing how Datasette responds to 403 forbidden errors. (:issue:`812`) Smaller changes @@ -1624,7 +1625,7 @@ A new debug page at ``/-/permissions`` shows recent permission checks, to help a Writable canned queries ~~~~~~~~~~~~~~~~~~~~~~~ -Datasette's :ref:`canned_queries` feature lets you define SQL queries in ``metadata.json`` which can then be executed by users visiting a specific URL. https://latest.datasette.io/fixtures/neighborhood_search for example. +Datasette's :ref:`queries` feature lets you define SQL queries in ``metadata.json`` which can then be executed by users visiting a specific URL. https://latest.datasette.io/fixtures/neighborhood_search for example. Canned queries were previously restricted to ``SELECT``, but Datasette 0.44 introduces the ability for canned queries to execute ``INSERT`` or ``UPDATE`` queries as well, using the new ``"write": true`` property (:issue:`800`): @@ -1643,7 +1644,7 @@ Canned queries were previously restricted to ``SELECT``, but Datasette 0.44 intr } } -See :ref:`canned_queries_writable` for more details. +See :ref:`queries_writable` for more details. Flash messages ~~~~~~~~~~~~~~ @@ -1698,7 +1699,7 @@ Smaller changes - New ``request.cookies`` property. - ``/-/plugins`` endpoint now shows a list of hooks implemented by each plugin, e.g. https://latest.datasette.io/-/plugins?all=1 - ``request.post_vars()`` method no longer discards empty values. -- New "params" canned query key for explicitly setting named parameters, see :ref:`canned_queries_named_parameters`. (:issue:`797`) +- New "params" canned query key for explicitly setting named parameters, see :ref:`queries_named_parameters`. (:issue:`797`) - ``request.args`` is now a :ref:`MultiParams ` object. - Fixed a bug with the ``datasette plugins`` command. (:issue:`802`) - Nicer pattern for using ``make_app_client()`` in tests. (:issue:`395`) @@ -1732,7 +1733,7 @@ The main focus of this release is a major upgrade to the :ref:`plugin_register_o * Visually distinguish float and integer columns - useful for figuring out why order-by-column might be returning unexpected results. (:issue:`729`) * The :ref:`internals_request`, which is passed to several plugin hooks, is now documented. (:issue:`706`) * New ``metadata.json`` option for setting a custom default page size for specific tables and views, see :ref:`table_configuration_size`. (:issue:`751`) -* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega `__, see :ref:`canned_queries_options`. (:issue:`706`) +* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega `__, see :ref:`queries_options`. (:issue:`706`) * Fixed a bug in ``datasette publish`` when running on operating systems where the ``/tmp`` directory lives in a different volume, using a backport of the Python 3.8 ``shutil.copytree()`` function. (:issue:`744`) * Every plugin hook is now covered by the unit tests, and a new unit test checks that each plugin hook has at least one corresponding test. (:issue:`771`, :issue:`773`) @@ -2249,7 +2250,7 @@ A number of small new features: - Documentation for :ref:`datasette publish and datasette package `, closes `#337 `_ - Fixed compatibility with Python 3.7 - ``datasette publish heroku`` now supports app names via the ``-n`` option, which can also be used to overwrite an existing application [Russ Garrett] -- Title and description metadata can now be set for :ref:`canned SQL queries `, closes `#342 `_ +- Title and description metadata can now be set for :ref:`canned SQL queries `, closes `#342 `_ - New ``force_https_on`` config option, fixes ``https://`` API URLs when deploying to Zeit Now - closes `#333 `_ - ``?_json_infinity=1`` query string argument for handling Infinity/-Infinity values in JSON, closes `#332 `_ - URLs displayed in the results of custom SQL queries are now URLified, closes `#298 `_ diff --git a/docs/configuration.rst b/docs/configuration.rst index cf9590b8..9213a640 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -434,12 +434,12 @@ Here is a simple example: :ref:`authentication_permissions_config` has the full details. -.. _configuration_reference_canned_queries: +.. _configuration_reference_queries: Queries configuration ~~~~~~~~~~~~~~~~~~~~~ -:ref:`Queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: +:ref:`Queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: .. [[[cog from metadata_doc import config_example, config_example @@ -484,7 +484,7 @@ Queries configuration } .. [[[end]]] -See the :ref:`queries documentation ` for more, including how to configure :ref:`writable queries `. +See the :ref:`queries documentation ` for more, including how to configure :ref:`writable queries `. .. _configuration_reference_css_js: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4737ca03..2a0ddc93 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1207,16 +1207,6 @@ Potential use-cases: Examples: `datasette-saved-queries `__, `datasette-init `__ -.. _plugin_hook_canned_queries: - -canned_queries(datasette, database, actor) ------------------------------------------- - -This hook has been removed. Plugins that need to add stored queries should use -the :ref:`plugin_hook_startup` hook and call ``await datasette.add_query(...)``. - -Example: `datasette-saved-queries `__ - .. _plugin_hook_actor_from_request: actor_from_request(datasette, request) @@ -1635,7 +1625,7 @@ register_magic_parameters(datasette) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. -:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`configured queries `. This plugin hook allows additional magic parameters to be defined by plugins. +:ref:`queries_magic_parameters` can be used to add automatic parameters to :ref:`configured queries `. This plugin hook allows additional magic parameters to be defined by plugins. Magic parameters all take this format: ``_prefix_rest_of_parameter``. The prefix indicates which magic parameter function should be called - the rest of the parameter is passed as an argument to that function. diff --git a/docs/spatialite.rst b/docs/spatialite.rst index 1999ab78..bea679aa 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -30,7 +30,7 @@ Warning The following steps are recommended: - Disable arbitrary SQL queries by untrusted users. See :ref:`authentication_permissions_execute_sql` for ways to do this. The easiest is to start Datasette with the ``datasette --setting default_allow_sql off`` option. - - Define :ref:`queries ` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. + - Define :ref:`queries ` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. The `Datasette SpatiaLite tutorial `__ includes detailed instructions for running SpatiaLite safely using these techniques diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index d60656e3..f593a534 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -66,7 +66,7 @@ You can also use the `sqlite-utils `__ tool sqlite-utils create-view sf-trees.db demo_view "select qSpecies from Street_Tree_List" -.. _canned_queries: +.. _queries: Queries ------- @@ -173,7 +173,7 @@ You can opt out of this behavior for a configured query using ``is_trusted: fals sql: select * from report is_trusted: false -.. _canned_queries_named_parameters: +.. _queries_named_parameters: Query parameters ~~~~~~~~~~~~~~~~ @@ -313,7 +313,7 @@ You can alternatively provide an explicit list of named parameters using the ``" } .. [[[end]]] -.. _canned_queries_options: +.. _queries_options: Additional query options ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -389,7 +389,7 @@ This example demonstrates both ``fragment`` and ``hide_sql``: `See here `__ for a demo of this in action. -.. _canned_queries_writable: +.. _queries_writable: Writable queries ~~~~~~~~~~~~~~~~ @@ -524,7 +524,7 @@ You can pre-populate form fields when the page first loads using a query string, If you specify a query in ``"on_success_message_sql"``, that query will be executed after the main query. The first column of the first row return by that query will be displayed as a success message. Named parameters from the main query will be made available to the success message query as well. -.. _canned_queries_magic_parameters: +.. _queries_magic_parameters: Magic parameters ~~~~~~~~~~~~~~~~ @@ -621,7 +621,7 @@ The form presented at ``/mydatabase/add_message`` will have just a field for ``m Additional custom magic parameters can be added by plugins using the :ref:`plugin_hook_register_magic_parameters` hook. -.. _canned_queries_json_api: +.. _queries_json_api: JSON API for writable queries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/upgrade-1.0a20.md b/docs/upgrade-1.0a20.md index fbc3f4a8..b27d8bd5 100644 --- a/docs/upgrade-1.0a20.md +++ b/docs/upgrade-1.0a20.md @@ -244,7 +244,7 @@ except (KeyError, TypeError): New code: ```python try: - query_info = await datasette.get_canned_query(database, query_name, request.actor) + query_info = await datasette.get_query(database, query_name) if query_info and "title" in query_info: title = query_info["title"] except (KeyError, TypeError): @@ -253,7 +253,7 @@ except (KeyError, TypeError): ### Update render functions to async -If your plugin's render function needs to call `datasette.get_canned_query()` or other async Datasette methods, it must be declared as async: +If your plugin's render function needs to call `datasette.get_query()` or other async Datasette methods, it must be declared as async: Old code: ```python @@ -268,7 +268,7 @@ New code: async def render_atom(datasette, request, sql, columns, rows, database, table, query_name, view_name, data): # ... if query_name: - query_info = await datasette.get_canned_query(database, query_name, request.actor) + query_info = await datasette.get_query(database, query_name) if query_info and "title" in query_info: title = query_info["title"] ``` diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py deleted file mode 100644 index ae2c74e0..00000000 --- a/tests/test_canned_queries.py +++ /dev/null @@ -1,473 +0,0 @@ -from bs4 import BeautifulSoup as Soup -from asgiref.sync import async_to_sync -import json -import pytest -import re -from .fixtures import make_app_client - - -def update_query(client, name, **kwargs): - async_to_sync(client.ds.invoke_startup)() - async_to_sync(client.ds.update_query)("data", name, **kwargs) - - -@pytest.fixture -def canned_write_client(tmpdir): - template_dir = tmpdir / "canned_write_templates" - template_dir.mkdir() - (template_dir / "query-data-update_name.html").write_text( - """ - {% extends "query.html" %} - {% block content %}!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!{{ super() }}{% endblock %} - """, - "utf-8", - ) - with make_app_client( - extra_databases={"data.db": "create table names (name text)"}, - template_dir=str(template_dir), - config={ - "databases": { - "data": { - "queries": { - "canned_read": {"sql": "select * from names"}, - "add_name": { - "sql": "insert into names (name) values (:name)", - "write": True, - "on_success_redirect": "/data/add_name?success", - }, - "add_name_specify_id": { - "sql": "insert into names (rowid, name) values (:rowid, :name)", - "on_success_message_sql": "select 'Name added: ' || :name || ' with rowid ' || :rowid", - "write": True, - "on_error_redirect": "/data/add_name_specify_id?error", - }, - "add_name_specify_id_with_error_in_on_success_message_sql": { - "sql": "insert into names (rowid, name) values (:rowid, :name)", - "on_success_message_sql": "select this is bad SQL", - "write": True, - }, - "delete_name": { - "sql": "delete from names where rowid = :rowid", - "write": True, - "on_success_message": "Name deleted", - "allow": {"id": "root"}, - }, - "update_name": { - "sql": "update names set name = :name where rowid = :rowid", - "params": ["rowid", "name", "extra"], - "write": True, - }, - } - } - } - }, - ) as client: - yield client - - -@pytest.fixture -def canned_write_immutable_client(): - with make_app_client( - is_immutable=True, - config={ - "databases": { - "fixtures": { - "queries": { - "add": { - "sql": "insert into sortable (text) values (:text)", - "write": True, - }, - } - } - } - }, - ) as client: - yield client - - -@pytest.mark.asyncio -async def test_canned_query_with_named_parameter(ds_client): - response = await ds_client.get( - "/fixtures/neighborhood_search.json?text=town&_shape=arrays" - ) - assert response.json()["rows"] == [ - ["Corktown", "Detroit", "MI"], - ["Downtown", "Los Angeles", "CA"], - ["Downtown", "Detroit", "MI"], - ["Greektown", "Detroit", "MI"], - ["Koreatown", "Los Angeles", "CA"], - ["Mexicantown", "Detroit", "MI"], - ] - - -def test_insert(canned_write_client): - response = canned_write_client.post( - "/data/add_name", - {"name": "Hello"}, - csrftoken_from=True, - cookies={"foo": "bar"}, - ) - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert messages == [["Query executed, 1 row affected", 1]] - assert response.status == 302 - assert response.headers["Location"] == "/data/add_name?success" - - -def test_insert_blocked_cross_site(canned_write_client): - # A cross-site POST (browser-originated) must be blocked - response = canned_write_client.post( - "/data/add_name", - {"name": "Hello"}, - headers={"sec-fetch-site": "cross-site"}, - ) - assert 403 == response.status - - -def test_insert_no_cookies_no_csrf(canned_write_client): - response = canned_write_client.post("/data/add_name", {"name": "Hello"}) - assert 302 == response.status - assert "/data/add_name?success" == response.headers["Location"] - - -def test_custom_success_message(canned_write_client): - response = canned_write_client.post( - "/data/delete_name", - {"rowid": 1}, - cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, - csrftoken_from=True, - ) - assert 302 == response.status - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert [["Name deleted", 1]] == messages - - -def test_insert_error(canned_write_client): - canned_write_client.post("/data/add_name", {"name": "Hello"}, csrftoken_from=True) - response = canned_write_client.post( - "/data/add_name_specify_id", - {"rowid": 1, "name": "Should fail"}, - csrftoken_from=True, - ) - assert 302 == response.status - assert "/data/add_name_specify_id?error" == response.headers["Location"] - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert [["UNIQUE constraint failed: names.rowid", 3]] == messages - # How about with a custom error message? - update_query(canned_write_client, "add_name_specify_id", on_error_message="ERROR") - response = canned_write_client.post( - "/data/add_name_specify_id", - {"rowid": 1, "name": "Should fail"}, - csrftoken_from=True, - ) - assert [["ERROR", 3]] == canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - - -def test_on_success_message_sql(canned_write_client): - response = canned_write_client.post( - "/data/add_name_specify_id", - {"rowid": 5, "name": "Should be OK"}, - csrftoken_from=True, - ) - assert response.status == 302 - assert response.headers["Location"] == "/data/add_name_specify_id" - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert messages == [["Name added: Should be OK with rowid 5", 1]] - - -def test_error_in_on_success_message_sql(canned_write_client): - response = canned_write_client.post( - "/data/add_name_specify_id_with_error_in_on_success_message_sql", - {"rowid": 1, "name": "Should fail"}, - csrftoken_from=True, - ) - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert messages == [ - ["Error running on_success_message_sql: no such column: bad", 3] - ] - - -def test_custom_params(canned_write_client): - response = canned_write_client.get("/data/update_name?extra=foo") - assert ( - '' - in response.text - ) - - -def test_canned_query_pages_no_vary_header(canned_write_client): - # These pages no longer embed per-cookie CSRF tokens, so they must not - # set Vary: Cookie - they should be cacheable across users. - assert "vary" not in canned_write_client.get("/data").headers - assert "vary" not in canned_write_client.get("/data/update_name").headers - - -def test_json_post_body(canned_write_client): - response = canned_write_client.post( - "/data/add_name", - body=json.dumps({"name": ["Hello", "there"]}), - ) - assert 302 == response.status - assert "/data/add_name?success" == response.headers["Location"] - rows = canned_write_client.get("/data/names.json?_shape=array").json - assert rows == [{"rowid": 1, "name": "['Hello', 'there']"}] - - -@pytest.mark.parametrize( - "headers,body,querystring", - ( - (None, "name=NameGoesHere", "?_json=1"), - ({"Accept": "application/json"}, "name=NameGoesHere", None), - (None, "name=NameGoesHere&_json=1", None), - (None, '{"name": "NameGoesHere", "_json": 1}', None), - ), -) -def test_json_response(canned_write_client, headers, body, querystring): - response = canned_write_client.post( - "/data/add_name" + (querystring or ""), - body=body, - headers=headers, - ) - assert 200 == response.status - assert response.headers["content-type"] == "application/json; charset=utf-8" - assert response.json == { - "ok": True, - "message": "Query executed, 1 row affected", - "redirect": "/data/add_name?success", - } - rows = canned_write_client.get("/data/names.json?_shape=array").json - assert rows == [{"rowid": 1, "name": "NameGoesHere"}] - - -def test_canned_query_permissions_on_database_page(canned_write_client): - # Without auth shows the five public queries - anon_response = canned_write_client.get("/data.json") - query_names = {q["name"] for q in anon_response.json["queries"]} - assert query_names == { - "add_name_specify_id_with_error_in_on_success_message_sql", - "update_name", - "add_name_specify_id", - "canned_read", - "add_name", - } - assert anon_response.json["queries_more"] is False - - # With auth the database page preview shows the first five queries - response = canned_write_client.get( - "/data.json", - cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, - ) - assert response.status == 200 - query_names_and_private = sorted( - [ - {"name": q["name"], "private": q["private"]} - for q in response.json["queries"] - ], - key=lambda q: q["name"], - ) - assert query_names_and_private == [ - {"name": "add_name", "private": False}, - {"name": "add_name_specify_id", "private": False}, - { - "name": "add_name_specify_id_with_error_in_on_success_message_sql", - "private": False, - }, - {"name": "canned_read", "private": False}, - {"name": "delete_name", "private": True}, - ] - assert response.json["queries_more"] is True - - # The full query list endpoint includes the remaining query - response = canned_write_client.get( - "/data/-/queries.json?_size=10", - cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, - ) - assert response.status == 200 - query_names_and_private = sorted( - [ - {"name": q["name"], "private": q["private"]} - for q in response.json["queries"] - ], - key=lambda q: q["name"], - ) - assert query_names_and_private == [ - {"name": "add_name", "private": False}, - {"name": "add_name_specify_id", "private": False}, - { - "name": "add_name_specify_id_with_error_in_on_success_message_sql", - "private": False, - }, - {"name": "canned_read", "private": False}, - {"name": "delete_name", "private": True}, - {"name": "update_name", "private": False}, - ] - - -def test_canned_query_permissions(canned_write_client): - assert 403 == canned_write_client.get("/data/delete_name").status - assert 200 == canned_write_client.get("/data/update_name").status - cookies = {"ds_actor": canned_write_client.actor_cookie({"id": "root"})} - assert 200 == canned_write_client.get("/data/delete_name", cookies=cookies).status - assert 200 == canned_write_client.get("/data/update_name", cookies=cookies).status - - -@pytest.fixture(scope="session") -def magic_parameters_client(): - with make_app_client( - extra_databases={"data.db": "create table logs (line text)"}, - config={ - "databases": { - "data": { - "queries": { - "runme_post": {"sql": "", "write": True}, - "runme_get": {"sql": ""}, - } - } - } - }, - ) as client: - yield client - - -@pytest.mark.parametrize( - "magic_parameter,expected_re", - [ - ("_actor_id", "root"), - ("_header_host", "localhost"), - ("_header_not_a_thing", ""), - ("_cookie_foo", "bar"), - ("_now_epoch", r"^\d+$"), - ("_now_date_utc", r"^\d{4}-\d{2}-\d{2}$"), - ("_now_datetime_utc", r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$"), - ("_random_chars_1", r"^\w$"), - ("_random_chars_10", r"^\w{10}$"), - ], -) -def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re): - update_query( - magic_parameters_client, - "runme_post", - sql=f"insert into logs (line) values (:{magic_parameter})", - ) - update_query( - magic_parameters_client, - "runme_get", - sql=f"select :{magic_parameter} as result", - ) - cookies = { - "ds_actor": magic_parameters_client.actor_cookie({"id": "root"}), - "foo": "bar", - } - # Test the GET version - get_response = magic_parameters_client.get( - "/data/runme_get.json?_shape=array", cookies=cookies - ) - get_actual = get_response.json[0]["result"] - assert re.match(expected_re, str(get_actual)) - # Test the form - form_response = magic_parameters_client.get("/data/runme_post") - soup = Soup(form_response.body, "html.parser") - # The magic parameter should not be represented as a form field - assert None is soup.find("input", {"name": magic_parameter}) - # Submit the form to create a log line - response = magic_parameters_client.post( - "/data/runme_post?_json=1", {}, csrftoken_from=True, cookies=cookies - ) - assert response.json == { - "ok": True, - "message": "Query executed, 1 row affected", - "redirect": None, - } - post_actual = magic_parameters_client.get( - "/data/logs.json?_sort_desc=rowid&_shape=array" - ).json[0]["line"] - assert re.match(expected_re, post_actual) - - -@pytest.mark.parametrize("use_csrf", [True, False]) -@pytest.mark.parametrize("return_json", [True, False]) -def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_json): - update_query( - magic_parameters_client, - "runme_post", - sql="insert into logs (line) values (:_header_host)", - ) - qs = "" - if return_json: - qs = "?_json=1" - response = magic_parameters_client.post( - f"/data/runme_post{qs}", - {}, - csrftoken_from=use_csrf or None, - ) - if return_json: - assert response.status == 200 - assert response.json["ok"], response.json - else: - assert response.status == 302 - messages = magic_parameters_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert [["Query executed, 1 row affected", 1]] == messages - post_actual = magic_parameters_client.get( - "/data/logs.json?_sort_desc=rowid&_shape=array" - ).json[0]["line"] - assert post_actual == "localhost" - - -def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client): - response = magic_parameters_client.get( - "/data/-/query.json?sql=select+:_header_host&_shape=array" - ) - assert 400 == response.status - assert response.json["error"].startswith("You did not supply a value for binding") - - -def test_canned_write_custom_template(canned_write_client): - response = canned_write_client.get("/data/update_name") - assert response.status == 200 - assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text - assert ( - "" - in response.text - ) - # And test for link rel=alternate while we're here: - assert ( - '' - in response.text - ) - assert ( - response.headers["link"] - == '; rel="alternate"; type="application/json+datasette"' - ) - - -def test_canned_write_query_disabled_for_immutable_database( - canned_write_immutable_client, -): - response = canned_write_immutable_client.get("/fixtures/add") - assert response.status == 200 - assert ( - "This query cannot be executed because the database is immutable." - in response.text - ) - assert '' in response.text - # Submitting form should get a forbidden error - response = canned_write_immutable_client.post( - "/fixtures/add", - {"text": "text"}, - csrftoken_from=True, - ) - assert response.status == 403 - assert "Database is immutable" in response.text diff --git a/tests/test_html.py b/tests/test_html.py index 8edb9f6e..a9de5e79 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -633,7 +633,7 @@ async def test_404_content_type(ds_client): @pytest.mark.asyncio -async def test_canned_query_default_title(ds_client): +async def test_stored_query_default_title(ds_client): response = await ds_client.get("/fixtures/magic_parameters") assert response.status_code == 200 soup = Soup(response.content, "html.parser") @@ -641,7 +641,7 @@ async def test_canned_query_default_title(ds_client): @pytest.mark.asyncio -async def test_canned_query_with_custom_metadata(ds_client): +async def test_stored_query_with_custom_metadata(ds_client): response = await ds_client.get("/fixtures/neighborhood_search?text=town") assert response.status_code == 200 soup = Soup(response.content, "html.parser") @@ -700,7 +700,7 @@ async def test_show_hide_sql_query(ds_client): @pytest.mark.asyncio -async def test_canned_query_with_hide_has_no_hidden_sql(ds_client): +async def test_stored_query_with_hide_has_no_hidden_sql(ds_client): # For a stored query the show/hide should NOT have a hidden SQL field # https://github.com/simonw/datasette/issues/1411 response = await ds_client.get("/fixtures/pragma_cache_size?_hide_sql=1") @@ -720,7 +720,7 @@ async def test_canned_query_with_hide_has_no_hidden_sql(ds_client): (True, "?_show_sql=1", "_show_sql", "/_memory/one", "hide"), ), ) -def test_canned_query_show_hide_metadata_option( +def test_stored_query_show_hide_metadata_option( hide_sql, querystring, expected_hidden, @@ -981,10 +981,10 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix): ("/fixtures/magic_parameters", None), ], ) -async def test_edit_sql_link_on_canned_queries(ds_client, path, expected): +async def test_edit_sql_link_on_stored_queries(ds_client, path, expected): response = await ds_client.get(path) assert response.status_code == 200 - expected_link = f'Edit SQL' + expected_link = f'Edit SQL' if expected: assert expected_link in response.text else: diff --git a/tests/test_jump.py b/tests/test_jump.py index f60af0fd..513a809f 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -76,7 +76,7 @@ async def ds_for_jump(): @pytest.mark.asyncio -async def test_jump_searches_tables_databases_views_and_canned_queries(ds_for_jump): +async def test_jump_searches_tables_databases_views_and_stored_queries(ds_for_jump): response = await ds_for_jump.client.get( "/-/jump.json?q=content", actor={"id": "user"} ) @@ -98,7 +98,7 @@ async def test_jump_searches_tables_databases_views_and_canned_queries(ds_for_ju @pytest.mark.asyncio -async def test_jump_uses_canned_query_names_not_titles(ds_for_jump): +async def test_jump_uses_stored_query_names_not_titles(ds_for_jump): response = await ds_for_jump.client.get( "/-/jump.json?q=datasette", actor={"id": "user"} ) From cafb6b9dbd77e438b81a7784c4434957b73fb856 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:20:29 -0700 Subject: [PATCH 725/767] Need is_trusted=True for the counters demo --- .github/workflows/deploy-latest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 166d33d0..b0640ae8 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -76,6 +76,7 @@ jobs: "update counters set value = value + 1 where name = '{}'".format(name), on_success_message_sql="select 'Counter {name} incremented to ' || value from counters where name = '{name}'".format(name=name), is_write=True, + is_trusted=True, ) await datasette.add_query( "counters", @@ -83,6 +84,7 @@ jobs: "update counters set value = value - 1 where name = '{}'".format(name), on_success_message_sql="select 'Counter {name} decremented to ' || value from counters where name = '{name}'".format(name=name), is_write=True, + is_trusted=True, ) return inner EOF From e2864fc895603c1fdf03d3c55812a0a6e56779ff Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:21:09 -0700 Subject: [PATCH 726/767] test_stored_queries.py --- tests/test_stored_queries.py | 473 +++++++++++++++++++++++++++++++++++ 1 file changed, 473 insertions(+) create mode 100644 tests/test_stored_queries.py diff --git a/tests/test_stored_queries.py b/tests/test_stored_queries.py new file mode 100644 index 00000000..2c648d5f --- /dev/null +++ b/tests/test_stored_queries.py @@ -0,0 +1,473 @@ +from bs4 import BeautifulSoup as Soup +from asgiref.sync import async_to_sync +import json +import pytest +import re +from .fixtures import make_app_client + + +def update_query(client, name, **kwargs): + async_to_sync(client.ds.invoke_startup)() + async_to_sync(client.ds.update_query)("data", name, **kwargs) + + +@pytest.fixture +def stored_write_client(tmpdir): + template_dir = tmpdir / "stored_write_templates" + template_dir.mkdir() + (template_dir / "query-data-update_name.html").write_text( + """ + {% extends "query.html" %} + {% block content %}!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!{{ super() }}{% endblock %} + """, + "utf-8", + ) + with make_app_client( + extra_databases={"data.db": "create table names (name text)"}, + template_dir=str(template_dir), + config={ + "databases": { + "data": { + "queries": { + "stored_read": {"sql": "select * from names"}, + "add_name": { + "sql": "insert into names (name) values (:name)", + "write": True, + "on_success_redirect": "/data/add_name?success", + }, + "add_name_specify_id": { + "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select 'Name added: ' || :name || ' with rowid ' || :rowid", + "write": True, + "on_error_redirect": "/data/add_name_specify_id?error", + }, + "add_name_specify_id_with_error_in_on_success_message_sql": { + "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select this is bad SQL", + "write": True, + }, + "delete_name": { + "sql": "delete from names where rowid = :rowid", + "write": True, + "on_success_message": "Name deleted", + "allow": {"id": "root"}, + }, + "update_name": { + "sql": "update names set name = :name where rowid = :rowid", + "params": ["rowid", "name", "extra"], + "write": True, + }, + } + } + } + }, + ) as client: + yield client + + +@pytest.fixture +def stored_write_immutable_client(): + with make_app_client( + is_immutable=True, + config={ + "databases": { + "fixtures": { + "queries": { + "add": { + "sql": "insert into sortable (text) values (:text)", + "write": True, + }, + } + } + } + }, + ) as client: + yield client + + +@pytest.mark.asyncio +async def test_stored_query_with_named_parameter(ds_client): + response = await ds_client.get( + "/fixtures/neighborhood_search.json?text=town&_shape=arrays" + ) + assert response.json()["rows"] == [ + ["Corktown", "Detroit", "MI"], + ["Downtown", "Los Angeles", "CA"], + ["Downtown", "Detroit", "MI"], + ["Greektown", "Detroit", "MI"], + ["Koreatown", "Los Angeles", "CA"], + ["Mexicantown", "Detroit", "MI"], + ] + + +def test_insert(stored_write_client): + response = stored_write_client.post( + "/data/add_name", + {"name": "Hello"}, + csrftoken_from=True, + cookies={"foo": "bar"}, + ) + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [["Query executed, 1 row affected", 1]] + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name?success" + + +def test_insert_blocked_cross_site(stored_write_client): + # A cross-site POST (browser-originated) must be blocked + response = stored_write_client.post( + "/data/add_name", + {"name": "Hello"}, + headers={"sec-fetch-site": "cross-site"}, + ) + assert 403 == response.status + + +def test_insert_no_cookies_no_csrf(stored_write_client): + response = stored_write_client.post("/data/add_name", {"name": "Hello"}) + assert 302 == response.status + assert "/data/add_name?success" == response.headers["Location"] + + +def test_custom_success_message(stored_write_client): + response = stored_write_client.post( + "/data/delete_name", + {"rowid": 1}, + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, + csrftoken_from=True, + ) + assert 302 == response.status + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["Name deleted", 1]] == messages + + +def test_insert_error(stored_write_client): + stored_write_client.post("/data/add_name", {"name": "Hello"}, csrftoken_from=True) + response = stored_write_client.post( + "/data/add_name_specify_id", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + assert 302 == response.status + assert "/data/add_name_specify_id?error" == response.headers["Location"] + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["UNIQUE constraint failed: names.rowid", 3]] == messages + # How about with a custom error message? + update_query(stored_write_client, "add_name_specify_id", on_error_message="ERROR") + response = stored_write_client.post( + "/data/add_name_specify_id", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + assert [["ERROR", 3]] == stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + + +def test_on_success_message_sql(stored_write_client): + response = stored_write_client.post( + "/data/add_name_specify_id", + {"rowid": 5, "name": "Should be OK"}, + csrftoken_from=True, + ) + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name_specify_id" + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [["Name added: Should be OK with rowid 5", 1]] + + +def test_error_in_on_success_message_sql(stored_write_client): + response = stored_write_client.post( + "/data/add_name_specify_id_with_error_in_on_success_message_sql", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [ + ["Error running on_success_message_sql: no such column: bad", 3] + ] + + +def test_custom_params(stored_write_client): + response = stored_write_client.get("/data/update_name?extra=foo") + assert ( + '' + in response.text + ) + + +def test_stored_query_pages_no_vary_header(stored_write_client): + # These pages no longer embed per-cookie CSRF tokens, so they must not + # set Vary: Cookie - they should be cacheable across users. + assert "vary" not in stored_write_client.get("/data").headers + assert "vary" not in stored_write_client.get("/data/update_name").headers + + +def test_json_post_body(stored_write_client): + response = stored_write_client.post( + "/data/add_name", + body=json.dumps({"name": ["Hello", "there"]}), + ) + assert 302 == response.status + assert "/data/add_name?success" == response.headers["Location"] + rows = stored_write_client.get("/data/names.json?_shape=array").json + assert rows == [{"rowid": 1, "name": "['Hello', 'there']"}] + + +@pytest.mark.parametrize( + "headers,body,querystring", + ( + (None, "name=NameGoesHere", "?_json=1"), + ({"Accept": "application/json"}, "name=NameGoesHere", None), + (None, "name=NameGoesHere&_json=1", None), + (None, '{"name": "NameGoesHere", "_json": 1}', None), + ), +) +def test_json_response(stored_write_client, headers, body, querystring): + response = stored_write_client.post( + "/data/add_name" + (querystring or ""), + body=body, + headers=headers, + ) + assert 200 == response.status + assert response.headers["content-type"] == "application/json; charset=utf-8" + assert response.json == { + "ok": True, + "message": "Query executed, 1 row affected", + "redirect": "/data/add_name?success", + } + rows = stored_write_client.get("/data/names.json?_shape=array").json + assert rows == [{"rowid": 1, "name": "NameGoesHere"}] + + +def test_stored_query_permissions_on_database_page(stored_write_client): + # Without auth shows the five public queries + anon_response = stored_write_client.get("/data.json") + query_names = {q["name"] for q in anon_response.json["queries"]} + assert query_names == { + "add_name_specify_id_with_error_in_on_success_message_sql", + "update_name", + "add_name_specify_id", + "stored_read", + "add_name", + } + assert anon_response.json["queries_more"] is False + + # With auth the database page preview shows the first five queries + response = stored_write_client.get( + "/data.json", + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, + ) + assert response.status == 200 + query_names_and_private = sorted( + [ + {"name": q["name"], "private": q["private"]} + for q in response.json["queries"] + ], + key=lambda q: q["name"], + ) + assert query_names_and_private == [ + {"name": "add_name", "private": False}, + {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, + {"name": "delete_name", "private": True}, + {"name": "stored_read", "private": False}, + ] + assert response.json["queries_more"] is True + + # The full query list endpoint includes the remaining query + response = stored_write_client.get( + "/data/-/queries.json?_size=10", + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, + ) + assert response.status == 200 + query_names_and_private = sorted( + [ + {"name": q["name"], "private": q["private"]} + for q in response.json["queries"] + ], + key=lambda q: q["name"], + ) + assert query_names_and_private == [ + {"name": "add_name", "private": False}, + {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, + {"name": "delete_name", "private": True}, + {"name": "stored_read", "private": False}, + {"name": "update_name", "private": False}, + ] + + +def test_stored_query_permissions(stored_write_client): + assert 403 == stored_write_client.get("/data/delete_name").status + assert 200 == stored_write_client.get("/data/update_name").status + cookies = {"ds_actor": stored_write_client.actor_cookie({"id": "root"})} + assert 200 == stored_write_client.get("/data/delete_name", cookies=cookies).status + assert 200 == stored_write_client.get("/data/update_name", cookies=cookies).status + + +@pytest.fixture(scope="session") +def magic_parameters_client(): + with make_app_client( + extra_databases={"data.db": "create table logs (line text)"}, + config={ + "databases": { + "data": { + "queries": { + "runme_post": {"sql": "", "write": True}, + "runme_get": {"sql": ""}, + } + } + } + }, + ) as client: + yield client + + +@pytest.mark.parametrize( + "magic_parameter,expected_re", + [ + ("_actor_id", "root"), + ("_header_host", "localhost"), + ("_header_not_a_thing", ""), + ("_cookie_foo", "bar"), + ("_now_epoch", r"^\d+$"), + ("_now_date_utc", r"^\d{4}-\d{2}-\d{2}$"), + ("_now_datetime_utc", r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$"), + ("_random_chars_1", r"^\w$"), + ("_random_chars_10", r"^\w{10}$"), + ], +) +def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re): + update_query( + magic_parameters_client, + "runme_post", + sql=f"insert into logs (line) values (:{magic_parameter})", + ) + update_query( + magic_parameters_client, + "runme_get", + sql=f"select :{magic_parameter} as result", + ) + cookies = { + "ds_actor": magic_parameters_client.actor_cookie({"id": "root"}), + "foo": "bar", + } + # Test the GET version + get_response = magic_parameters_client.get( + "/data/runme_get.json?_shape=array", cookies=cookies + ) + get_actual = get_response.json[0]["result"] + assert re.match(expected_re, str(get_actual)) + # Test the form + form_response = magic_parameters_client.get("/data/runme_post") + soup = Soup(form_response.body, "html.parser") + # The magic parameter should not be represented as a form field + assert None is soup.find("input", {"name": magic_parameter}) + # Submit the form to create a log line + response = magic_parameters_client.post( + "/data/runme_post?_json=1", {}, csrftoken_from=True, cookies=cookies + ) + assert response.json == { + "ok": True, + "message": "Query executed, 1 row affected", + "redirect": None, + } + post_actual = magic_parameters_client.get( + "/data/logs.json?_sort_desc=rowid&_shape=array" + ).json[0]["line"] + assert re.match(expected_re, post_actual) + + +@pytest.mark.parametrize("use_csrf", [True, False]) +@pytest.mark.parametrize("return_json", [True, False]) +def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_json): + update_query( + magic_parameters_client, + "runme_post", + sql="insert into logs (line) values (:_header_host)", + ) + qs = "" + if return_json: + qs = "?_json=1" + response = magic_parameters_client.post( + f"/data/runme_post{qs}", + {}, + csrftoken_from=use_csrf or None, + ) + if return_json: + assert response.status == 200 + assert response.json["ok"], response.json + else: + assert response.status == 302 + messages = magic_parameters_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["Query executed, 1 row affected", 1]] == messages + post_actual = magic_parameters_client.get( + "/data/logs.json?_sort_desc=rowid&_shape=array" + ).json[0]["line"] + assert post_actual == "localhost" + + +def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client): + response = magic_parameters_client.get( + "/data/-/query.json?sql=select+:_header_host&_shape=array" + ) + assert 400 == response.status + assert response.json["error"].startswith("You did not supply a value for binding") + + +def test_stored_write_custom_template(stored_write_client): + response = stored_write_client.get("/data/update_name") + assert response.status == 200 + assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text + assert ( + "" + in response.text + ) + # And test for link rel=alternate while we're here: + assert ( + '' + in response.text + ) + assert ( + response.headers["link"] + == '; rel="alternate"; type="application/json+datasette"' + ) + + +def test_stored_write_query_disabled_for_immutable_database( + stored_write_immutable_client, +): + response = stored_write_immutable_client.get("/fixtures/add") + assert response.status == 200 + assert ( + "This query cannot be executed because the database is immutable." + in response.text + ) + assert '' in response.text + # Submitting form should get a forbidden error + response = stored_write_immutable_client.post( + "/fixtures/add", + {"text": "text"}, + csrftoken_from=True, + ) + assert response.status == 403 + assert "Database is immutable" in response.text From ca4907ab6b01f5ba5fa241534be7c9557f269050 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:30:36 -0700 Subject: [PATCH 727/767] Make _save_queries_from_config a private method --- datasette/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 52d100cc..6b66a53b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -598,7 +598,7 @@ class Datasette: # TODO(alex) is metadata.json was loaded in, and --internal is not memory, then log # a warning to user that they should delete their metadata.json file - async def apply_queries_config(self): + async def _save_queries_from_config(self): # Apply configured query entries from datasette.yaml to the internal table. await self.get_internal_database().execute_write( "DELETE FROM queries WHERE source = 'config'" @@ -788,7 +788,7 @@ class Datasette: await await_me_maybe(hook) # Ensure internal tables and metadata are populated before startup hooks await self._refresh_schemas() - await self.apply_queries_config() + await self._save_queries_from_config() # Load column_types from config into internal DB await self._apply_column_types_config() for hook in pm.hook.startup(datasette=self): From e89ffa0e0634a306be37bdd4894057329fee54a2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:37:21 -0700 Subject: [PATCH 728/767] Fixed broken test caused by apply_queries_config() rename --- tests/test_permissions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 0e38c876..e5e75432 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -937,7 +937,7 @@ async def test_permissions_in_config( updated_config = copy.deepcopy(previous_config) updated_config.update(config) perms_ds.config = updated_config - await perms_ds.apply_queries_config() + await perms_ds._save_queries_from_config() try: # Convert old-style resource to Resource object from datasette.resources import DatabaseResource, QueryResource, TableResource @@ -964,7 +964,7 @@ async def test_permissions_in_config( assert result == expected_result finally: perms_ds.config = previous_config - await perms_ds.apply_queries_config() + await perms_ds._save_queries_from_config() @pytest.mark.asyncio From 1bcd99df90d09049616fae411c2b6cddb6a82c7a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:42:36 -0700 Subject: [PATCH 729/767] Refactor code from datasette.app into datasette.stored_queries The datasette/app.py file had grown a lot in this branch. --- datasette/app.py | 537 ++++++----------------------------- datasette/stored_queries.py | 544 ++++++++++++++++++++++++++++++++++++ 2 files changed, 634 insertions(+), 447 deletions(-) create mode 100644 datasette/stored_queries.py diff --git a/datasette/app.py b/datasette/app.py index 6b66a53b..dd54446a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -42,6 +42,7 @@ from jinja2.exceptions import TemplateNotFound from .events import Event from .column_types import SQLiteType +from . import stored_queries from .views import Context from .views.database import ( database_download, @@ -108,7 +109,6 @@ from .utils import ( module_from_path, move_plugins_and_allow, move_table_config, - named_parameters, parse_metadata, resolve_env_secrets, resolve_routes, @@ -284,17 +284,6 @@ DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS} FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" DEFAULT_NOT_SET = object() -UNCHANGED = object() - -QUERY_OPTION_FIELDS = ( - "hide_sql", - "fragment", - "on_success_message", - "on_success_message_sql", - "on_success_redirect", - "on_error_message", - "on_error_redirect", -) ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) @@ -599,34 +588,7 @@ class Datasette: # a warning to user that they should delete their metadata.json file async def _save_queries_from_config(self): - # Apply configured query entries from datasette.yaml to the internal table. - await self.get_internal_database().execute_write( - "DELETE FROM queries WHERE source = 'config'" - ) - for dbname, db_config in ((self.config or {}).get("databases") or {}).items(): - for query_name, query_config in (db_config.get("queries") or {}).items(): - if not isinstance(query_config, dict): - query_config = {"sql": query_config} - await self.add_query( - dbname, - query_name, - query_config["sql"], - title=query_config.get("title"), - description=query_config.get("description"), - description_html=query_config.get("description_html"), - hide_sql=bool(query_config.get("hide_sql")), - fragment=query_config.get("fragment"), - parameters=query_config.get("params"), - is_write=bool(query_config.get("write")), - is_private=bool(query_config.get("is_private")), - is_trusted=bool(query_config.get("is_trusted", True)), - source="config", - on_success_message=query_config.get("on_success_message"), - on_success_message_sql=query_config.get("on_success_message_sql"), - on_success_redirect=query_config.get("on_success_redirect"), - on_error_message=query_config.get("on_error_message"), - on_error_redirect=query_config.get("on_error_redirect"), - ) + await stored_queries.save_queries_from_config(self) def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env @@ -1067,46 +1029,11 @@ class Datasette: @staticmethod def _query_row_to_dict(row): - if row is None: - return None - parameters = json.loads(row["parameters"] or "[]") - options = json.loads(row["options"] or "{}") - is_write = bool(row["is_write"]) - return { - "database": row["database_name"], - "name": row["name"], - "sql": row["sql"], - "title": row["title"], - "description": row["description"], - "description_html": row["description_html"], - "hide_sql": bool(options.get("hide_sql")), - "fragment": options.get("fragment"), - "params": parameters, - "parameters": parameters, - "is_write": is_write, - "write": is_write, - "is_private": bool(row["is_private"]), - "is_trusted": bool(row["is_trusted"]), - "source": row["source"], - "owner_id": row["owner_id"], - "on_success_message": options.get("on_success_message"), - "on_success_message_sql": options.get("on_success_message_sql"), - "on_success_redirect": options.get("on_success_redirect"), - "on_error_message": options.get("on_error_message"), - "on_error_redirect": options.get("on_error_redirect"), - } + return stored_queries.query_row_to_dict(row) @staticmethod def _query_options_json(options): - options_dict = {} - for field in QUERY_OPTION_FIELDS: - value = options.get(field) - if field == "hide_sql": - if value: - options_dict[field] = True - elif value is not None: - options_dict[field] = value - return json.dumps(options_dict, sort_keys=True) + return stored_queries.query_options_json(options) async def add_query( self, @@ -1132,57 +1059,28 @@ class Datasette: on_error_redirect=None, replace=True, ): - parameters_json = json.dumps(list(parameters or [])) - options_json = self._query_options_json( - { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - ) - sql_statement = """ - INSERT INTO queries ( - database_name, name, sql, title, description, description_html, - options, parameters, is_write, is_private, is_trusted, source, owner_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """ - if replace: - sql_statement += """ - ON CONFLICT(database_name, name) DO UPDATE SET - sql = excluded.sql, - title = excluded.title, - description = excluded.description, - description_html = excluded.description_html, - options = excluded.options, - parameters = excluded.parameters, - is_write = excluded.is_write, - is_private = excluded.is_private, - is_trusted = excluded.is_trusted, - source = excluded.source, - owner_id = excluded.owner_id, - updated_at = CURRENT_TIMESTAMP - """ - await self.get_internal_database().execute_write( - sql_statement, - [ - database, - name, - sql, - title, - description, - description_html, - options_json, - parameters_json, - int(bool(is_write)), - int(bool(is_private)), - int(bool(is_trusted)), - source, - owner_id, - ], + return await stored_queries.add_query( + self, + database, + name, + sql, + title=title, + description=description, + description_html=description_html, + hide_sql=hide_sql, + fragment=fragment, + parameters=parameters, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + on_success_message=on_success_message, + on_success_message_sql=on_success_message_sql, + on_success_redirect=on_success_redirect, + on_error_message=on_error_message, + on_error_redirect=on_error_redirect, + replace=replace, ) async def update_query( @@ -1190,113 +1088,52 @@ class Datasette: database, name, *, - sql=UNCHANGED, - title=UNCHANGED, - description=UNCHANGED, - description_html=UNCHANGED, - hide_sql=UNCHANGED, - fragment=UNCHANGED, - parameters=UNCHANGED, - is_write=UNCHANGED, - is_private=UNCHANGED, - is_trusted=UNCHANGED, - source=UNCHANGED, - owner_id=UNCHANGED, - on_success_message=UNCHANGED, - on_success_message_sql=UNCHANGED, - on_success_redirect=UNCHANGED, - on_error_message=UNCHANGED, - on_error_redirect=UNCHANGED, + sql=stored_queries.UNCHANGED, + title=stored_queries.UNCHANGED, + description=stored_queries.UNCHANGED, + description_html=stored_queries.UNCHANGED, + hide_sql=stored_queries.UNCHANGED, + fragment=stored_queries.UNCHANGED, + parameters=stored_queries.UNCHANGED, + is_write=stored_queries.UNCHANGED, + is_private=stored_queries.UNCHANGED, + is_trusted=stored_queries.UNCHANGED, + source=stored_queries.UNCHANGED, + owner_id=stored_queries.UNCHANGED, + on_success_message=stored_queries.UNCHANGED, + on_success_message_sql=stored_queries.UNCHANGED, + on_success_redirect=stored_queries.UNCHANGED, + on_error_message=stored_queries.UNCHANGED, + on_error_redirect=stored_queries.UNCHANGED, ): - fields = { - "sql": sql, - "title": title, - "description": description, - "description_html": description_html, - "parameters": parameters, - "is_write": is_write, - "is_private": is_private, - "is_trusted": is_trusted, - "source": source, - "owner_id": owner_id, - } - option_fields = { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - updates = [] - params = [] - for field, value in fields.items(): - if value is UNCHANGED: - continue - if field in {"is_write", "is_private", "is_trusted"}: - value = int(bool(value)) - elif field == "parameters": - value = json.dumps(list(value or [])) - updates.append(f"{field} = ?") - params.append(value) - changed_options = { - field: value - for field, value in option_fields.items() - if value is not UNCHANGED - } - if changed_options: - rows = await self.get_internal_database().execute( - """ - SELECT options FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - row = rows.first() - options = json.loads(row["options"] or "{}") if row is not None else {} - for field, value in changed_options.items(): - if field == "hide_sql": - if value: - options[field] = True - else: - options.pop(field, None) - elif value is None: - options.pop(field, None) - else: - options[field] = value - updates.append("options = ?") - params.append(json.dumps(options, sort_keys=True)) - if not updates: - return - updates.append("updated_at = CURRENT_TIMESTAMP") - params.extend([database, name]) - await self.get_internal_database().execute_write( - """ - UPDATE queries - SET {} - WHERE database_name = ? AND name = ? - """.format(", ".join(updates)), - params, + return await stored_queries.update_query( + self, + database, + name, + sql=sql, + title=title, + description=description, + description_html=description_html, + hide_sql=hide_sql, + fragment=fragment, + parameters=parameters, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + on_success_message=on_success_message, + on_success_message_sql=on_success_message_sql, + on_success_redirect=on_success_redirect, + on_error_message=on_error_message, + on_error_redirect=on_error_redirect, ) async def remove_query(self, database, name, source=None): - sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" - params = [database, name] - if source is not None: - sql += " AND source = ?" - params.append(source) - await self.get_internal_database().execute_write(sql, params) + return await stored_queries.remove_query(self, database, name, source=source) async def get_query(self, database, name): - rows = await self.get_internal_database().execute( - """ - SELECT * FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - return self._query_row_to_dict(rows.first()) + return await stored_queries.get_query(self, database, name) async def count_queries( self, @@ -1310,62 +1147,17 @@ class Datasette: source=None, owner_id=None, ): - allowed_sql, allowed_params = await self.allowed_resources_sql( - action="view-query", + return await stored_queries.count_queries( + self, + database, actor=actor, - parent=database, + q=q, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, ) - params = dict(allowed_params) - where_clauses = [] - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - row = ( - await self.get_internal_database().execute( - """ - SELECT count(*) AS count - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - """.format( - allowed_sql=allowed_sql, - where=" AND ".join(where_clauses) or "1 = 1", - ), - params, - ) - ).first() - return row["count"] async def list_queries( self, @@ -1382,176 +1174,27 @@ class Datasette: owner_id=None, include_private=False, ): - limit = min(max(1, int(limit)), 1000) - allowed_sql, allowed_params = await self.allowed_resources_sql( - action="view-query", + return await stored_queries.list_queries( + self, + database, actor=actor, - parent=database, - include_is_private=include_private, + limit=limit, + cursor=cursor, + q=q, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + include_private=include_private, ) - params = dict(allowed_params) - params.update({"limit": limit + 1}) - sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" - where_clauses = [] - order_by = "q.database_name, sort_key, q.name" - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - order_by = "sort_key, q.name" - - if cursor: - try: - components = urlsafe_components(cursor) - except ValueError: - components = [] - if database is None and len(components) == 3: - where_clauses.append(""" - ( - q.database_name > :cursor_database - OR ( - q.database_name = :cursor_database - AND ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_database"] = components[0] - params["cursor_sort_key"] = components[1] - params["cursor_name"] = components[2] - elif database is not None and len(components) == 2: - where_clauses.append(""" - ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_sort_key"] = components[0] - params["cursor_name"] = components[1] - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - private_select = ", allowed.is_private AS private" if include_private else "" - rows = list( - ( - await self.get_internal_database().execute( - """ - SELECT q.*, {sort_key_sql} AS sort_key{private_select} - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - ORDER BY {order_by} - LIMIT :limit - """.format( - allowed_sql=allowed_sql, - private_select=private_select, - sort_key_sql=sort_key_sql, - where=" AND ".join(where_clauses) or "1 = 1", - order_by=order_by, - ), - params, - ) - ).rows - ) - has_more = len(rows) > limit - if has_more: - rows = rows[:limit] - - queries = [] - for row in rows: - query = self._query_row_to_dict(row) - if include_private: - query["private"] = bool(row["private"]) - queries.append(query) - - next_token = None - if has_more and rows: - last_row = rows[-1] - if database is None: - next_token = "{},{},{}".format( - tilde_encode(last_row["database_name"]), - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - else: - next_token = "{},{}".format( - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - return { - "queries": queries, - "next": next_token, - "has_more": has_more, - "limit": limit, - } async def ensure_query_write_permissions( self, database, sql, *, actor=None, params=None, analysis=None ): - write_actions = { - "insert": "insert-row", - "update": "update-row", - "delete": "delete-row", - } - db = self.get_database(database) - if analysis is None: - if params is None: - params = {name: "" for name in named_parameters(sql)} - try: - analysis = await db.analyze_sql(sql, params) - except sqlite3.DatabaseError as ex: - raise Forbidden(f"Could not analyze query: {ex}") from ex - - for access in analysis.table_accesses: - action = write_actions.get(access.operation) - if action is None: - continue - if access.database != database: - raise Forbidden("Writable queries may not write to attached databases") - if not await self.allowed( - action=action, - resource=TableResource(database=access.database, table=access.table), - actor=actor, - ): - raise Forbidden( - f"Permission denied: need {action} on {access.database}/{access.table}" - ) - return analysis + return await stored_queries.ensure_query_write_permissions( + self, database, sql, actor=actor, params=params, analysis=analysis + ) # Column types API diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py new file mode 100644 index 00000000..e88902e7 --- /dev/null +++ b/datasette/stored_queries.py @@ -0,0 +1,544 @@ +from __future__ import annotations + +import json + +from .resources import TableResource +from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components +from .utils.asgi import Forbidden + + +UNCHANGED = object() + +QUERY_OPTION_FIELDS = ( + "hide_sql", + "fragment", + "on_success_message", + "on_success_message_sql", + "on_success_redirect", + "on_error_message", + "on_error_redirect", +) + + +async def save_queries_from_config(datasette): + # Apply configured query entries from datasette.yaml to the internal table. + await datasette.get_internal_database().execute_write( + "DELETE FROM queries WHERE source = 'config'" + ) + for dbname, db_config in ((datasette.config or {}).get("databases") or {}).items(): + for query_name, query_config in (db_config.get("queries") or {}).items(): + if not isinstance(query_config, dict): + query_config = {"sql": query_config} + await datasette.add_query( + dbname, + query_name, + query_config["sql"], + title=query_config.get("title"), + description=query_config.get("description"), + description_html=query_config.get("description_html"), + hide_sql=bool(query_config.get("hide_sql")), + fragment=query_config.get("fragment"), + parameters=query_config.get("params"), + is_write=bool(query_config.get("write")), + is_private=bool(query_config.get("is_private")), + is_trusted=bool(query_config.get("is_trusted", True)), + source="config", + on_success_message=query_config.get("on_success_message"), + on_success_message_sql=query_config.get("on_success_message_sql"), + on_success_redirect=query_config.get("on_success_redirect"), + on_error_message=query_config.get("on_error_message"), + on_error_redirect=query_config.get("on_error_redirect"), + ) + + +def query_row_to_dict(row): + if row is None: + return None + parameters = json.loads(row["parameters"] or "[]") + options = json.loads(row["options"] or "{}") + is_write = bool(row["is_write"]) + return { + "database": row["database_name"], + "name": row["name"], + "sql": row["sql"], + "title": row["title"], + "description": row["description"], + "description_html": row["description_html"], + "hide_sql": bool(options.get("hide_sql")), + "fragment": options.get("fragment"), + "params": parameters, + "parameters": parameters, + "is_write": is_write, + "write": is_write, + "is_private": bool(row["is_private"]), + "is_trusted": bool(row["is_trusted"]), + "source": row["source"], + "owner_id": row["owner_id"], + "on_success_message": options.get("on_success_message"), + "on_success_message_sql": options.get("on_success_message_sql"), + "on_success_redirect": options.get("on_success_redirect"), + "on_error_message": options.get("on_error_message"), + "on_error_redirect": options.get("on_error_redirect"), + } + + +def query_options_json(options): + options_dict = {} + for field in QUERY_OPTION_FIELDS: + value = options.get(field) + if field == "hide_sql": + if value: + options_dict[field] = True + elif value is not None: + options_dict[field] = value + return json.dumps(options_dict, sort_keys=True) + + +async def add_query( + datasette, + database, + name, + sql, + *, + title=None, + description=None, + description_html=None, + hide_sql=False, + fragment=None, + parameters=None, + is_write=False, + is_private=False, + is_trusted=False, + source="plugin", + owner_id=None, + on_success_message=None, + on_success_message_sql=None, + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + replace=True, +): + parameters_json = json.dumps(list(parameters or [])) + options_json = query_options_json( + { + "hide_sql": hide_sql, + "fragment": fragment, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + ) + sql_statement = """ + INSERT INTO queries ( + database_name, name, sql, title, description, description_html, + options, parameters, is_write, is_private, is_trusted, source, owner_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + if replace: + sql_statement += """ + ON CONFLICT(database_name, name) DO UPDATE SET + sql = excluded.sql, + title = excluded.title, + description = excluded.description, + description_html = excluded.description_html, + options = excluded.options, + parameters = excluded.parameters, + is_write = excluded.is_write, + is_private = excluded.is_private, + is_trusted = excluded.is_trusted, + source = excluded.source, + owner_id = excluded.owner_id, + updated_at = CURRENT_TIMESTAMP + """ + await datasette.get_internal_database().execute_write( + sql_statement, + [ + database, + name, + sql, + title, + description, + description_html, + options_json, + parameters_json, + int(bool(is_write)), + int(bool(is_private)), + int(bool(is_trusted)), + source, + owner_id, + ], + ) + + +async def update_query( + datasette, + database, + name, + *, + sql=UNCHANGED, + title=UNCHANGED, + description=UNCHANGED, + description_html=UNCHANGED, + hide_sql=UNCHANGED, + fragment=UNCHANGED, + parameters=UNCHANGED, + is_write=UNCHANGED, + is_private=UNCHANGED, + is_trusted=UNCHANGED, + source=UNCHANGED, + owner_id=UNCHANGED, + on_success_message=UNCHANGED, + on_success_message_sql=UNCHANGED, + on_success_redirect=UNCHANGED, + on_error_message=UNCHANGED, + on_error_redirect=UNCHANGED, +): + fields = { + "sql": sql, + "title": title, + "description": description, + "description_html": description_html, + "parameters": parameters, + "is_write": is_write, + "is_private": is_private, + "is_trusted": is_trusted, + "source": source, + "owner_id": owner_id, + } + option_fields = { + "hide_sql": hide_sql, + "fragment": fragment, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + updates = [] + params = [] + for field, value in fields.items(): + if value is UNCHANGED: + continue + if field in {"is_write", "is_private", "is_trusted"}: + value = int(bool(value)) + elif field == "parameters": + value = json.dumps(list(value or [])) + updates.append(f"{field} = ?") + params.append(value) + changed_options = { + field: value for field, value in option_fields.items() if value is not UNCHANGED + } + if changed_options: + rows = await datasette.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + row = rows.first() + options = json.loads(row["options"] or "{}") if row is not None else {} + for field, value in changed_options.items(): + if field == "hide_sql": + if value: + options[field] = True + else: + options.pop(field, None) + elif value is None: + options.pop(field, None) + else: + options[field] = value + updates.append("options = ?") + params.append(json.dumps(options, sort_keys=True)) + if not updates: + return + updates.append("updated_at = CURRENT_TIMESTAMP") + params.extend([database, name]) + await datasette.get_internal_database().execute_write( + """ + UPDATE queries + SET {} + WHERE database_name = ? AND name = ? + """.format(", ".join(updates)), + params, + ) + + +async def remove_query(datasette, database, name, source=None): + sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" + params = [database, name] + if source is not None: + sql += " AND source = ?" + params.append(source) + await datasette.get_internal_database().execute_write(sql, params) + + +async def get_query(datasette, database, name): + rows = await datasette.get_internal_database().execute( + """ + SELECT * FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + return query_row_to_dict(rows.first()) + + +async def count_queries( + datasette, + database=None, + *, + actor=None, + q=None, + is_write=None, + is_private=None, + is_trusted=None, + source=None, + owner_id=None, +): + allowed_sql, allowed_params = await datasette.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + ) + params = dict(allowed_params) + where_clauses = [] + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + row = ( + await datasette.get_internal_database().execute( + """ + SELECT count(*) AS count + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + """.format( + allowed_sql=allowed_sql, + where=" AND ".join(where_clauses) or "1 = 1", + ), + params, + ) + ).first() + return row["count"] + + +async def list_queries( + datasette, + database=None, + *, + actor=None, + limit=50, + cursor=None, + q=None, + is_write=None, + is_private=None, + is_trusted=None, + source=None, + owner_id=None, + include_private=False, +): + limit = min(max(1, int(limit)), 1000) + allowed_sql, allowed_params = await datasette.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + include_is_private=include_private, + ) + params = dict(allowed_params) + params.update({"limit": limit + 1}) + sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" + where_clauses = [] + order_by = "q.database_name, sort_key, q.name" + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + order_by = "sort_key, q.name" + + if cursor: + try: + components = urlsafe_components(cursor) + except ValueError: + components = [] + if database is None and len(components) == 3: + where_clauses.append(""" + ( + q.database_name > :cursor_database + OR ( + q.database_name = :cursor_database + AND ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_database"] = components[0] + params["cursor_sort_key"] = components[1] + params["cursor_name"] = components[2] + elif database is not None and len(components) == 2: + where_clauses.append(""" + ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_sort_key"] = components[0] + params["cursor_name"] = components[1] + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + private_select = ", allowed.is_private AS private" if include_private else "" + rows = list( + ( + await datasette.get_internal_database().execute( + """ + SELECT q.*, {sort_key_sql} AS sort_key{private_select} + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + ORDER BY {order_by} + LIMIT :limit + """.format( + allowed_sql=allowed_sql, + private_select=private_select, + sort_key_sql=sort_key_sql, + where=" AND ".join(where_clauses) or "1 = 1", + order_by=order_by, + ), + params, + ) + ).rows + ) + has_more = len(rows) > limit + if has_more: + rows = rows[:limit] + + queries = [] + for row in rows: + query = query_row_to_dict(row) + if include_private: + query["private"] = bool(row["private"]) + queries.append(query) + + next_token = None + if has_more and rows: + last_row = rows[-1] + if database is None: + next_token = "{},{},{}".format( + tilde_encode(last_row["database_name"]), + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + else: + next_token = "{},{}".format( + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + return { + "queries": queries, + "next": next_token, + "has_more": has_more, + "limit": limit, + } + + +async def ensure_query_write_permissions( + datasette, database, sql, *, actor=None, params=None, analysis=None +): + write_actions = { + "insert": "insert-row", + "update": "update-row", + "delete": "delete-row", + } + db = datasette.get_database(database) + if analysis is None: + if params is None: + params = {name: "" for name in named_parameters(sql)} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise Forbidden(f"Could not analyze query: {ex}") from ex + + for access in analysis.table_accesses: + action = write_actions.get(access.operation) + if action is None: + continue + if access.database != database: + raise Forbidden("Writable queries may not write to attached databases") + if not await datasette.allowed( + action=action, + resource=TableResource(database=access.database, table=access.table), + actor=actor, + ): + raise Forbidden( + f"Permission denied: need {action} on {access.database}/{access.table}" + ) + return analysis From 58e2e3a8ab9adfeda2686be5213eaf3f4118da33 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:43:34 -0700 Subject: [PATCH 730/767] Ran cog --- docs/authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index cec47f97..f8b1dc49 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -496,7 +496,7 @@ Here's how to restrict access to your entire Datasette instance to just the ``"i title: My private Datasette instance allow: id: root - + .. tab:: datasette.json From c3ceabae03c2c9cf78f63f7c12c8d90301097cb4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:51:40 -0700 Subject: [PATCH 731/767] Ran Black --- datasette/stored_queries.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index e88902e7..5b005c07 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -6,7 +6,6 @@ from .resources import TableResource from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components from .utils.asgi import Forbidden - UNCHANGED = object() QUERY_OPTION_FIELDS = ( From d6de8e752083a014bdd7ffb701dc08519da82554 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:52:16 -0700 Subject: [PATCH 732/767] Link to save query from /-/execute-write --- datasette/templates/execute_write.html | 29 +++++++++++++++++++++++++- datasette/views/database.py | 22 +++++++++++++++++++ tests/test_queries.py | 17 +++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 7a627a7a..6b626f8d 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -119,7 +119,10 @@ {% endif %}
-

+

+ + {% if save_query_base_url %}Save this query{% endif %} +

", + "on_success_message_sql": "select 'secret'", + } + }, + ) + form_response = await ds.client.post( + "/data/-/queries/store", + actor={"id": "root"}, + data={ + "name": "unsafe_form", + "sql": "select 1", + "description_html": "", + }, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == [ + "Invalid keys: description_html, on_success_message_sql" + ] + assert form_response.status_code == 400 + assert "Invalid keys: description_html" in form_response.text + assert await ds.get_query("data", "unsafe") is None + assert await ds.get_query("data", "unsafe_form") is None + + @pytest.mark.asyncio async def test_query_store_api_creates_writable_query(): ds = Datasette(memory=True, default_deny=True) @@ -959,6 +1000,42 @@ async def test_query_update_and_delete_api(): assert await ds.get_query("data", "editable") is None +@pytest.mark.asyncio +async def test_query_update_api_rejects_config_only_fields(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_update_config_only_fields", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "editable", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="root", + ) + + response = await ds.client.post( + "/data/editable/-/update", + actor={"id": "root"}, + json={ + "update": { + "description_html": "", + "on_success_message_sql": "select 'secret'", + } + }, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == [ + "Invalid keys: description_html, on_success_message_sql" + ] + query = await ds.get_query("data", "editable") + assert query["description_html"] is None + assert query["on_success_message_sql"] is None + + @pytest.mark.asyncio async def test_query_update_api_rejects_trusted_queries_but_internal_update_allowed(): ds = Datasette( From b1289a73f9869e83a433a088c2a6c48285e67f2d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 16:51:00 -0700 Subject: [PATCH 741/767] stored_queries.StoredQuery dataclass --- datasette/app.py | 102 ++++++------ datasette/stored_queries.py | 258 ++++++++++++++++++++---------- datasette/views/database.py | 56 +++---- datasette/views/query_helpers.py | 19 +-- datasette/views/stored_queries.py | 37 +++-- docs/internals.rst | 14 +- tests/test_queries.py | 128 +++++++-------- 7 files changed, 357 insertions(+), 257 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 96683895..56b89789 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1029,8 +1029,8 @@ class Datasette: ) @staticmethod - def _query_row_to_dict(row): - return stored_queries.query_row_to_dict(row) + def _query_row_to_stored_query(row) -> stored_queries.StoredQuery | None: + return stored_queries.query_row_to_stored_query(row) @staticmethod def _query_options_json(options): @@ -1038,28 +1038,28 @@ class Datasette: async def add_query( self, - database, - name, - sql, + database: str, + name: str, + sql: str, *, - title=None, - description=None, - description_html=None, - hide_sql=False, - fragment=None, - parameters=None, - is_write=False, - is_private=False, - is_trusted=False, - source="plugin", - owner_id=None, - on_success_message=None, - on_success_message_sql=None, - on_success_redirect=None, - on_error_message=None, - on_error_redirect=None, - replace=True, - ): + title: str | None = None, + description: str | None = None, + description_html: str | None = None, + hide_sql: bool = False, + fragment: str | None = None, + parameters: Iterable[str] | None = None, + is_write: bool = False, + is_private: bool = False, + is_trusted: bool = False, + source: str = "plugin", + owner_id: str | None = None, + on_success_message: str | None = None, + on_success_message_sql: str | None = None, + on_success_redirect: str | None = None, + on_error_message: str | None = None, + on_error_redirect: str | None = None, + replace: bool = True, + ) -> None: return await stored_queries.add_query( self, database, @@ -1086,8 +1086,8 @@ class Datasette: async def update_query( self, - database, - name, + database: str, + name: str, *, sql=stored_queries.UNCHANGED, title=stored_queries.UNCHANGED, @@ -1106,7 +1106,7 @@ class Datasette: on_success_redirect=stored_queries.UNCHANGED, on_error_message=stored_queries.UNCHANGED, on_error_redirect=stored_queries.UNCHANGED, - ): + ) -> None: return await stored_queries.update_query( self, database, @@ -1130,24 +1130,28 @@ class Datasette: on_error_redirect=on_error_redirect, ) - async def remove_query(self, database, name, source=None): + async def remove_query( + self, database: str, name: str, source: str | None = None + ) -> None: return await stored_queries.remove_query(self, database, name, source=source) - async def get_query(self, database, name): + async def get_query( + self, database: str, name: str + ) -> stored_queries.StoredQuery | None: return await stored_queries.get_query(self, database, name) async def count_queries( self, - database=None, + database: str | None = None, *, - actor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, - ): + actor: dict[str, Any] | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + ) -> int: return await stored_queries.count_queries( self, database, @@ -1162,19 +1166,19 @@ class Datasette: async def list_queries( self, - database=None, + database: str | None = None, *, - actor=None, - limit=50, - cursor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, - include_private=False, - ): + actor: dict[str, Any] | None = None, + limit: int = 50, + cursor: str | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + include_private: bool = False, + ) -> stored_queries.StoredQueryPage: return await stored_queries.list_queries( self, database, diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index a28b71bf..bcfdfdb4 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -1,6 +1,8 @@ from __future__ import annotations +from dataclasses import dataclass import json +from typing import Any, Iterable from .resources import TableResource from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components @@ -19,7 +21,76 @@ QUERY_OPTION_FIELDS = ( ) -async def save_queries_from_config(datasette): +@dataclass +class StoredQuery: + database: str + name: str + sql: str + title: str | None + description: str | None + description_html: str | None + hide_sql: bool + fragment: str | None + parameters: list[str] + is_write: bool + is_private: bool + is_trusted: bool + source: str + owner_id: str | None + on_success_message: str | None + on_success_message_sql: str | None + on_success_redirect: str | None + on_error_message: str | None + on_error_redirect: str | None + private: bool | None = None + + +@dataclass +class StoredQueryPage: + queries: list[StoredQuery] + next: str | None + has_more: bool + limit: int + + +def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]: + data = { + "database": query.database, + "name": query.name, + "sql": query.sql, + "title": query.title, + "description": query.description, + "description_html": query.description_html, + "hide_sql": query.hide_sql, + "fragment": query.fragment, + "params": list(query.parameters), + "parameters": list(query.parameters), + "is_write": query.is_write, + "is_private": query.is_private, + "is_trusted": query.is_trusted, + "source": query.source, + "owner_id": query.owner_id, + "on_success_message": query.on_success_message, + "on_success_message_sql": query.on_success_message_sql, + "on_success_redirect": query.on_success_redirect, + "on_error_message": query.on_error_message, + "on_error_redirect": query.on_error_redirect, + } + if query.private is not None: + data["private"] = query.private + return data + + +def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]: + return { + "queries": [stored_query_to_dict(query) for query in page.queries], + "next": page.next, + "has_more": page.has_more, + "limit": page.limit, + } + + +async def save_queries_from_config(datasette: Any) -> None: # Apply configured query entries from datasette.yaml to the internal table. await datasette.get_internal_database().execute_write( "DELETE FROM queries WHERE source = 'config'" @@ -50,36 +121,38 @@ async def save_queries_from_config(datasette): ) -def query_row_to_dict(row): +def query_row_to_stored_query( + row: Any, private: bool | None = None +) -> StoredQuery | None: if row is None: return None parameters = json.loads(row["parameters"] or "[]") options = json.loads(row["options"] or "{}") - return { - "database": row["database_name"], - "name": row["name"], - "sql": row["sql"], - "title": row["title"], - "description": row["description"], - "description_html": row["description_html"], - "hide_sql": bool(options.get("hide_sql")), - "fragment": options.get("fragment"), - "params": parameters, - "parameters": parameters, - "is_write": bool(row["is_write"]), - "is_private": bool(row["is_private"]), - "is_trusted": bool(row["is_trusted"]), - "source": row["source"], - "owner_id": row["owner_id"], - "on_success_message": options.get("on_success_message"), - "on_success_message_sql": options.get("on_success_message_sql"), - "on_success_redirect": options.get("on_success_redirect"), - "on_error_message": options.get("on_error_message"), - "on_error_redirect": options.get("on_error_redirect"), - } + return StoredQuery( + database=row["database_name"], + name=row["name"], + sql=row["sql"], + title=row["title"], + description=row["description"], + description_html=row["description_html"], + hide_sql=bool(options.get("hide_sql")), + fragment=options.get("fragment"), + parameters=parameters, + is_write=bool(row["is_write"]), + is_private=bool(row["is_private"]), + is_trusted=bool(row["is_trusted"]), + source=row["source"], + owner_id=row["owner_id"], + on_success_message=options.get("on_success_message"), + on_success_message_sql=options.get("on_success_message_sql"), + on_success_redirect=options.get("on_success_redirect"), + on_error_message=options.get("on_error_message"), + on_error_redirect=options.get("on_error_redirect"), + private=private, + ) -def query_options_json(options): +def query_options_json(options: dict[str, Any]) -> str: options_dict = {} for field in QUERY_OPTION_FIELDS: value = options.get(field) @@ -92,29 +165,29 @@ def query_options_json(options): async def add_query( - datasette, - database, - name, - sql, + datasette: Any, + database: str, + name: str, + sql: str, *, - title=None, - description=None, - description_html=None, - hide_sql=False, - fragment=None, - parameters=None, - is_write=False, - is_private=False, - is_trusted=False, - source="plugin", - owner_id=None, - on_success_message=None, - on_success_message_sql=None, - on_success_redirect=None, - on_error_message=None, - on_error_redirect=None, - replace=True, -): + title: str | None = None, + description: str | None = None, + description_html: str | None = None, + hide_sql: bool = False, + fragment: str | None = None, + parameters: Iterable[str] | None = None, + is_write: bool = False, + is_private: bool = False, + is_trusted: bool = False, + source: str = "plugin", + owner_id: str | None = None, + on_success_message: str | None = None, + on_success_message_sql: str | None = None, + on_success_redirect: str | None = None, + on_error_message: str | None = None, + on_error_redirect: str | None = None, + replace: bool = True, +) -> None: parameters_json = json.dumps(list(parameters or [])) options_json = query_options_json( { @@ -170,9 +243,9 @@ async def add_query( async def update_query( - datasette, - database, - name, + datasette: Any, + database: str, + name: str, *, sql=UNCHANGED, title=UNCHANGED, @@ -191,7 +264,7 @@ async def update_query( on_success_redirect=UNCHANGED, on_error_message=UNCHANGED, on_error_redirect=UNCHANGED, -): +) -> None: fields = { "sql": sql, "title": title, @@ -263,7 +336,9 @@ async def update_query( ) -async def remove_query(datasette, database, name, source=None): +async def remove_query( + datasette: Any, database: str, name: str, source: str | None = None +) -> None: sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" params = [database, name] if source is not None: @@ -272,7 +347,7 @@ async def remove_query(datasette, database, name, source=None): await datasette.get_internal_database().execute_write(sql, params) -async def get_query(datasette, database, name): +async def get_query(datasette: Any, database: str, name: str) -> StoredQuery | None: rows = await datasette.get_internal_database().execute( """ SELECT * FROM queries @@ -280,21 +355,21 @@ async def get_query(datasette, database, name): """, [database, name], ) - return query_row_to_dict(rows.first()) + return query_row_to_stored_query(rows.first()) async def count_queries( - datasette, - database=None, + datasette: Any, + database: str | None = None, *, - actor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, -): + actor: dict[str, Any] | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, +) -> int: allowed_sql, allowed_params = await datasette.allowed_resources_sql( action="view-query", actor=actor, @@ -354,20 +429,20 @@ async def count_queries( async def list_queries( - datasette, - database=None, + datasette: Any, + database: str | None = None, *, - actor=None, - limit=50, - cursor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, - include_private=False, -): + actor: dict[str, Any] | None = None, + limit: int = 50, + cursor: str | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + include_private: bool = False, +) -> StoredQueryPage: limit = min(max(1, int(limit)), 1000) allowed_sql, allowed_params = await datasette.allowed_resources_sql( action="view-query", @@ -480,9 +555,10 @@ async def list_queries( queries = [] for row in rows: - query = query_row_to_dict(row) - if include_private: - query["private"] = bool(row["private"]) + query = query_row_to_stored_query( + row, private=bool(row["private"]) if include_private else None + ) + assert query is not None queries.append(query) next_token = None @@ -499,17 +575,23 @@ async def list_queries( tilde_encode(last_row["sort_key"]), tilde_encode(last_row["name"]), ) - return { - "queries": queries, - "next": next_token, - "has_more": has_more, - "limit": limit, - } + return StoredQueryPage( + queries=queries, + next=next_token, + has_more=has_more, + limit=limit, + ) async def ensure_query_write_permissions( - datasette, database, sql, *, actor=None, params=None, analysis=None -): + datasette: Any, + database: str, + sql: str, + *, + actor: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + analysis: Any = None, +) -> Any: write_actions = { "insert": "insert-row", "update": "update-row", diff --git a/datasette/views/database.py b/datasette/views/database.py index 98ca989c..b558b002 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -13,6 +13,7 @@ import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import stored_query_to_dict from datasette.utils import ( add_cors_headers, await_me_maybe, @@ -99,8 +100,8 @@ class DatabaseView(View): limit=5, include_private=True, ) - stored_queries = queries_page["queries"] - queries_more = queries_page["has_more"] + stored_queries = queries_page.queries + queries_more = queries_page.has_more queries_count = ( await datasette.count_queries(database, actor=request.actor) if queries_more @@ -136,7 +137,7 @@ class DatabaseView(View): "tables": tables, "hidden_count": len([t for t in tables if t["hidden"]]), "views": sql_views, - "queries": stored_queries, + "queries": [stored_query_to_dict(query) for query in stored_queries], "queries_more": queries_more, "queries_count": queries_count, "allow_execute_sql": allow_execute_sql, @@ -447,7 +448,7 @@ class QueryView(View): if not await datasette.allowed( action="view-query", - resource=QueryResource(database=db.name, query=stored_query["name"]), + resource=QueryResource(database=db.name, query=stored_query.name), actor=request.actor, ): raise Forbidden("You do not have permission to view this query") @@ -480,20 +481,18 @@ class QueryView(View): or request.args.get("_json") or params.get("_json") ) - params_for_query = MagicParameters( - stored_query["sql"], params, request, datasette - ) + params_for_query = MagicParameters(stored_query.sql, params, request, datasette) await params_for_query.execute_params() ok = None redirect_url = None try: cursor = await db.execute_write( - stored_query["sql"], params_for_query, request=request + stored_query.sql, params_for_query, request=request ) # success message can come from on_success_message or on_success_message_sql message = None message_type = datasette.INFO - on_success_message_sql = stored_query.get("on_success_message_sql") + on_success_message_sql = stored_query.on_success_message_sql if on_success_message_sql: try: message_result = ( @@ -505,18 +504,19 @@ class QueryView(View): message = "Error running on_success_message_sql: {}".format(ex) message_type = datasette.ERROR if not message: - message = stored_query.get( - "on_success_message" - ) or "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" + message = ( + stored_query.on_success_message + or "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) ) - redirect_url = stored_query.get("on_success_redirect") + redirect_url = stored_query.on_success_redirect ok = True except Exception as ex: - message = stored_query.get("on_error_message") or str(ex) + message = stored_query.on_error_message or str(ex) message_type = datasette.ERROR - redirect_url = stored_query.get("on_error_redirect") + redirect_url = stored_query.on_error_redirect ok = False if should_return_json: return Response.json( @@ -562,7 +562,7 @@ class QueryView(View): ) if stored_query is None: raise - stored_query_write = bool(stored_query.get("is_write")) + stored_query_write = stored_query.is_write private = False if stored_query: @@ -570,7 +570,7 @@ class QueryView(View): visible, private = await datasette.check_visibility( request.actor, action="view-query", - resource=QueryResource(database=database, query=stored_query["name"]), + resource=QueryResource(database=database, query=stored_query.name), ) if not visible: raise Forbidden("You do not have permission to view this query") @@ -591,14 +591,14 @@ class QueryView(View): sql = None if stored_query: - sql = stored_query["sql"] + sql = stored_query.sql elif "sql" in params: sql = params.pop("sql") # Extract any :named parameters named_parameters = [] - if stored_query and stored_query.get("params"): - named_parameters = stored_query["params"] + if stored_query and stored_query.parameters: + named_parameters = stored_query.parameters if not named_parameters and sql: named_parameters = derive_named_parameters(sql) named_parameter_values = { @@ -686,7 +686,7 @@ class QueryView(View): columns=columns, rows=rows, sql=sql, - query_name=stored_query["name"] if stored_query else None, + query_name=stored_query.name if stored_query else None, database=database, table=None, request=request, @@ -721,7 +721,7 @@ class QueryView(View): if stored_query: templates.insert( 0, - f"query-{to_css_class(database)}-{to_css_class(stored_query['name'])}.html", + f"query-{to_css_class(database)}-{to_css_class(stored_query.name)}.html", ) environment = datasette.get_jinja_environment(request) @@ -740,7 +740,7 @@ class QueryView(View): ) metadata = await datasette.get_database_metadata(database) if stored_query: - metadata = dict(stored_query) + metadata = stored_query_to_dict(stored_query) metadata.pop("source", None) renderers = {} @@ -775,7 +775,7 @@ class QueryView(View): ) show_hide_hidden = "" - if stored_query and stored_query.get("hide_sql"): + if stored_query and stored_query.hide_sql: if bool(params.get("_show_sql")): show_hide_link = path_with_removed_args(request, {"_show_sql"}) show_hide_text = "hide" @@ -843,7 +843,7 @@ class QueryView(View): datasette=datasette, actor=request.actor, database=database, - query_name=stored_query["name"] if stored_query else None, + query_name=stored_query.name if stored_query else None, request=request, sql=sql, params=params, @@ -863,7 +863,7 @@ class QueryView(View): "sql": sql, "params": params, }, - stored_query=stored_query["name"] if stored_query else None, + stored_query=stored_query.name if stored_query else None, private=private, stored_query_write=stored_query_write, db_is_immutable=not db.is_mutable, @@ -907,7 +907,7 @@ class QueryView(View): datasette, request, database=database, - query_name=stored_query["name"] if stored_query else None, + query_name=stored_query.name if stored_query else None, ), query_actions=query_actions, ), diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index de732431..46d71b8e 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -2,6 +2,7 @@ import json import re from datasette.resources import DatabaseResource, TableResource +from datasette.stored_queries import StoredQuery from datasette.utils import ( named_parameters as derive_named_parameters, escape_sqlite, @@ -281,18 +282,18 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis -async def _ensure_stored_query_execution_permissions(datasette, db, query, actor): - if query.get("is_trusted"): +async def _ensure_stored_query_execution_permissions( + datasette, db, query: StoredQuery, actor +): + if query.is_trusted: return - if query.get("is_write"): + if query.is_write: await datasette.ensure_permission( action="execute-write-sql", resource=DatabaseResource(db.name), actor=actor, ) - await datasette.ensure_query_write_permissions( - db.name, query["sql"], actor=actor - ) + await datasette.ensure_query_write_permissions(db.name, query.sql, actor=actor) else: await datasette.ensure_permission( action="execute-sql", @@ -482,7 +483,7 @@ async def _prepare_query_create(datasette, request, db, data): } -async def _prepare_query_update(datasette, request, db, existing, update): +async def _prepare_query_update(datasette, request, db, existing: StoredQuery, update): invalid_keys = set(update) - _query_update_fields if invalid_keys: raise QueryValidationError( @@ -490,8 +491,8 @@ async def _prepare_query_update(datasette, request, db, existing, update): ) update = _apply_query_data_types(update) - sql = update.get("sql", existing["sql"]) - query_is_write = existing["is_write"] + sql = update.get("sql", existing.sql) + query_is_write = existing.is_write derived = _derived_query_parameters(sql) parameters = None diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 1a2c5d00..8c4e849e 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -1,6 +1,7 @@ from urllib.parse import parse_qsl, urlencode from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import stored_query_to_dict from datasette.utils import sqlite3, tilde_decode from datasette.utils.asgi import Response @@ -100,7 +101,7 @@ class QueryListView(BaseView): ) query_list_path = self.query_list_path(database) next_url = None - if page["next"]: + if page.next: pairs = [ (key, value) for key, value in parse_qsl( @@ -108,7 +109,7 @@ class QueryListView(BaseView): ) if key != "_next" ] - pairs.append(("_next", page["next"])) + pairs.append(("_next", page.next)) next_url = "{}?{}".format( query_list_path, urlencode(pairs), @@ -194,13 +195,13 @@ class QueryListView(BaseView): "database_color": ( self.ds.get_database(database).color if database is not None else None ), - "queries": page["queries"], - "next": page["next"], + "queries": page.queries, + "next": page.next, "next_url": next_url, - "has_more": page["has_more"], - "limit": page["limit"], - "show_private_note": any(query["is_private"] for query in page["queries"]), - "show_trusted_note": any(query["is_trusted"] for query in page["queries"]), + "has_more": page.has_more, + "limit": page.limit, + "show_private_note": any(query.is_private for query in page.queries), + "show_trusted_note": any(query.is_trusted for query in page.queries), "query_list_path": query_list_path, "show_database": database is None, "facets": facets, @@ -213,7 +214,12 @@ class QueryListView(BaseView): }, } if format_ == "json": - return Response.json(data) + return Response.json( + { + **data, + "queries": [stored_query_to_dict(query) for query in page.queries], + } + ) return await self.render( ["query_list.html"], request, @@ -374,8 +380,11 @@ class QueryStoreView(QueryCreateView): return _error([str(ex)], 400) query = await self.ds.get_query(db.name, name) + assert query is not None if is_json: - return Response.json({"ok": True, "query": query}, status=201) + return Response.json( + {"ok": True, "query": stored_query_to_dict(query)}, status=201 + ) self.ds.add_message(request, "Query saved", self.ds.INFO) return Response.redirect(self.ds.urls.path(self.ds.urls.table(db.name, name))) @@ -395,7 +404,7 @@ class QueryDefinitionView(BaseView): actor=request.actor, ): return _error(["Permission denied"], 403) - return Response.json({"ok": True, "query": query}) + return Response.json({"ok": True, "query": stored_query_to_dict(query)}) class QueryUpdateView(BaseView): @@ -413,7 +422,7 @@ class QueryUpdateView(BaseView): actor=request.actor, ): return _error(["Permission denied: need update-query"], 403) - if existing.get("is_trusted"): + if existing.is_trusted: return _error(["Trusted queries cannot be updated using the API"], 403) try: @@ -444,10 +453,12 @@ class QueryUpdateView(BaseView): await self.ds.update_query(db.name, query_name, **update_kwargs) if data.get("return"): + query = await self.ds.get_query(db.name, query_name) + assert query is not None return Response.json( { "ok": True, - "query": await self.ds.get_query(db.name, query_name), + "query": stored_query_to_dict(query), } ) return Response.json({"ok": True}) diff --git a/docs/internals.rst b/docs/internals.rst index 66724aa9..4980ee8b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1039,11 +1039,11 @@ Example: await .get_query(database, name) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns a stored query dictionary, or ``None`` if the query does not exist. +Returns a ``StoredQuery`` dataclass instance, or ``None`` if the query does not exist. -The dictionary contains ``database``, ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``params``, ``is_write``, ``is_private``, ``is_trusted``, ``source``, ``owner_id``, ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``. +``StoredQuery`` has the following attributes: ``database``, ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write``, ``is_private``, ``is_trusted``, ``source``, ``owner_id``, ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``. -``parameters`` and ``params`` contain the same list of explicit parameter names. +``parameters`` is a list of explicit parameter names. .. _datasette_list_queries: @@ -1087,12 +1087,12 @@ Lists stored queries visible to the specified actor. ``owner_id`` - string, optional Filter by owner actor ID. ``include_private`` - boolean, optional - Set to ``True`` to include a ``private`` boolean in each returned query dictionary indicating if anonymous users would be unable to view that query. + Set to ``True`` to populate a ``private`` boolean on each returned ``StoredQuery`` indicating if anonymous users would be unable to view that query. -The return value is a dictionary with these keys: +The return value is a ``StoredQueryPage`` dataclass instance with these attributes: -``queries`` - list of dictionaries - Stored query dictionaries, in the same format returned by :ref:`datasette_get_query`. +``queries`` - list of StoredQuery instances + Stored queries in the same format returned by :ref:`datasette_get_query`. ``next`` - string or None Pagination cursor for the next page, if one exists. ``has_more`` - boolean diff --git a/tests/test_queries.py b/tests/test_queries.py index 70fb7a03..59fab8c0 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -4,6 +4,7 @@ import pytest from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import StoredQuery, StoredQueryPage from datasette.utils.asgi import Forbidden @@ -87,38 +88,41 @@ async def test_add_get_and_remove_query(): } query = await ds.get_query("data", "top_customers") - assert query == { - "database": "data", - "name": "top_customers", - "sql": "select * from customers where region = :region", - "title": "Top customers", - "description": "Customers by region", - "description_html": None, - "hide_sql": True, - "fragment": "chart", - "params": ["region"], - "parameters": ["region"], - "is_write": False, - "is_private": False, - "is_trusted": True, - "source": "user", - "owner_id": "alice", - "on_success_message": None, - "on_success_message_sql": None, - "on_success_redirect": None, - "on_error_message": None, - "on_error_redirect": None, - } + assert query == StoredQuery( + database="data", + name="top_customers", + sql="select * from customers where region = :region", + title="Top customers", + description="Customers by region", + description_html=None, + hide_sql=True, + fragment="chart", + parameters=["region"], + is_write=False, + is_private=False, + is_trusted=True, + source="user", + owner_id="alice", + on_success_message=None, + on_success_message_sql=None, + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + ) queries_page = await ds.list_queries("data", actor=None) - assert queries_page["queries"] == [query] - assert queries_page["next"] is None + assert queries_page == StoredQueryPage( + queries=[query], + next=None, + has_more=False, + limit=50, + ) await ds.remove_query("data", "top_customers") assert await ds.get_query("data", "top_customers") is None queries_page = await ds.list_queries("data", actor=None) - assert queries_page["queries"] == [] - assert queries_page["next"] is None + assert queries_page.queries == [] + assert queries_page.next is None @pytest.mark.asyncio @@ -156,13 +160,12 @@ async def test_update_query_only_updates_provided_fields(): ) query = await ds.get_query("data", "redirect") - assert query["title"] == "Updated" - assert query["parameters"] == [] - assert query["params"] == [] - assert query["on_success_redirect"] is None - assert query["sql"] == "select 1" - assert query["is_private"] is False - assert query["is_trusted"] is False + assert query.title == "Updated" + assert query.parameters == [] + assert query.on_success_redirect is None + assert query.sql == "select 1" + assert query.is_private is False + assert query.is_trusted is False options_row = ( await ds.get_internal_database().execute( """ @@ -198,28 +201,27 @@ async def test_config_queries_imported_to_internal_table(): ds.add_memory_database("query_config", name="data") await ds.invoke_startup() - assert await ds.get_query("data", "configured") == { - "database": "data", - "name": "configured", - "sql": "select :name as name", - "title": "Configured query", - "description": None, - "description_html": "

Configured HTML

", - "hide_sql": False, - "fragment": None, - "params": ["name"], - "parameters": ["name"], - "is_write": False, - "is_private": False, - "is_trusted": True, - "source": "config", - "owner_id": None, - "on_success_message": None, - "on_success_message_sql": "select 'Hello ' || :name", - "on_success_redirect": None, - "on_error_message": None, - "on_error_redirect": None, - } + assert await ds.get_query("data", "configured") == StoredQuery( + database="data", + name="configured", + sql="select :name as name", + title="Configured query", + description=None, + description_html="

Configured HTML

", + hide_sql=False, + fragment=None, + parameters=["name"], + is_write=False, + is_private=False, + is_trusted=True, + source="config", + owner_id=None, + on_success_message=None, + on_success_message_sql="select 'Hello ' || :name", + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + ) @pytest.mark.asyncio @@ -1032,8 +1034,8 @@ async def test_query_update_api_rejects_config_only_fields(): "Invalid keys: description_html, on_success_message_sql" ] query = await ds.get_query("data", "editable") - assert query["description_html"] is None - assert query["on_success_message_sql"] is None + assert query.description_html is None + assert query.on_success_message_sql is None @pytest.mark.asyncio @@ -1072,9 +1074,9 @@ async def test_query_update_api_rejects_trusted_queries_but_internal_update_allo "Trusted queries cannot be updated using the API" ] query = await ds.get_query("data", "trusted_report") - assert query["is_trusted"] is True - assert query["sql"] == "select 1 as one" - assert query["title"] == "Original" + assert query.is_trusted is True + assert query.sql == "select 1 as one" + assert query.title == "Original" await ds.update_query( "data", @@ -1083,9 +1085,9 @@ async def test_query_update_api_rejects_trusted_queries_but_internal_update_allo title="Internal", ) query = await ds.get_query("data", "trusted_report") - assert query["is_trusted"] is True - assert query["sql"] == "select 3 as three" - assert query["title"] == "Internal" + assert query.is_trusted is True + assert query.sql == "select 3 as three" + assert query.title == "Internal" @pytest.mark.asyncio From 9f66cf72c1c9170f10e863d750ac4eee47113a7f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 21:42:50 -0700 Subject: [PATCH 742/767] Removed execute write SQL from query create page --- datasette/templates/query_create.html | 7 ------- 1 file changed, 7 deletions(-) diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index f5dadbff..ec910456 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -106,9 +106,6 @@ form.sql .query-create-sql textarea#sql-editor { .query-create-analysis-note { margin: 0; } -.query-create-action { - margin: 0.35rem 0 1rem; -} .query-create-analysis { margin-top: 0.8rem; } @@ -171,10 +168,6 @@ form.sql .query-create-sql textarea#sql-editor { Queries marked private can only be seen by you, their creator.

- {% if sql and analysis_is_write %} -

Execute write SQL

- {% endif %} -

From 737ff03efbb2bdc99b10d2654b7818526ec51e13 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 22:11:06 -0700 Subject: [PATCH 743/767] Expanded analysis of SQL operations, refs #2748 --- datasette/permissions.py | 10 ++ datasette/stored_queries.py | 137 +++++++++++++-- datasette/utils/sql_analysis.py | 289 +++++++++++++++++++++++++++---- datasette/views/execute_write.py | 9 +- datasette/views/query_helpers.py | 104 +++++++---- tests/test_actions_sql.py | 14 +- tests/test_internals_database.py | 34 ++-- tests/test_queries.py | 166 ++++++++++++++++++ tests/test_utils_sql_analysis.py | 97 +++++++++-- 9 files changed, 740 insertions(+), 120 deletions(-) diff --git a/datasette/permissions.py b/datasette/permissions.py index 917c58ab..a9a3cc7c 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -58,6 +58,16 @@ class Resource(ABC): self.child = child self._private = None # Sentinel to track if private was set + def __str__(self) -> str: + return "/".join( + str(part) for part in (self.parent, self.child) if part is not None + ) + + def __repr__(self) -> str: + return "{}(parent={!r}, child={!r})".format( + self.__class__.__name__, self.parent, self.child + ) + @property def private(self) -> bool: """ diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index bcfdfdb4..c4b083e5 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -2,11 +2,16 @@ from __future__ import annotations from dataclasses import dataclass import json -from typing import Any, Iterable +from typing import Any, Iterable, TYPE_CHECKING -from .resources import TableResource +from .resources import DatabaseResource, TableResource +from .permissions import Resource from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components from .utils.asgi import Forbidden +from .utils.sql_analysis import Operation, SQLAnalysis + +if TYPE_CHECKING: + from .app import Datasette UNCHANGED = object() @@ -583,20 +588,94 @@ async def list_queries( ) -async def ensure_query_write_permissions( - datasette: Any, - database: str, - sql: str, - *, - actor: dict[str, Any] | None = None, - params: dict[str, Any] | None = None, - analysis: Any = None, -) -> Any: +PermissionRequirement = tuple[str, Resource] + + +def permission_for_operation(operation: Operation) -> PermissionRequirement | None: write_actions = { "insert": "insert-row", "update": "update-row", "delete": "delete-row", } + action = write_actions.get(operation.operation) + if ( + action + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return ( + action, + TableResource(database=operation.database, table=operation.table), + ) + if operation.operation == "create" and operation.target_type == "table": + if operation.database is None: + return None + return ( + "create-table", + DatabaseResource(database=operation.database), + ) + if ( + operation.operation == "alter" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return ( + "alter-table", + TableResource(database=operation.database, table=operation.table), + ) + if ( + operation.operation == "drop" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return ( + "drop-table", + TableResource(database=operation.database, table=operation.table), + ) + if ( + operation.operation in {"create", "drop"} + and operation.target_type == "index" + and operation.database is not None + and operation.table is not None + ): + return ( + "alter-table", + TableResource(database=operation.database, table=operation.table), + ) + return None + + +def operation_is_write(operation: Operation) -> bool: + return operation.operation in { + "insert", + "update", + "delete", + "create", + "alter", + "drop", + "begin", + "commit", + "rollback", + "attach", + "detach", + "pragma", + "analyze", + "reindex", + } + + +async def ensure_query_write_permissions( + datasette: Datasette, + database: str, + sql: str, + *, + actor: dict[str, object] | None = None, + params: dict[str, object] | None = None, + analysis: SQLAnalysis | None = None, +) -> SQLAnalysis: db = datasette.get_database(database) if analysis is None: if params is None: @@ -606,18 +685,38 @@ async def ensure_query_write_permissions( except sqlite3.DatabaseError as ex: raise Forbidden(f"Could not analyze query: {ex}") from ex - for access in analysis.table_accesses: - action = write_actions.get(access.operation) - if action is None: + has_semantic_schema_operation = any( + operation.operation in {"create", "alter", "drop"} + and operation.target_type in {"table", "index", "view", "trigger"} + for operation in analysis.operations + ) + for operation in analysis.operations: + if operation.internal and has_semantic_schema_operation: continue - if access.database != database: + if has_semantic_schema_operation and operation.operation in { + "read", + "insert", + "update", + "delete", + "reindex", + }: + continue + permission = permission_for_operation(operation) + if permission is None: + if operation_is_write(operation): + raise Forbidden( + "Unsupported SQL operation: {} {}".format( + operation.operation, operation.target_type + ) + ) + continue + action, resource = permission + if operation.database != database: raise Forbidden("Writable queries may not write to attached databases") if not await datasette.allowed( action=action, - resource=TableResource(database=access.database, table=access.table), + resource=resource, actor=actor, ): - raise Forbidden( - f"Permission denied: need {action} on {access.database}/{access.table}" - ) + raise Forbidden(f"Permission denied: need {action} on {resource}") return analysis diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index b5317b62..54f310fe 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -3,22 +3,66 @@ from typing import Literal from datasette.utils.sqlite import sqlite3 +SQLOperation = Literal[ + "read", + "insert", + "update", + "delete", + "create", + "alter", + "drop", + "begin", + "commit", + "rollback", + "attach", + "detach", + "pragma", + "analyze", + "reindex", +] +SQLTargetType = Literal[ + "table", + "index", + "view", + "trigger", + "schema", + "transaction", + "database", + "pragma", + "unknown", +] SQLTableOperation = Literal["read", "insert", "update", "delete"] @dataclass(frozen=True) -class SQLTableAccess: - operation: SQLTableOperation +class Operation: + operation: SQLOperation + target_type: SQLTargetType database: str | None - table: str + table: str | None sqlite_schema: str | None + target: str | None = None columns: tuple[str, ...] = () source: str | None = None + internal: bool = False @dataclass(frozen=True) class SQLAnalysis: - table_accesses: tuple[SQLTableAccess, ...] + operations: tuple[Operation, ...] + + +# Hashable dict key for grouping repeated authorizer callbacks while collecting columns. +@dataclass(frozen=True) +class OperationKey: + operation: SQLOperation + target_type: SQLTargetType + database: str | None + table: str | None + sqlite_schema: str | None + target: str | None + source: str | None + internal: bool _ACTION_TO_OPERATION: dict[int, SQLTableOperation] = { @@ -28,6 +72,36 @@ _ACTION_TO_OPERATION: dict[int, SQLTableOperation] = { sqlite3.SQLITE_DELETE: "delete", } +# Values are (operation, target_type) pairs used to construct Operation objects. +_CREATE_ACTIONS = { + sqlite3.SQLITE_CREATE_INDEX: ("create", "index"), + sqlite3.SQLITE_CREATE_TABLE: ("create", "table"), + sqlite3.SQLITE_CREATE_TRIGGER: ("create", "trigger"), + sqlite3.SQLITE_CREATE_VIEW: ("create", "view"), +} +_DROP_ACTIONS = { + sqlite3.SQLITE_DROP_INDEX: ("drop", "index"), + sqlite3.SQLITE_DROP_TABLE: ("drop", "table"), + sqlite3.SQLITE_DROP_TRIGGER: ("drop", "trigger"), + sqlite3.SQLITE_DROP_VIEW: ("drop", "view"), +} +for action_name, operation, target_type in ( + ("SQLITE_CREATE_TEMP_INDEX", "create", "index"), + ("SQLITE_CREATE_TEMP_TABLE", "create", "table"), + ("SQLITE_CREATE_TEMP_TRIGGER", "create", "trigger"), + ("SQLITE_CREATE_TEMP_VIEW", "create", "view"), + ("SQLITE_DROP_TEMP_INDEX", "drop", "index"), + ("SQLITE_DROP_TEMP_TABLE", "drop", "table"), + ("SQLITE_DROP_TEMP_TRIGGER", "drop", "trigger"), + ("SQLITE_DROP_TEMP_VIEW", "drop", "view"), +): + action_value = getattr(sqlite3, action_name, None) + if action_value is not None: + actions = _CREATE_ACTIONS if operation == "create" else _DROP_ACTIONS + actions[action_value] = (operation, target_type) + +_SQLITE_SCHEMA_TABLES = {"sqlite_master", "sqlite_schema"} + def analyze_sql_tables( conn, @@ -38,15 +112,13 @@ def analyze_sql_tables( schema_to_database: dict[str, str] | None = None, ) -> SQLAnalysis: """ - Return tables accessed by a SQL statement according to SQLite's authorizer. + Return operations performed by a SQL statement according to SQLite's authorizer. This function is synchronous and connection-based. It temporarily installs a - SQLite authorizer, prepares ``EXPLAIN ``, and returns the table access + SQLite authorizer, prepares ``EXPLAIN ``, and returns the operation callbacks observed while SQLite compiles the statement. """ - accesses: dict[ - tuple[SQLTableOperation, str | None, str, str | None, str | None], set[str] - ] = {} + operations: dict[OperationKey, set[str]] = {} def database_for_schema(sqlite_schema): if schema_to_database and sqlite_schema in schema_to_database: @@ -55,21 +127,166 @@ def analyze_sql_tables( return database_name return sqlite_schema + def record( + operation: SQLOperation, + target_type: SQLTargetType, + *, + database: str | None, + table: str | None, + sqlite_schema: str | None, + target: str | None, + source: str | None, + column: str | None = None, + internal: bool = False, + ): + key = OperationKey( + operation=operation, + target_type=target_type, + database=database, + table=table, + sqlite_schema=sqlite_schema, + target=target, + source=source, + internal=internal, + ) + columns = operations.setdefault(key, set()) + if column is not None: + columns.add(column) + def authorizer(action, arg1, arg2, sqlite_schema, source): operation = _ACTION_TO_OPERATION.get(action) - if operation is None or arg1 is None: + if operation is not None and arg1 is not None: + target_type = "schema" if arg1 in _SQLITE_SCHEMA_TABLES else "table" + column = ( + arg2 if operation in ("read", "update") and arg2 is not None else None + ) + record( + operation, + target_type, + database=database_for_schema(sqlite_schema), + table=arg1 if target_type == "table" else None, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + column=column, + internal=target_type == "schema", + ) + return sqlite3.SQLITE_OK + + create_operation = _CREATE_ACTIONS.get(action) + if create_operation is not None and arg1 is not None: + operation, target_type = create_operation + related_table = arg2 if target_type in {"index", "trigger"} else arg1 + record( + operation, + target_type, + database=database_for_schema(sqlite_schema), + table=related_table, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + drop_operation = _DROP_ACTIONS.get(action) + if drop_operation is not None and arg1 is not None: + operation, target_type = drop_operation + related_table = arg2 if target_type in {"index", "trigger"} else arg1 + record( + operation, + target_type, + database=database_for_schema(sqlite_schema), + table=related_table, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_ALTER_TABLE and arg2 is not None: + record( + "alter", + "table", + database=database_for_schema(arg1), + table=arg2, + sqlite_schema=arg1, + target=arg2, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_TRANSACTION and arg1 is not None: + record( + arg1.lower(), + "transaction", + database=None, + table=None, + sqlite_schema=None, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_ATTACH and arg1 is not None: + record( + "attach", + "database", + database=None, + table=None, + sqlite_schema=None, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_DETACH and arg1 is not None: + record( + "detach", + "database", + database=None, + table=None, + sqlite_schema=None, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_PRAGMA and arg1 is not None: + record( + "pragma", + "pragma", + database=None, + table=None, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_ANALYZE: + record( + "analyze", + "database" if arg1 is None else "table", + database=database_for_schema(sqlite_schema), + table=arg1, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_REINDEX and arg1 is not None: + record( + "reindex", + "index", + database=database_for_schema(sqlite_schema), + table=None, + sqlite_schema=sqlite_schema, + target=arg1, + source=source, + ) return sqlite3.SQLITE_OK - key = ( - operation, - database_for_schema(sqlite_schema), - arg1, - sqlite_schema, - source, - ) - columns = accesses.setdefault(key, set()) - if operation in ("read", "update") and arg2 is not None: - columns.add(arg2) return sqlite3.SQLITE_OK conn.set_authorizer(authorizer) @@ -78,22 +295,26 @@ def analyze_sql_tables( finally: conn.set_authorizer(None) + has_schema_operation = any( + key.target_type in {"table", "index", "view", "trigger"} + and key.operation in {"create", "alter", "drop"} + for key in operations + ) + return SQLAnalysis( - table_accesses=tuple( - SQLTableAccess( - operation=operation, - database=database, - table=table, - sqlite_schema=sqlite_schema, + operations=tuple( + Operation( + operation=key.operation, + target_type=key.target_type, + database=key.database, + table=key.table, + sqlite_schema=key.sqlite_schema, + target=key.target, columns=tuple(sorted(columns)), - source=source, + source=key.source, + internal=key.internal + or (has_schema_operation and key.target_type == "schema"), ) - for ( - operation, - database, - table, - sqlite_schema, - source, - ), columns in accesses.items() + for key, columns in operations.items() ) ) diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index 0054300c..cead8926 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -193,9 +193,12 @@ class ExecuteWriteView(BaseView): status=400, ) - message = "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" - ) + if cursor.rowcount == -1: + message = "Query executed" + else: + message = "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) if _wants_json(request, is_json, data): return _block_framing( Response.json( diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 46d71b8e..922f4e52 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -1,8 +1,12 @@ import json import re -from datasette.resources import DatabaseResource, TableResource -from datasette.stored_queries import StoredQuery +from datasette.resources import DatabaseResource +from datasette.stored_queries import ( + StoredQuery, + operation_is_write, + permission_for_operation, +) from datasette.utils import ( named_parameters as derive_named_parameters, escape_sqlite, @@ -12,6 +16,7 @@ from datasette.utils import ( InvalidSql, ) from datasette.utils.asgi import Forbidden +from datasette.utils.sql_analysis import Operation, SQLAnalysis _query_name_re = re.compile(r"^[^/\.\n]+$") @@ -123,11 +128,8 @@ def _coerce_query_parameters(value, derived): return parameters -def _analysis_is_write(analysis): - return any( - access.operation in {"insert", "update", "delete"} - for access in analysis.table_accesses - ) +def _analysis_is_write(analysis: SQLAnalysis) -> bool: + return any(operation_is_write(operation) for operation in analysis.operations) def _block_framing(response): @@ -201,34 +203,66 @@ async def _analyze_user_query(datasette, db, sql, *, actor): return is_write, derived, analysis -def _analysis_rows(analysis): - write_actions = { - "insert": "insert-row", - "update": "update-row", - "delete": "delete-row", - } - return [ - { - "operation": access.operation, - "database": access.database, - "table": access.table, - "required_permission": write_actions.get(access.operation, ""), - "source": access.source, - } - for access in analysis.table_accesses - ] +def _semantic_schema_operation_is_present(operations: tuple[Operation, ...]) -> bool: + return any( + operation.operation in {"create", "alter", "drop"} + and operation.target_type in {"table", "index", "view", "trigger"} + for operation in operations + ) -async def _analysis_rows_with_permissions(datasette, analysis, actor): +def _display_operations(analysis: SQLAnalysis) -> list[Operation]: + has_semantic_schema_operation = _semantic_schema_operation_is_present( + analysis.operations + ) + operations = [] + for operation in analysis.operations: + if operation.internal and has_semantic_schema_operation: + continue + if has_semantic_schema_operation and operation.operation in { + "read", + "insert", + "update", + "delete", + "reindex", + }: + continue + operations.append(operation) + return operations + + +def _analysis_rows(analysis: SQLAnalysis) -> list[dict[str, object]]: + rows = [] + for operation in _display_operations(analysis): + permission = permission_for_operation(operation) + required_permission = permission[0] if permission else "" + rows.append( + { + "operation": operation.operation, + "database": operation.database, + "table": operation.table or operation.target, + "required_permission": required_permission, + "source": operation.source, + } + ) + return rows + + +async def _analysis_rows_with_permissions( + datasette, analysis: SQLAnalysis, actor +) -> list[dict[str, object]]: rows = _analysis_rows(analysis) - for row in rows: - permission = row["required_permission"] + for row, operation in zip(rows, _display_operations(analysis)): + permission = permission_for_operation(operation) if permission: + action, resource = permission row["allowed"] = await datasette.allowed( - action=permission, - resource=TableResource(row["database"], row["table"]), + action=action, + resource=resource, actor=actor, ) + elif operation_is_write(operation): + row["allowed"] = False else: row["allowed"] = None return rows @@ -398,15 +432,19 @@ async def _inserted_row_url(datasette, db, analysis, cursor): if lastrowid is None: return None direct_inserts = [ - access - for access in analysis.table_accesses - if access.operation == "insert" - and access.source is None - and access.database == db.name + operation + for operation in analysis.operations + if operation.operation == "insert" + and operation.target_type == "table" + and not operation.internal + and operation.source is None + and operation.database == db.name ] if len(direct_inserts) != 1: return None table = direct_inserts[0].table + if table is None: + return None pks = await db.primary_keys(table) use_rowid = not pks select = ( diff --git a/tests/test_actions_sql.py b/tests/test_actions_sql.py index 863d2529..a1fca971 100644 --- a/tests/test_actions_sql.py +++ b/tests/test_actions_sql.py @@ -12,10 +12,22 @@ import pytest import pytest_asyncio from datasette.app import Datasette from datasette.permissions import PermissionSQL -from datasette.resources import TableResource +from datasette.resources import DatabaseResource, QueryResource, TableResource from datasette import hookimpl +def test_resource_string_representations(): + assert str(DatabaseResource("content")) == "content" + assert repr(DatabaseResource("content")) == ( + "DatabaseResource(parent='content', child=None)" + ) + assert str(TableResource("content", "dogs")) == "content/dogs" + assert repr(TableResource("content", "dogs")) == ( + "TableResource(parent='content', child='dogs')" + ) + assert str(QueryResource("content", "insert-a-dog")) == "content/insert-a-dog" + + # Test plugin that provides permission rules class PermissionRulesPlugin: def __init__(self, rules_callback): diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 5481a398..d6e130b4 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -698,14 +698,17 @@ async def test_analyze_sql(): assert [ ( - access.operation, - access.database, - access.sqlite_schema, - access.table, - access.columns, - access.source, + operation.operation, + operation.database, + operation.sqlite_schema, + operation.table, + operation.columns, + operation.source, ) - for access in analysis.table_accesses + for operation in analysis.operations + if operation.target_type == "table" + and operation.operation in {"read", "insert", "update", "delete"} + and not operation.internal ] == [ ("read", "data", "main", "dogs", ("id", "name"), None), ] @@ -722,14 +725,17 @@ async def test_analyze_sql_insert_select(): assert { ( - access.operation, - access.database, - access.sqlite_schema, - access.table, - access.columns, - access.source, + operation.operation, + operation.database, + operation.sqlite_schema, + operation.table, + operation.columns, + operation.source, ) - for access in analysis.table_accesses + for operation in analysis.operations + if operation.target_type == "table" + and operation.operation in {"read", "insert", "update", "delete"} + and not operation.internal } == { ("insert", "data", "main", "dogs", (), None), ("read", "data", "main", "cats", ("name",), None), diff --git a/tests/test_queries.py b/tests/test_queries.py index 59fab8c0..4b8a6486 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1643,6 +1643,172 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.asyncio +async def test_execute_write_create_table_uses_create_table_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "permissions": { + "insert-row": {"id": "row-writer"}, + "update-row": {"id": "row-writer"}, + }, + "databases": { + "data": { + "permissions": { + "view-database": {"id": ["creator", "row-writer"]}, + "execute-write-sql": {"id": ["creator", "row-writer"]}, + "create-table": {"id": "creator"}, + } + } + }, + }, + ) + db = ds.add_memory_database("execute_write_create_table", name="data") + await ds.invoke_startup() + + analysis_response = await ds.client.get( + "/data/-/execute-write/analyze", + actor={"id": "creator"}, + params={"sql": "create table foobar (id integer primary key, name text)"}, + ) + allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "creator"}, + json={"sql": "create table foobar (id integer primary key, name text)"}, + ) + row_permission_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "row-writer"}, + json={"sql": "create table should_not_exist (id integer primary key)"}, + ) + + assert analysis_response.status_code == 200 + analysis_data = analysis_response.json() + assert analysis_data["ok"] is True + assert analysis_data["execute_disabled"] is False + assert analysis_data["analysis_rows"] == [ + { + "operation": "create", + "database": "data", + "table": "foobar", + "required_permission": "create-table", + "source": None, + "allowed": True, + } + ] + + assert allowed_response.status_code == 200 + assert allowed_response.json()["ok"] is True + assert allowed_response.json()["message"] == "Query executed" + assert await db.table_exists("foobar") + + assert row_permission_response.status_code == 403 + assert row_permission_response.json()["errors"] == [ + "Permission denied: need create-table on data" + ] + assert not await db.table_exists("should_not_exist") + + +@pytest.mark.asyncio +async def test_execute_write_alter_and_drop_table_use_schema_permissions(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "permissions": { + "delete-row": {"id": "row-writer"}, + "update-row": {"id": "row-writer"}, + }, + "databases": { + "data": { + "permissions": { + "view-database": {"id": ["alterer", "dropper", "row-writer"]}, + "execute-write-sql": { + "id": ["alterer", "dropper", "row-writer"] + }, + }, + "tables": { + "dogs": { + "permissions": { + "alter-table": {"id": "alterer"}, + "drop-table": {"id": "dropper"}, + } + } + }, + } + }, + }, + ) + db = ds.add_memory_database("execute_write_alter_drop_table", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write("create table cats (id integer primary key, name text)") + await ds.invoke_startup() + + alter_allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "alterer"}, + json={"sql": "alter table dogs add column age integer"}, + ) + alter_row_permission_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "row-writer"}, + json={"sql": "alter table cats add column age integer"}, + ) + + assert alter_allowed_response.status_code == 200 + assert "age" in [column.name for column in await db.table_column_details("dogs")] + assert alter_row_permission_response.status_code == 403 + assert alter_row_permission_response.json()["errors"] == [ + "Permission denied: need alter-table on data/cats" + ] + assert "age" not in [ + column.name for column in await db.table_column_details("cats") + ] + + create_index_allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "alterer"}, + json={"sql": "create index idx_dogs_name on dogs(name)"}, + ) + create_index_row_permission_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "row-writer"}, + json={"sql": "create index idx_cats_name on cats(name)"}, + ) + drop_index_allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "alterer"}, + json={"sql": "drop index idx_dogs_name"}, + ) + + assert create_index_allowed_response.status_code == 200 + assert create_index_row_permission_response.status_code == 403 + assert create_index_row_permission_response.json()["errors"] == [ + "Permission denied: need alter-table on data/cats" + ] + assert drop_index_allowed_response.status_code == 200 + + drop_allowed_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "dropper"}, + json={"sql": "drop table dogs"}, + ) + drop_row_permission_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "row-writer"}, + json={"sql": "drop table cats"}, + ) + + assert drop_allowed_response.status_code == 200 + assert not await db.table_exists("dogs") + assert drop_row_permission_response.status_code == 403 + assert drop_row_permission_response.json()["errors"] == [ + "Permission denied: need drop-table on data/cats" + ] + assert await db.table_exists("cats") + + @pytest.mark.asyncio async def test_execute_write_insert_links_to_inserted_row(): ds = Datasette(memory=True, default_deny=True) diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index 5730cd0d..5306a515 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -26,17 +26,20 @@ def conn(): conn.close() -def as_tuples(analysis): +def table_operation_tuples(analysis): return [ ( - access.operation, - access.database, - access.sqlite_schema, - access.table, - access.columns, - access.source, + operation.operation, + operation.database, + operation.sqlite_schema, + operation.table, + operation.columns, + operation.source, ) - for access in analysis.table_accesses + for operation in analysis.operations + if operation.target_type == "table" + and operation.operation in {"read", "insert", "update", "delete"} + and not operation.internal ] @@ -48,7 +51,7 @@ def test_analyze_select_tables(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("read", "data", "main", "cats", ("id", "name"), None), ("read", "data", "main", "dogs", ("age", "id", "name"), None), } @@ -57,11 +60,73 @@ def test_analyze_select_tables(conn): def test_analyze_uses_sqlite_schema_as_default_database(conn): analysis = analyze_sql_tables(conn, "select name from dogs") - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("read", "main", "main", "dogs", ("name",), None), } +def operation_dict(operation): + return { + "operation": operation.operation, + "target_type": operation.target_type, + "database": operation.database, + "sqlite_schema": operation.sqlite_schema, + "table": operation.table, + "target": operation.target, + "columns": operation.columns, + "source": operation.source, + "internal": operation.internal, + } + + +def test_analyze_create_table_operation(): + conn = sqlite3.connect(":memory:") + try: + analysis = analyze_sql_tables( + conn, + "create table foobar (id integer primary key, name text)", + database_name="data", + ) + finally: + conn.close() + + assert { + "operation": "create", + "target_type": "table", + "database": "data", + "sqlite_schema": "main", + "table": "foobar", + "target": "foobar", + "columns": (), + "source": None, + "internal": False, + } in [operation_dict(operation) for operation in analysis.operations] + assert not [ + operation + for operation in analysis.operations + if operation.table in {"sqlite_master", "sqlite_schema"} + and not operation.internal + ] + + +def test_analyze_transaction_operation(conn): + analysis = analyze_sql_tables(conn, "commit", database_name="data") + + assert [operation_dict(operation) for operation in analysis.operations] == [ + { + "operation": "commit", + "target_type": "transaction", + "database": None, + "sqlite_schema": None, + "table": None, + "target": "COMMIT", + "columns": (), + "source": None, + "internal": False, + } + ] + + def test_analyze_insert_tables(conn): analysis = analyze_sql_tables( conn, @@ -70,7 +135,7 @@ def test_analyze_insert_tables(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("insert", "data", "main", "dogs", (), None), ("read", "data", "main", "dogs", ("id", "name"), "dogs_after_insert"), ("update", "data", "main", "cats", ("name",), "dogs_after_insert"), @@ -87,7 +152,7 @@ def test_analyze_update_tables(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("update", "data", "main", "dogs", ("age",), None), ("read", "data", "main", "dogs", ("age", "name"), None), } @@ -101,7 +166,7 @@ def test_analyze_delete_tables(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("delete", "data", "main", "dogs", (), None), ("read", "data", "main", "dogs", ("name",), None), } @@ -121,7 +186,7 @@ def test_analyze_insert_select_with_cte(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("insert", "data", "main", "cats", (), None), ("read", "data", "main", "dogs", ("age", "name"), "old_dogs"), } @@ -135,7 +200,7 @@ def test_analyze_view_with_instead_of_trigger(conn): database_name="data", ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("update", "data", "main", "dog_names", ("name",), None), ("read", "data", "main", "dogs", ("id", "name"), "dog_names"), ("read", "data", "main", "dog_names", ("id", "name"), "dog_names"), @@ -163,7 +228,7 @@ def test_analyze_attached_database_tables(conn): schema_to_database={"extra": "extra_db"}, ) - assert set(as_tuples(analysis)) == { + assert set(table_operation_tuples(analysis)) == { ("insert", "extra_db", "extra", "people", (), None), ("read", "data", "main", "dogs", ("name",), None), } From 86d0e7335f98a88874df31ec0adb64967446dfac Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 14:52:52 -0700 Subject: [PATCH 744/767] Deny unsupported write SQL operations by default Require view-table permission for reads discovered inside write SQL analysis, including INSERT ... SELECT and CREATE TABLE ... AS SELECT. Record additional SQLite authorizer callbacks as Operation values so unsupported functions, savepoints, virtual table DDL, and unknown callbacks are denied unless explicitly handled. --- datasette/stored_queries.py | 43 +++---- datasette/utils/sql_analysis.py | 192 +++++++++++++++++++++++++++++-- datasette/views/execute_write.py | 4 +- datasette/views/query_helpers.py | 32 ++---- tests/test_queries.py | 136 ++++++++++++++++++++-- tests/test_utils_sql_analysis.py | 94 +++++++++++++++ 6 files changed, 433 insertions(+), 68 deletions(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index c4b083e5..4b0fe6a6 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -592,6 +592,16 @@ PermissionRequirement = tuple[str, Resource] def permission_for_operation(operation: Operation) -> PermissionRequirement | None: + if ( + operation.operation == "read" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return ( + "view-table", + TableResource(database=operation.database, table=operation.table), + ) write_actions = { "insert": "insert-row", "update": "update-row", @@ -648,6 +658,10 @@ def permission_for_operation(operation: Operation) -> PermissionRequirement | No return None +def operation_should_be_ignored(operation: Operation) -> bool: + return operation.internal or operation.operation == "select" + + def operation_is_write(operation: Operation) -> bool: return operation.operation in { "insert", @@ -659,11 +673,13 @@ def operation_is_write(operation: Operation) -> bool: "begin", "commit", "rollback", + "savepoint", "attach", "detach", "pragma", "analyze", "reindex", + "unknown", } @@ -685,34 +701,19 @@ async def ensure_query_write_permissions( except sqlite3.DatabaseError as ex: raise Forbidden(f"Could not analyze query: {ex}") from ex - has_semantic_schema_operation = any( - operation.operation in {"create", "alter", "drop"} - and operation.target_type in {"table", "index", "view", "trigger"} - for operation in analysis.operations - ) for operation in analysis.operations: - if operation.internal and has_semantic_schema_operation: - continue - if has_semantic_schema_operation and operation.operation in { - "read", - "insert", - "update", - "delete", - "reindex", - }: + if operation_should_be_ignored(operation): continue permission = permission_for_operation(operation) if permission is None: - if operation_is_write(operation): - raise Forbidden( - "Unsupported SQL operation: {} {}".format( - operation.operation, operation.target_type - ) + raise Forbidden( + "Unsupported SQL operation: {} {}".format( + operation.operation, operation.target_type ) - continue + ) action, resource = permission if operation.database != database: - raise Forbidden("Writable queries may not write to attached databases") + raise Forbidden("Writable queries may not access attached databases") if not await datasette.allowed( action=action, resource=resource, diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 54f310fe..8963da77 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -8,30 +8,39 @@ SQLOperation = Literal[ "insert", "update", "delete", + "select", + "function", "create", "alter", "drop", "begin", "commit", "rollback", + "savepoint", "attach", "detach", "pragma", "analyze", "reindex", + "unknown", ] SQLTargetType = Literal[ "table", "index", "view", "trigger", + "virtual-table", "schema", + "statement", "transaction", "database", "pragma", + "function", "unknown", ] SQLTableOperation = Literal["read", "insert", "update", "delete"] +SQLSchemaOperation = Literal["create", "drop"] +SQLSchemaTargetType = Literal["index", "table", "trigger", "view", "virtual-table"] @dataclass(frozen=True) @@ -73,19 +82,34 @@ _ACTION_TO_OPERATION: dict[int, SQLTableOperation] = { } # Values are (operation, target_type) pairs used to construct Operation objects. -_CREATE_ACTIONS = { +_CREATE_ACTIONS: dict[int, tuple[SQLSchemaOperation, SQLSchemaTargetType]] = { sqlite3.SQLITE_CREATE_INDEX: ("create", "index"), sqlite3.SQLITE_CREATE_TABLE: ("create", "table"), sqlite3.SQLITE_CREATE_TRIGGER: ("create", "trigger"), sqlite3.SQLITE_CREATE_VIEW: ("create", "view"), } -_DROP_ACTIONS = { +_DROP_ACTIONS: dict[int, tuple[SQLSchemaOperation, SQLSchemaTargetType]] = { sqlite3.SQLITE_DROP_INDEX: ("drop", "index"), sqlite3.SQLITE_DROP_TABLE: ("drop", "table"), sqlite3.SQLITE_DROP_TRIGGER: ("drop", "trigger"), sqlite3.SQLITE_DROP_VIEW: ("drop", "view"), } -for action_name, operation, target_type in ( + + +def _add_schema_action( + action_name: str, + operation: SQLSchemaOperation, + target_type: SQLSchemaTargetType, +) -> None: + action_value = getattr(sqlite3, action_name, None) + if action_value is not None: + actions = _CREATE_ACTIONS if operation == "create" else _DROP_ACTIONS + actions[action_value] = (operation, target_type) + + +_TEMP_SCHEMA_ACTIONS: tuple[ + tuple[str, SQLSchemaOperation, SQLSchemaTargetType], ... +] = ( ("SQLITE_CREATE_TEMP_INDEX", "create", "index"), ("SQLITE_CREATE_TEMP_TABLE", "create", "table"), ("SQLITE_CREATE_TEMP_TRIGGER", "create", "trigger"), @@ -94,13 +118,76 @@ for action_name, operation, target_type in ( ("SQLITE_DROP_TEMP_TABLE", "drop", "table"), ("SQLITE_DROP_TEMP_TRIGGER", "drop", "trigger"), ("SQLITE_DROP_TEMP_VIEW", "drop", "view"), -): - action_value = getattr(sqlite3, action_name, None) - if action_value is not None: - actions = _CREATE_ACTIONS if operation == "create" else _DROP_ACTIONS - actions[action_value] = (operation, target_type) +) +for schema_action in _TEMP_SCHEMA_ACTIONS: + _add_schema_action(*schema_action) -_SQLITE_SCHEMA_TABLES = {"sqlite_master", "sqlite_schema"} +_VTABLE_SCHEMA_ACTIONS: tuple[ + tuple[str, SQLSchemaOperation, SQLSchemaTargetType], ... +] = ( + ("SQLITE_CREATE_VTABLE", "create", "virtual-table"), + ("SQLITE_DROP_VTABLE", "drop", "virtual-table"), +) +for schema_action in _VTABLE_SCHEMA_ACTIONS: + _add_schema_action(*schema_action) + +_SQLITE_SCHEMA_TABLES = { + "sqlite_master", + "sqlite_schema", + "sqlite_temp_master", + "sqlite_temp_schema", +} +_SQLITE_INTERNAL_SCHEMA_FUNCTIONS = { + "length", + "like", + "printf", + "sqlite_drop_column", + "sqlite_rename_column", + "sqlite_rename_quotefix", + "sqlite_rename_table", + "sqlite_rename_test", + "substr", +} + +_AUTHORIZER_ACTION_NAMES = { + getattr(sqlite3, name): name + for name in ( + "SQLITE_CREATE_INDEX", + "SQLITE_CREATE_TABLE", + "SQLITE_CREATE_TEMP_INDEX", + "SQLITE_CREATE_TEMP_TABLE", + "SQLITE_CREATE_TEMP_TRIGGER", + "SQLITE_CREATE_TEMP_VIEW", + "SQLITE_CREATE_TRIGGER", + "SQLITE_CREATE_VIEW", + "SQLITE_DELETE", + "SQLITE_DROP_INDEX", + "SQLITE_DROP_TABLE", + "SQLITE_DROP_TEMP_INDEX", + "SQLITE_DROP_TEMP_TABLE", + "SQLITE_DROP_TEMP_TRIGGER", + "SQLITE_DROP_TEMP_VIEW", + "SQLITE_DROP_TRIGGER", + "SQLITE_DROP_VIEW", + "SQLITE_INSERT", + "SQLITE_PRAGMA", + "SQLITE_READ", + "SQLITE_SELECT", + "SQLITE_TRANSACTION", + "SQLITE_UPDATE", + "SQLITE_ATTACH", + "SQLITE_DETACH", + "SQLITE_ALTER_TABLE", + "SQLITE_REINDEX", + "SQLITE_ANALYZE", + "SQLITE_CREATE_VTABLE", + "SQLITE_DROP_VTABLE", + "SQLITE_FUNCTION", + "SQLITE_SAVEPOINT", + "SQLITE_RECURSIVE", + ) + if hasattr(sqlite3, name) +} def analyze_sql_tables( @@ -287,6 +374,52 @@ def analyze_sql_tables( ) return sqlite3.SQLITE_OK + if action == sqlite3.SQLITE_SELECT: + record( + "select", + "statement", + database=None, + table=None, + sqlite_schema=sqlite_schema, + target=None, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_FUNCTION and arg2 is not None: + record( + "function", + "function", + database=None, + table=None, + sqlite_schema=sqlite_schema, + target=arg2, + source=source, + ) + return sqlite3.SQLITE_OK + + if action == sqlite3.SQLITE_SAVEPOINT and arg1 is not None: + record( + "savepoint", + "transaction", + database=None, + table=None, + sqlite_schema=sqlite_schema, + target="{} {}".format(arg1, arg2) if arg2 is not None else arg1, + source=source, + ) + return sqlite3.SQLITE_OK + + action_name = _AUTHORIZER_ACTION_NAMES.get(action, "SQLITE_{}".format(action)) + record( + "unknown", + "unknown", + database=database_for_schema(sqlite_schema), + table=None, + sqlite_schema=sqlite_schema, + target=action_name, + source=source, + ) return sqlite3.SQLITE_OK conn.set_authorizer(authorizer) @@ -296,10 +429,46 @@ def analyze_sql_tables( conn.set_authorizer(None) has_schema_operation = any( - key.target_type in {"table", "index", "view", "trigger"} + key.target_type in {"table", "index", "view", "trigger", "virtual-table"} and key.operation in {"create", "alter", "drop"} for key in operations ) + dropped_tables = { + (key.database, key.table) + for key in operations + if key.operation == "drop" and key.target_type == "table" + } + + def key_is_drop_table_delete(key: OperationKey) -> bool: + return ( + key.operation == "delete" + and key.target_type == "table" + and (key.database, key.table) in dropped_tables + ) + + has_user_table_access_in_schema_operation = any( + key.operation in {"read", "insert", "update", "delete"} + and key.target_type == "table" + and not key.internal + and not key_is_drop_table_delete(key) + for key in operations + ) + + def operation_is_internal(key: OperationKey) -> bool: + if key.internal or (has_schema_operation and key.target_type == "schema"): + return True + if has_schema_operation and key.operation == "reindex": + return True + if ( + has_schema_operation + and not has_user_table_access_in_schema_operation + and key.operation == "function" + and key.target in _SQLITE_INTERNAL_SCHEMA_FUNCTIONS + ): + return True + if key_is_drop_table_delete(key): + return True + return False return SQLAnalysis( operations=tuple( @@ -312,8 +481,7 @@ def analyze_sql_tables( target=key.target, columns=tuple(sorted(columns)), source=key.source, - internal=key.internal - or (has_schema_operation and key.target_type == "schema"), + internal=operation_is_internal(key), ) for key, columns in operations.items() ) diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index cead8926..19006ac5 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -99,9 +99,7 @@ class ExecuteWriteView(BaseView): "parameter_names": parameter_names, "parameter_values": parameter_values, "analysis_error": analysis_error, - "analysis_rows": [ - row for row in analysis_rows if row["operation"] != "read" - ], + "analysis_rows": analysis_rows, "execution_message": execution_message, "execution_links": execution_links, "execution_ok": execution_ok, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 922f4e52..05a0d73e 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -5,6 +5,7 @@ from datasette.resources import DatabaseResource from datasette.stored_queries import ( StoredQuery, operation_is_write, + operation_should_be_ignored, permission_for_operation, ) from datasette.utils import ( @@ -203,29 +204,10 @@ async def _analyze_user_query(datasette, db, sql, *, actor): return is_write, derived, analysis -def _semantic_schema_operation_is_present(operations: tuple[Operation, ...]) -> bool: - return any( - operation.operation in {"create", "alter", "drop"} - and operation.target_type in {"table", "index", "view", "trigger"} - for operation in operations - ) - - def _display_operations(analysis: SQLAnalysis) -> list[Operation]: - has_semantic_schema_operation = _semantic_schema_operation_is_present( - analysis.operations - ) operations = [] for operation in analysis.operations: - if operation.internal and has_semantic_schema_operation: - continue - if has_semantic_schema_operation and operation.operation in { - "read", - "insert", - "update", - "delete", - "reindex", - }: + if operation_should_be_ignored(operation): continue operations.append(operation) return operations @@ -252,6 +234,7 @@ async def _analysis_rows_with_permissions( datasette, analysis: SQLAnalysis, actor ) -> list[dict[str, object]]: rows = _analysis_rows(analysis) + is_write = _analysis_is_write(analysis) for row, operation in zip(rows, _display_operations(analysis)): permission = permission_for_operation(operation) if permission: @@ -261,7 +244,7 @@ async def _analysis_rows_with_permissions( resource=resource, actor=actor, ) - elif operation_is_write(operation): + elif is_write: row["allowed"] = False else: row["allowed"] = None @@ -360,7 +343,7 @@ async def _execute_write_analysis_data(datasette, db, sql, actor): "ok": analysis_error is None, "parameters": parameter_names, "analysis_error": analysis_error, - "analysis_rows": [row for row in analysis_rows if row["operation"] != "read"], + "analysis_rows": analysis_rows, "execute_disabled": bool( (not sql) or analysis_error @@ -374,6 +357,7 @@ async def _query_create_analysis_data(datasette, db, sql, actor): parameter_names = [] analysis_rows = [] analysis_error = None + analysis: SQLAnalysis | None = None if has_sql: try: parameter_names = _derived_query_parameters(sql) @@ -390,9 +374,7 @@ async def _query_create_analysis_data(datasette, db, sql, actor): "analysis_error": analysis_error, "analysis_rows": analysis_rows, "has_sql": has_sql, - "analysis_is_write": bool( - analysis_rows and any(row["required_permission"] for row in analysis_rows) - ), + "analysis_is_write": _analysis_is_write(analysis) if analysis else False, "save_disabled": bool( (not has_sql) or analysis_error diff --git a/tests/test_queries.py b/tests/test_queries.py index 4b8a6486..97ec973f 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1181,11 +1181,10 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): assert 'Required permission' in create_response.text assert 'Source' not in create_response.text assert "read" in create_response.text + assert "view-table" in create_response.text assert ( - create_response.text.count( - 'n/a' - ) - == 2 + 'n/a' + not in create_response.text ) assert create_response.text.index( 'value="Save query"' @@ -1255,9 +1254,9 @@ async def test_create_query_analyze_endpoint_uses_sql_only(): "operation": "read", "database": "data", "table": "dogs", - "required_permission": "", + "required_permission": "view-table", "source": None, - "allowed": None, + "allowed": True, } ] @@ -1375,7 +1374,8 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'Required permission' in response.text assert "insert" in response.text assert "update" in response.text - assert "read" not in response.text + assert "read" in response.text + assert "view-table" in response.text assert 'action="/data/-/execute-write"' in response.text assert "insert into dogs (name) values ('Cleo')" in response.text assert (await db.execute("select count(*) from dogs")).first()[0] == 0 @@ -1643,6 +1643,127 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.asyncio +async def test_execute_write_insert_select_requires_view_table_on_source(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "secret": { + "permissions": {"view-table": {"id": "someone-else"}} + }, + "public_log": {"permissions": {"insert-row": {"id": "writer"}}}, + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_insert_select_source", name="data") + await db.execute_write("create table secret (value text)") + await db.execute_write("create table public_log (value text)") + await db.execute_write("insert into secret values ('sensitive')") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={"sql": "insert into public_log(value) select value from secret"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Permission denied: need view-table on data/secret" + ] + assert (await db.execute("select value from public_log")).dicts() == [] + + +@pytest.mark.asyncio +async def test_execute_write_create_table_as_select_requires_view_table_on_source(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "creator"}, + "execute-write-sql": {"id": "creator"}, + "create-table": {"id": "creator"}, + }, + "tables": { + "secret": { + "permissions": {"view-table": {"id": "someone-else"}} + } + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_create_as_select_source", name="data") + await db.execute_write("create table secret (value text)") + await db.execute_write("insert into secret values ('sensitive')") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "creator"}, + json={"sql": "create table copied_secret as select value from secret"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Permission denied: need view-table on data/secret" + ] + assert not await db.table_exists("copied_secret") + + +@pytest.mark.asyncio +async def test_execute_write_rejects_function_operations(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "dogs": { + "permissions": { + "insert-row": {"id": "writer"}, + } + } + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_function_operation", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={"sql": "insert into dogs (name) values (upper('cleo'))"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Unsupported SQL operation: function function" + ] + assert (await db.execute("select name from dogs")).dicts() == [] + + @pytest.mark.asyncio async def test_execute_write_create_table_uses_create_table_permission(): ds = Datasette( @@ -1733,6 +1854,7 @@ async def test_execute_write_alter_and_drop_table_use_schema_permissions(): "permissions": { "alter-table": {"id": "alterer"}, "drop-table": {"id": "dropper"}, + "view-table": {"id": "alterer"}, } } }, diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index 5306a515..2ae11502 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -127,6 +127,100 @@ def test_analyze_transaction_operation(conn): ] +def test_analyze_savepoint_operation(conn): + analysis = analyze_sql_tables(conn, "savepoint s", database_name="data") + + assert [operation_dict(operation) for operation in analysis.operations] == [ + { + "operation": "savepoint", + "target_type": "transaction", + "database": None, + "sqlite_schema": None, + "table": None, + "target": "BEGIN s", + "columns": (), + "source": None, + "internal": False, + } + ] + + +def test_analyze_function_operation(conn): + analysis = analyze_sql_tables( + conn, + "insert into dogs (name) values (upper(:name))", + {"name": "Cleo"}, + database_name="data", + ) + + assert { + ( + operation.operation, + operation.target_type, + operation.target, + operation.database, + operation.table, + ) + for operation in analysis.operations + } == { + ("insert", "table", "dogs", "data", "dogs"), + ("function", "function", "upper", None, None), + ("read", "table", "dogs", "data", "dogs"), + ("update", "table", "cats", "data", "cats"), + ("read", "table", "cats", "data", "cats"), + ("insert", "table", "log", "data", "log"), + } + + +def test_analyze_create_virtual_table_operation(): + conn = sqlite3.connect(":memory:") + try: + analysis = analyze_sql_tables( + conn, + "create virtual table docs using fts5(body)", + database_name="data", + ) + finally: + conn.close() + + assert { + "operation": "create", + "target_type": "virtual-table", + "database": "data", + "sqlite_schema": "main", + "table": "docs", + "target": "docs", + "columns": (), + "source": None, + "internal": False, + } in [operation_dict(operation) for operation in analysis.operations] + + +def test_analyze_create_table_as_select_function_is_not_internal(): + conn = sqlite3.connect(":memory:") + try: + conn.execute("create table secret(value text)") + analysis = analyze_sql_tables( + conn, + "create table copied as select substr(value, 1, 1) from secret", + database_name="data", + ) + finally: + conn.close() + + assert { + "operation": "function", + "target_type": "function", + "database": None, + "sqlite_schema": None, + "table": None, + "target": "substr", + "columns": (), + "source": None, + "internal": False, + } in [operation_dict(operation) for operation in analysis.operations] + + def test_analyze_insert_tables(conn): analysis = analyze_sql_tables( conn, From 03b2c66f6312b8317d87eb4c1326977f6f63b26d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 15:17:10 -0700 Subject: [PATCH 745/767] Require full row mutation permissions for raw SQL Raw SQL insert and update statements can have broader effects than their SQLite authorizer callbacks reveal. INSERT OR REPLACE and UPDATE OR REPLACE can delete conflicting rows while only surfacing insert or update operations. Expand table insert and update operations to require insert-row, update-row, and delete-row together. Keep delete operations mapped to delete-row, and update the analysis UI/API to report and evaluate multiple required permissions for a single operation. Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4559083539 --- datasette/stored_queries.py | 108 ++++++++++++----- datasette/views/query_helpers.py | 27 +++-- tests/test_queries.py | 200 ++++++++++++++++++++++++++++++- 3 files changed, 290 insertions(+), 45 deletions(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index 4b0fe6a6..cf44a9ff 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -588,10 +588,25 @@ async def list_queries( ) -PermissionRequirement = tuple[str, Resource] +@dataclass(frozen=True) +class PermissionRequirement: + action: str + resource: Resource -def permission_for_operation(operation: Operation) -> PermissionRequirement | None: +def row_mutation_requirements( + database: str, table: str +) -> tuple[PermissionRequirement, ...]: + resource = TableResource(database=database, table=table) + return tuple( + PermissionRequirement(action=action, resource=resource) + for action in ("insert-row", "update-row", "delete-row") + ) + + +def permission_requirements_for_operation( + operation: Operation, +) -> tuple[PermissionRequirement, ...]: if ( operation.operation == "read" and operation.target_type == "table" @@ -599,31 +614,45 @@ def permission_for_operation(operation: Operation) -> PermissionRequirement | No and operation.table is not None ): return ( - "view-table", - TableResource(database=operation.database, table=operation.table), + PermissionRequirement( + action="view-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), ) - write_actions = { - "insert": "insert-row", - "update": "update-row", - "delete": "delete-row", - } - action = write_actions.get(operation.operation) if ( - action + operation.operation in {"insert", "update"} + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return row_mutation_requirements( + database=operation.database, + table=operation.table, + ) + if ( + operation.operation == "delete" and operation.target_type == "table" and operation.database is not None and operation.table is not None ): return ( - action, - TableResource(database=operation.database, table=operation.table), + PermissionRequirement( + action="delete-row", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), ) if operation.operation == "create" and operation.target_type == "table": if operation.database is None: - return None + return () return ( - "create-table", - DatabaseResource(database=operation.database), + PermissionRequirement( + action="create-table", + resource=DatabaseResource(database=operation.database), + ), ) if ( operation.operation == "alter" @@ -632,8 +661,12 @@ def permission_for_operation(operation: Operation) -> PermissionRequirement | No and operation.table is not None ): return ( - "alter-table", - TableResource(database=operation.database, table=operation.table), + PermissionRequirement( + action="alter-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), ) if ( operation.operation == "drop" @@ -642,8 +675,12 @@ def permission_for_operation(operation: Operation) -> PermissionRequirement | No and operation.table is not None ): return ( - "drop-table", - TableResource(database=operation.database, table=operation.table), + PermissionRequirement( + action="drop-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), ) if ( operation.operation in {"create", "drop"} @@ -652,10 +689,14 @@ def permission_for_operation(operation: Operation) -> PermissionRequirement | No and operation.table is not None ): return ( - "alter-table", - TableResource(database=operation.database, table=operation.table), + PermissionRequirement( + action="alter-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), ) - return None + return () def operation_should_be_ignored(operation: Operation) -> bool: @@ -704,20 +745,23 @@ async def ensure_query_write_permissions( for operation in analysis.operations: if operation_should_be_ignored(operation): continue - permission = permission_for_operation(operation) - if permission is None: + permissions = permission_requirements_for_operation(operation) + if not permissions: raise Forbidden( "Unsupported SQL operation: {} {}".format( operation.operation, operation.target_type ) ) - action, resource = permission if operation.database != database: raise Forbidden("Writable queries may not access attached databases") - if not await datasette.allowed( - action=action, - resource=resource, - actor=actor, - ): - raise Forbidden(f"Permission denied: need {action} on {resource}") + for permission in permissions: + if not await datasette.allowed( + action=permission.action, + resource=permission.resource, + actor=actor, + ): + raise Forbidden( + f"Permission denied: need {permission.action} " + f"on {permission.resource}" + ) return analysis diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 05a0d73e..7f3ef1bc 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -6,7 +6,7 @@ from datasette.stored_queries import ( StoredQuery, operation_is_write, operation_should_be_ignored, - permission_for_operation, + permission_requirements_for_operation, ) from datasette.utils import ( named_parameters as derive_named_parameters, @@ -216,8 +216,10 @@ def _display_operations(analysis: SQLAnalysis) -> list[Operation]: def _analysis_rows(analysis: SQLAnalysis) -> list[dict[str, object]]: rows = [] for operation in _display_operations(analysis): - permission = permission_for_operation(operation) - required_permission = permission[0] if permission else "" + permissions = permission_requirements_for_operation(operation) + required_permission = ", ".join( + permission.action for permission in permissions + ) rows.append( { "operation": operation.operation, @@ -236,14 +238,17 @@ async def _analysis_rows_with_permissions( rows = _analysis_rows(analysis) is_write = _analysis_is_write(analysis) for row, operation in zip(rows, _display_operations(analysis)): - permission = permission_for_operation(operation) - if permission: - action, resource = permission - row["allowed"] = await datasette.allowed( - action=action, - resource=resource, - actor=actor, - ) + permissions = permission_requirements_for_operation(operation) + if permissions: + row["allowed"] = True + for permission in permissions: + if not await datasette.allowed( + action=permission.action, + resource=permission.resource, + actor=actor, + ): + row["allowed"] = False + break elif is_write: row["allowed"] = False else: diff --git a/tests/test_queries.py b/tests/test_queries.py index 97ec973f..fcd19d1c 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -508,6 +508,8 @@ async def test_analyze_write_query_requires_table_permissions(): "dogs": { "permissions": { "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, } } } @@ -1429,7 +1431,7 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): "operation": "insert", "database": "data", "table": "dogs", - "required_permission": "insert-row", + "required_permission": "insert-row, update-row, delete-row", "source": None, "allowed": True, } @@ -1627,6 +1629,40 @@ async def test_execute_write_post_requires_database_and_table_permissions(): } } } + missing_update_permission = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": "insert into dogs (name) values (:name)", + "params": {"name": "Cleo"}, + }, + ) + + assert missing_update_permission.status_code == 403 + assert missing_update_permission.json()["errors"] == [ + "Permission denied: need update-row on data/dogs" + ] + + ds.config["databases"]["data"]["tables"]["dogs"]["permissions"][ + "update-row" + ] = {"id": "writer"} + missing_delete_permission = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": "insert into dogs (name) values (:name)", + "params": {"name": "Cleo"}, + }, + ) + + assert missing_delete_permission.status_code == 403 + assert missing_delete_permission.json()["errors"] == [ + "Permission denied: need delete-row on data/dogs" + ] + + ds.config["databases"]["data"]["tables"]["dogs"]["permissions"][ + "delete-row" + ] = {"id": "writer"} allowed = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, @@ -1643,6 +1679,156 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.asyncio +async def test_execute_write_insert_or_replace_requires_delete_row_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "users": { + "permissions": { + "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "view-table": {"id": "writer"}, + } + } + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_insert_or_replace", name="data") + await db.execute_write( + "create table users (id integer primary key, email text unique)" + ) + await db.execute_write( + "insert into users (id, email) values " + "(1, 'a@example.com'), (2, 'b@example.com')" + ) + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": ( + "insert or replace into users(id, email) " + "values (3, 'b@example.com')" + ) + }, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Permission denied: need delete-row on data/users" + ] + assert (await db.execute("select id, email from users order by id")).dicts() == [ + {"id": 1, "email": "a@example.com"}, + {"id": 2, "email": "b@example.com"}, + ] + + +@pytest.mark.asyncio +async def test_execute_write_update_or_replace_requires_delete_row_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "users": { + "permissions": { + "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "view-table": {"id": "writer"}, + } + } + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_update_or_replace", name="data") + await db.execute_write( + "create table users (id integer primary key, email text unique)" + ) + await db.execute_write( + "insert into users (id, email) values " + "(1, 'a@example.com'), (2, 'b@example.com')" + ) + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={"sql": "update or replace users set email = 'b@example.com' where id = 1"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Permission denied: need delete-row on data/users" + ] + assert (await db.execute("select id, email from users order by id")).dicts() == [ + {"id": 1, "email": "a@example.com"}, + {"id": 2, "email": "b@example.com"}, + ] + + +@pytest.mark.asyncio +async def test_execute_write_update_requires_insert_row_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "users": { + "permissions": { + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, + "view-table": {"id": "writer"}, + } + } + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_update_requires_insert", name="data") + await db.execute_write("create table users (id integer primary key, name text)") + await db.execute_write("insert into users (id, name) values (1, 'Alice')") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={"sql": "update users set name = 'Alicia' where id = 1"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Permission denied: need insert-row on data/users" + ] + assert (await db.execute("select name from users where id = 1")).first()[0] == "Alice" + + @pytest.mark.asyncio async def test_execute_write_insert_select_requires_view_table_on_source(): ds = Datasette( @@ -1659,7 +1845,13 @@ async def test_execute_write_insert_select_requires_view_table_on_source(): "secret": { "permissions": {"view-table": {"id": "someone-else"}} }, - "public_log": {"permissions": {"insert-row": {"id": "writer"}}}, + "public_log": { + "permissions": { + "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, + } + }, }, } } @@ -1740,6 +1932,8 @@ async def test_execute_write_rejects_function_operations(): "dogs": { "permissions": { "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, } } }, @@ -2117,6 +2311,8 @@ async def test_user_writable_query_execution_rechecks_table_permissions(): "dogs": { "permissions": { "insert-row": {"id": "alice"}, + "update-row": {"id": "alice"}, + "delete-row": {"id": "alice"}, } } }, From 1932f8429fd3259d48fb848fdf893f9a004276e9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 16:14:50 -0700 Subject: [PATCH 746/767] Deny user-authored schema table reads in write SQL Stop marking sqlite_master and sqlite_schema reads as internal as soon as the SQLite authorizer reports them. The later DDL-aware pass still treats schema catalog access as internal when it accompanies semantic CREATE, ALTER, or DROP operations. This makes explicit catalog reads in write SQL fall through to the deny-by-default path as unsupported read schema operations, preventing queries from copying private table definitions into writable tables. Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4559073803 --- datasette/utils/sql_analysis.py | 1 - datasette/views/query_helpers.py | 4 +- tests/test_queries.py | 73 +++++++++++++++++++++++++++----- tests/test_utils_sql_analysis.py | 20 +++++++++ 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 8963da77..91216501 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -256,7 +256,6 @@ def analyze_sql_tables( target=arg1, source=source, column=column, - internal=target_type == "schema", ) return sqlite3.SQLITE_OK diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 7f3ef1bc..0e3d4e01 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -217,9 +217,7 @@ def _analysis_rows(analysis: SQLAnalysis) -> list[dict[str, object]]: rows = [] for operation in _display_operations(analysis): permissions = permission_requirements_for_operation(operation) - required_permission = ", ".join( - permission.action for permission in permissions - ) + required_permission = ", ".join(permission.action for permission in permissions) rows.append( { "operation": operation.operation, diff --git a/tests/test_queries.py b/tests/test_queries.py index fcd19d1c..40bc5052 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1643,9 +1643,9 @@ async def test_execute_write_post_requires_database_and_table_permissions(): "Permission denied: need update-row on data/dogs" ] - ds.config["databases"]["data"]["tables"]["dogs"]["permissions"][ - "update-row" - ] = {"id": "writer"} + ds.config["databases"]["data"]["tables"]["dogs"]["permissions"]["update-row"] = { + "id": "writer" + } missing_delete_permission = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, @@ -1660,9 +1660,9 @@ async def test_execute_write_post_requires_database_and_table_permissions(): "Permission denied: need delete-row on data/dogs" ] - ds.config["databases"]["data"]["tables"]["dogs"]["permissions"][ - "delete-row" - ] = {"id": "writer"} + ds.config["databases"]["data"]["tables"]["dogs"]["permissions"]["delete-row"] = { + "id": "writer" + } allowed = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, @@ -1719,8 +1719,7 @@ async def test_execute_write_insert_or_replace_requires_delete_row_permission(): actor={"id": "writer"}, json={ "sql": ( - "insert or replace into users(id, email) " - "values (3, 'b@example.com')" + "insert or replace into users(id, email) " "values (3, 'b@example.com')" ) }, ) @@ -1773,7 +1772,9 @@ async def test_execute_write_update_or_replace_requires_delete_row_permission(): denied_response = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, - json={"sql": "update or replace users set email = 'b@example.com' where id = 1"}, + json={ + "sql": "update or replace users set email = 'b@example.com' where id = 1" + }, ) assert denied_response.status_code == 403 @@ -1826,7 +1827,9 @@ async def test_execute_write_update_requires_insert_row_permission(): assert denied_response.json()["errors"] == [ "Permission denied: need insert-row on data/users" ] - assert (await db.execute("select name from users where id = 1")).first()[0] == "Alice" + assert (await db.execute("select name from users where id = 1")).first()[ + 0 + ] == "Alice" @pytest.mark.asyncio @@ -1876,6 +1879,56 @@ async def test_execute_write_insert_select_requires_view_table_on_source(): assert (await db.execute("select value from public_log")).dicts() == [] +@pytest.mark.asyncio +async def test_execute_write_rejects_sqlite_master_reads(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "secret": { + "permissions": {"view-table": {"id": "someone-else"}} + }, + "log": { + "permissions": { + "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, + } + }, + }, + } + } + }, + ) + db = ds.add_memory_database("execute_write_sqlite_master_read", name="data") + await db.execute_write("create table secret (value text)") + await db.execute_write("create table log (value text)") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": ( + "insert into log " "select sql from sqlite_master where name = 'secret'" + ) + }, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Unsupported SQL operation: read schema" + ] + assert (await db.execute("select value from log")).dicts() == [] + + @pytest.mark.asyncio async def test_execute_write_create_table_as_select_requires_view_table_on_source(): ds = Datasette( diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index 2ae11502..f931be51 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -65,6 +65,26 @@ def test_analyze_uses_sqlite_schema_as_default_database(conn): } +def test_analyze_user_schema_table_read_is_not_internal(conn): + analysis = analyze_sql_tables( + conn, + "insert into log select sql from sqlite_master where name = 'dogs'", + database_name="data", + ) + + assert { + "operation": "read", + "target_type": "schema", + "database": "data", + "sqlite_schema": "main", + "table": None, + "target": "sqlite_master", + "columns": ("name", "sql"), + "source": None, + "internal": False, + } in [operation_dict(operation) for operation in analysis.operations] + + def operation_dict(operation): return { "operation": operation.operation, From 951f5a9f306ebe0bb8b3668ee698dc6cb6051d78 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 16:30:05 -0700 Subject: [PATCH 747/767] Detect VACUUM in SQL analysis Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4559073803 --- datasette/stored_queries.py | 1 + datasette/utils/sql_analysis.py | 33 +++++++++++++++++++++++- tests/test_queries.py | 31 ++++++++++++++++++++++ tests/test_utils_sql_analysis.py | 44 ++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 1 deletion(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index cf44a9ff..6746124a 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -720,6 +720,7 @@ def operation_is_write(operation: Operation) -> bool: "pragma", "analyze", "reindex", + "vacuum", "unknown", } diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index 91216501..f2eb903f 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -22,6 +22,7 @@ SQLOperation = Literal[ "pragma", "analyze", "reindex", + "vacuum", "unknown", ] SQLTargetType = Literal[ @@ -423,10 +424,40 @@ def analyze_sql_tables( conn.set_authorizer(authorizer) try: - conn.execute("EXPLAIN " + sql, params if params is not None else {}).fetchall() + explain_rows = conn.execute( + "EXPLAIN " + sql, params if params is not None else {} + ).fetchall() finally: conn.set_authorizer(None) + if not operations: + vacuum_row = next((row for row in explain_rows if row[1] == "Vacuum"), None) + if vacuum_row is not None: + schema_by_index = { + row[0]: row[1] for row in conn.execute("PRAGMA database_list") + } + sqlite_schema = schema_by_index.get(vacuum_row[2]) + database = database_for_schema(sqlite_schema) + record( + "vacuum", + "database", + database=database, + table=None, + sqlite_schema=sqlite_schema, + target=database, + source=None, + ) + else: + record( + "unknown", + "statement", + database=database_name, + table=None, + sqlite_schema=None, + target=None, + source=None, + ) + has_schema_operation = any( key.target_type in {"table", "index", "view", "trigger", "virtual-table"} and key.operation in {"create", "alter", "drop"} diff --git a/tests/test_queries.py b/tests/test_queries.py index 40bc5052..bf371a80 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -2011,6 +2011,37 @@ async def test_execute_write_rejects_function_operations(): assert (await db.execute("select name from dogs")).dicts() == [] +@pytest.mark.asyncio +async def test_execute_write_rejects_vacuum_operation(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("execute_write_vacuum_operation", name="data") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={"sql": "vacuum"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Unsupported SQL operation: vacuum database" + ] + + @pytest.mark.asyncio async def test_execute_write_create_table_uses_create_table_permission(): ds = Datasette( diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index f931be51..df4b3625 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -129,6 +129,50 @@ def test_analyze_create_table_operation(): ] +def test_analyze_vacuum_operation(): + conn = sqlite3.connect(":memory:") + try: + analysis = analyze_sql_tables(conn, "vacuum", database_name="data") + finally: + conn.close() + + assert [operation_dict(operation) for operation in analysis.operations] == [ + { + "operation": "vacuum", + "target_type": "database", + "database": "data", + "sqlite_schema": "main", + "table": None, + "target": "data", + "columns": (), + "source": None, + "internal": False, + } + ] + + +def test_analyze_statement_with_no_authorizer_callbacks_is_unknown(): + conn = sqlite3.connect(":memory:") + try: + analysis = analyze_sql_tables(conn, "reindex", database_name="data") + finally: + conn.close() + + assert [operation_dict(operation) for operation in analysis.operations] == [ + { + "operation": "unknown", + "target_type": "statement", + "database": "data", + "sqlite_schema": None, + "table": None, + "target": None, + "columns": (), + "source": None, + "internal": False, + } + ] + + def test_analyze_transaction_operation(conn): analysis = analyze_sql_tables(conn, "commit", database_name="data") From 11bddc891918849e7c4a006c64d0217072aa499c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 16:51:12 -0700 Subject: [PATCH 748/767] Deny VACUUM in user-authored SQL Reject VACUUM explicitly during write-query permission analysis so arbitrary write SQL and untrusted stored write queries cannot run it, even when the actor has execute-write-sql. Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4559073803 (P3) --- datasette/stored_queries.py | 16 ++++ datasette/views/database.py | 23 ++++- datasette/views/execute_write.py | 6 +- datasette/views/query_helpers.py | 9 +- tests/test_queries.py | 153 ++++++++++++++++++++++++++++++- 5 files changed, 199 insertions(+), 8 deletions(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index 6746124a..fd1cabf3 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -15,6 +15,13 @@ if TYPE_CHECKING: UNCHANGED = object() + +class QueryWriteRejected(Exception): + def __init__(self, message: str): + self.message = message + super().__init__(message) + + QUERY_OPTION_FIELDS = ( "hide_sql", "fragment", @@ -703,6 +710,12 @@ def operation_should_be_ignored(operation: Operation) -> bool: return operation.internal or operation.operation == "select" +def operation_forbidden_message(operation: Operation) -> str | None: + if operation.operation == "vacuum": + return "VACUUM is not allowed in user-supplied SQL" + return None + + def operation_is_write(operation: Operation) -> bool: return operation.operation in { "insert", @@ -746,6 +759,9 @@ async def ensure_query_write_permissions( for operation in analysis.operations: if operation_should_be_ignored(operation): continue + forbidden_message = operation_forbidden_message(operation) + if forbidden_message is not None: + raise QueryWriteRejected(forbidden_message) permissions = permission_requirements_for_operation(operation) if not permissions: raise Forbidden( diff --git a/datasette/views/database.py b/datasette/views/database.py index b558b002..ae1cf375 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -13,7 +13,7 @@ import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource -from datasette.stored_queries import stored_query_to_dict +from datasette.stored_queries import QueryWriteRejected, stored_query_to_dict from datasette.utils import ( add_cors_headers, await_me_maybe, @@ -453,9 +453,24 @@ class QueryView(View): ): raise Forbidden("You do not have permission to view this query") - await _ensure_stored_query_execution_permissions( - datasette, db, stored_query, request.actor - ) + try: + await _ensure_stored_query_execution_permissions( + datasette, db, stored_query, request.actor + ) + except QueryWriteRejected as ex: + if request.headers.get("accept") == "application/json" or request.args.get( + "_json" + ): + return Response.json( + { + "ok": False, + "message": ex.message, + "redirect": None, + }, + status=403, + ) + datasette.add_message(request, ex.message, datasette.ERROR) + return Response.redirect(stored_query.on_error_redirect or request.path) # If database is immutable, return an error if not db.is_mutable: diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index 19006ac5..57c4d78e 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -163,13 +163,15 @@ class ExecuteWriteView(BaseView): except QueryValidationError as ex: if _wants_json(request, is_json, data): return _block_framing(_error([ex.message], ex.status)) + if ex.flash: + self.ds.add_message(request, ex.message, self.ds.ERROR) return await self._render_form( request, db, sql=sql or "", parameter_values=provided_params, - analysis_error=ex.message, - execution_message=ex.message, + analysis_error=None if ex.flash else ex.message, + execution_message=None if ex.flash else ex.message, execution_ok=False, status=ex.status, ) diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 0e3d4e01..92328ff3 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -3,6 +3,7 @@ import re from datasette.resources import DatabaseResource from datasette.stored_queries import ( + QueryWriteRejected, StoredQuery, operation_is_write, operation_should_be_ignored, @@ -47,9 +48,11 @@ _query_write_fields = { class QueryValidationError(Exception): - def __init__(self, message, status=400): + def __init__(self, message, status=400, *, flash=False): self.message = message self.status = status + self.flash = flash + super().__init__(message) def _actor_id(actor): @@ -194,6 +197,8 @@ async def _analyze_user_query(datasette, db, sql, *, actor): await datasette.ensure_query_write_permissions( db.name, sql, actor=actor, analysis=analysis ) + except QueryWriteRejected as ex: + raise QueryValidationError(ex.message, status=403, flash=True) from ex except Forbidden as ex: raise QueryValidationError(str(ex), status=403) from ex else: @@ -297,6 +302,8 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): await datasette.ensure_query_write_permissions( db.name, sql, actor=actor, analysis=analysis ) + except QueryWriteRejected as ex: + raise QueryValidationError(ex.message, status=403, flash=True) from ex except Forbidden as ex: raise QueryValidationError(str(ex), status=403) from ex return parameter_names, params, analysis diff --git a/tests/test_queries.py b/tests/test_queries.py index bf371a80..b6e1637d 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -2038,10 +2038,161 @@ async def test_execute_write_rejects_vacuum_operation(): assert denied_response.status_code == 403 assert denied_response.json()["errors"] == [ - "Unsupported SQL operation: vacuum database" + "VACUUM is not allowed in user-supplied SQL" ] +@pytest.mark.asyncio +async def test_execute_write_form_rejects_vacuum_operation_with_flash_error(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("execute_write_vacuum_operation_form", name="data") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + data={"sql": "vacuum"}, + ) + + assert denied_response.status_code == 403 + assert ( + '

VACUUM is not allowed in user-supplied SQL

' + in denied_response.text + ) + assert denied_response.text.count("VACUUM is not allowed in user-supplied SQL") == 1 + + +@pytest.mark.asyncio +async def test_untrusted_stored_write_query_rejects_vacuum_operation(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "view-query": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("stored_query_vacuum_operation", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "vacuum_db", + "vacuum", + is_write=True, + is_trusted=False, + source="user", + owner_id="writer", + ) + + denied_response = await ds.client.post( + "/data/vacuum_db?_json=1", + actor={"id": "writer"}, + data={}, + ) + + assert denied_response.status_code == 403 + assert "VACUUM is not allowed in user-supplied SQL" in denied_response.text + + +@pytest.mark.asyncio +async def test_untrusted_stored_write_query_rejects_vacuum_operation_with_flash_error(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "view-query": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("stored_query_vacuum_operation_form", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "vacuum_db", + "vacuum", + is_write=True, + is_trusted=False, + source="user", + owner_id="writer", + ) + + denied_response = await ds.client.post( + "/data/vacuum_db", + actor={"id": "writer"}, + data={}, + ) + + assert denied_response.status_code == 302 + assert denied_response.headers["location"] == "/data/vacuum_db" + assert ds.unsign(denied_response.cookies["ds_messages"], "messages") == [ + ["VACUUM is not allowed in user-supplied SQL", ds.ERROR] + ] + + +@pytest.mark.asyncio +async def test_trusted_stored_write_query_skips_vacuum_filtering(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "view-query": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("trusted_stored_query_vacuum", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "trusted_vacuum", + "vacuum", + is_write=True, + is_trusted=True, + source="config", + ) + + response = await ds.client.post( + "/data/trusted_vacuum?_json=1", + actor={"id": "writer"}, + data={}, + ) + + assert response.status_code == 200 + assert response.json()["ok"] is True + + @pytest.mark.asyncio async def test_execute_write_create_table_uses_create_table_permission(): ds = Datasette( From 0c5053cdf64a0dc2d1e9808fa712b88233760512 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 27 May 2026 17:26:50 -0700 Subject: [PATCH 749/767] Docs for //-/execute-write JSON API Closes #2750, refs #2742 --- docs/json_api.rst | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/docs/json_api.rst b/docs/json_api.rst index 48c70af6..fffc16d7 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -505,6 +505,68 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`. +.. _ExecuteWriteView: + +Executing write SQL +~~~~~~~~~~~~~~~~~~~ + +Actors with the :ref:`actions_execute_write_sql` permission can execute arbitrary writable SQL against a mutable database using ``/-/execute-write``. + +:: + + POST //-/execute-write + Content-Type: application/json + Authorization: Bearer dstok_ + +The request body must include a ``"sql"`` string. Named SQL parameters can be provided using the optional ``"params"`` object: + +.. code-block:: json + + { + "sql": "insert into dogs (name) values (:name)", + "params": { + "name": "Cleo" + } + } + +The SQL must be writable. Read-only ``select`` queries should use the regular :ref:`custom SQL query API ` instead. + +Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. + +A successful response includes a message, the SQLite ``rowcount`` and a summary of the operations that were executed: + +The shape of the ``"analysis"`` block is not yet considered a stable API and may change in future Datasette releases. + +.. code-block:: json + + { + "ok": true, + "message": "Query executed, 1 row affected", + "rowcount": 1, + "analysis": [ + { + "operation": "insert", + "database": "data", + "table": "dogs", + "required_permission": "insert-row, update-row, delete-row", + "source": null + } + ] + } + +If SQLite reports ``-1`` for the row count, the message will be ``"Query executed"``. + +Errors use the standard Datasette error format: + +.. code-block:: json + + { + "ok": false, + "errors": [ + "Permission denied: need execute-write-sql" + ] + } + .. _TableInsertView: Inserting rows From bcd989f4f8802a73a60c75f9bda77649c1347986 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 08:36:59 -0700 Subject: [PATCH 750/767] Detect and disallow insert to virtual/shadow table Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4565727978 --- datasette/stored_queries.py | 5 + datasette/utils/sql_analysis.py | 21 ++++- datasette/utils/sqlite.py | 112 ++++++++++++++++++++++ tests/test_queries.py | 153 +++++++++++++++++++++++++++++++ tests/test_utils.py | 45 ++++++++- tests/test_utils_sql_analysis.py | 47 ++++++++++ 6 files changed, 381 insertions(+), 2 deletions(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index fd1cabf3..b5aea221 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -713,6 +713,11 @@ def operation_should_be_ignored(operation: Operation) -> bool: def operation_forbidden_message(operation: Operation) -> str | None: if operation.operation == "vacuum": return "VACUUM is not allowed in user-supplied SQL" + if operation.operation in {"insert", "update", "delete"}: + if operation.table_kind == "virtual": + return "Writes to virtual tables are not allowed in user-supplied SQL" + if operation.table_kind == "shadow": + return "Writes to shadow tables are not allowed in user-supplied SQL" return None diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index f2eb903f..a71fa315 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Literal -from datasette.utils.sqlite import sqlite3 +from datasette.utils.sqlite import SQLiteTableType, sqlite3, sqlite_table_type SQLOperation = Literal[ "read", @@ -42,6 +42,7 @@ SQLTargetType = Literal[ SQLTableOperation = Literal["read", "insert", "update", "delete"] SQLSchemaOperation = Literal["create", "drop"] SQLSchemaTargetType = Literal["index", "table", "trigger", "view", "virtual-table"] +SQLTableKind = SQLiteTableType @dataclass(frozen=True) @@ -51,6 +52,7 @@ class Operation: database: str | None table: str | None sqlite_schema: str | None + table_kind: SQLTableKind | None = None target: str | None = None columns: tuple[str, ...] = () source: str | None = None @@ -500,6 +502,22 @@ def analyze_sql_tables( return True return False + table_kind_cache: dict[tuple[str | None, str], SQLTableKind | None] = {} + + def table_kind_for(key: OperationKey) -> SQLTableKind | None: + if ( + key.target_type != "table" + or key.operation not in {"read", "insert", "update", "delete"} + or key.table is None + ): + return None + cache_key = (key.sqlite_schema, key.table) + if cache_key not in table_kind_cache: + table_kind_cache[cache_key] = sqlite_table_type( + conn, key.table, schema=key.sqlite_schema + ) + return table_kind_cache[cache_key] + return SQLAnalysis( operations=tuple( Operation( @@ -508,6 +526,7 @@ def analyze_sql_tables( database=key.database, table=key.table, sqlite_schema=key.sqlite_schema, + table_kind=table_kind_for(key), target=key.target, columns=tuple(sorted(columns)), source=key.source, diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index d0a2d783..130c5f62 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -1,3 +1,6 @@ +import re +from typing import Literal + using_pysqlite3 = False try: import pysqlite3 as sqlite3 @@ -10,6 +13,18 @@ if hasattr(sqlite3, "enable_callback_tracebacks"): sqlite3.enable_callback_tracebacks(True) _cached_sqlite_version = None +SQLiteTableType = Literal["table", "view", "virtual", "shadow"] +_VIRTUAL_TABLE_MODULE_RE = re.compile( + r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)", + re.IGNORECASE, +) +_VIRTUAL_TABLE_SHADOW_SUFFIXES = { + "fts3": ("_content", "_segdir", "_segments", "_stat", "_docsize"), + "fts4": ("_content", "_segdir", "_segments", "_stat", "_docsize"), + "fts5": ("_data", "_idx", "_docsize", "_content", "_config"), + "rtree": ("_node", "_parent", "_rowid"), + "rtree_i32": ("_node", "_parent", "_rowid"), +} def sqlite_version(): @@ -36,5 +51,102 @@ def supports_table_xinfo(): return sqlite_version() >= (3, 26, 0) +def supports_table_list(): + return sqlite_version() >= (3, 37, 0) + + def supports_generated_columns(): return sqlite_version() >= (3, 31, 0) + + +def sqlite_table_type( + conn, + table: str, + *, + schema: str | None = "main", +) -> SQLiteTableType | None: + if supports_table_list(): + try: + query = "select type from pragma_table_list where name = ?" + params: tuple[str, ...] = (table,) + if schema is not None: + query += " and schema = ?" + params = (table, schema) + row = conn.execute(query, params).fetchone() + if row is not None and row[0] in {"table", "view", "virtual", "shadow"}: + return row[0] + except sqlite3.DatabaseError: + pass + return _sqlite_table_type_from_schema(conn, table, schema=schema) + + +def _sqlite_table_type_from_schema( + conn, + table: str, + *, + schema: str | None = "main", +) -> SQLiteTableType | None: + schema_table = _sqlite_schema_table(schema) + try: + row = conn.execute( + "select type, sql from {} where name = ?".format(schema_table), + (table,), + ).fetchone() + except sqlite3.DatabaseError: + return None + if row is None: + return None + object_type, sql = row + if object_type == "view": + return "view" + if object_type != "table": + return None + if _virtual_table_module(sql) is not None: + return "virtual" + if _is_known_shadow_table(conn, table, schema=schema): + return "shadow" + return "table" + + +def _is_known_shadow_table( + conn, + table: str, + *, + schema: str | None = "main", +) -> bool: + schema_table = _sqlite_schema_table(schema) + try: + rows = conn.execute( + "select name, sql from {} where type = 'table'".format(schema_table) + ).fetchall() + except sqlite3.DatabaseError: + return False + for virtual_table, sql in rows: + module = _virtual_table_module(sql) + if module is None: + continue + for suffix in _VIRTUAL_TABLE_SHADOW_SUFFIXES.get(module, ()): + if table == virtual_table + suffix: + return True + return False + + +def _sqlite_schema_table(schema: str | None) -> str: + if schema is None or schema == "main": + return "sqlite_master" + if schema == "temp": + return "sqlite_temp_master" + return "{}.sqlite_master".format(_quote_identifier(schema)) + + +def _quote_identifier(value: str) -> str: + return '"{}"'.format(value.replace('"', '""')) + + +def _virtual_table_module(sql: str | None) -> str | None: + if not sql: + return None + match = _VIRTUAL_TABLE_MODULE_RE.search(sql) + if match is None: + return None + return match.group(1).strip("\"'[]`").lower() diff --git a/tests/test_queries.py b/tests/test_queries.py index b6e1637d..73f8f3cf 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -2193,6 +2193,159 @@ async def test_trusted_stored_write_query_skips_vacuum_filtering(): assert response.json()["ok"] is True +@pytest.mark.asyncio +async def test_execute_write_rejects_virtual_table_control_insert(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_virtual_table_control", name="data") + await db.execute_write(""" + create virtual table docs using fts5(title, body, content='') + """) + await db.execute_write(""" + insert into docs(rowid, title, body) values (1, 'hello', 'world') + """) + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + json={"sql": "insert into docs(docs) values('delete-all')"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Writes to virtual tables are not allowed in user-supplied SQL" + ] + assert ( + await db.execute("select count(*) from docs where docs match 'hello'") + ).first()[0] == 1 + + +@pytest.mark.asyncio +async def test_execute_write_rejects_regular_virtual_table_insert(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_virtual_table_insert", name="data") + await db.execute_write("create virtual table docs using fts5(title, body)") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + json={"sql": "insert into docs(rowid, title, body) values (1, 'a', 'b')"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Writes to virtual tables are not allowed in user-supplied SQL" + ] + assert (await db.execute("select count(*) from docs")).first()[0] == 0 + + +@pytest.mark.asyncio +async def test_execute_write_rejects_shadow_table_insert(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_shadow_table_insert", name="data") + await db.execute_write("create virtual table docs using fts5(title, body)") + await ds.invoke_startup() + + denied_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + json={"sql": "insert into docs_config(k, v) values ('x', 1)"}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [ + "Writes to shadow tables are not allowed in user-supplied SQL" + ] + assert (await db.execute("select count(*) from docs_config")).first()[0] == 1 + + +@pytest.mark.asyncio +async def test_untrusted_stored_write_query_rejects_virtual_table_control_insert(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("stored_query_virtual_table_control", name="data") + await db.execute_write(""" + create virtual table docs using fts5(title, body, content='') + """) + await db.execute_write(""" + insert into docs(rowid, title, body) values (1, 'hello', 'world') + """) + await ds.invoke_startup() + await ds.add_query( + "data", + "delete_all_docs", + "insert into docs(docs) values('delete-all')", + is_write=True, + is_trusted=False, + source="user", + owner_id="root", + ) + + denied_response = await ds.client.post( + "/data/delete_all_docs?_json=1", + actor={"id": "root"}, + data={}, + ) + + assert denied_response.status_code == 403 + assert denied_response.json()["message"] == ( + "Writes to virtual tables are not allowed in user-supplied SQL" + ) + assert ( + await db.execute("select count(*) from docs where docs match 'hello'") + ).first()[0] == 1 + + +@pytest.mark.asyncio +async def test_trusted_stored_write_query_can_write_virtual_table(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "view-query": {"id": "writer"}, + } + } + } + }, + ) + db = ds.add_memory_database("trusted_stored_query_virtual_table", name="data") + await db.execute_write(""" + create virtual table docs using fts5(title, body, content='') + """) + await db.execute_write(""" + insert into docs(rowid, title, body) values (1, 'hello', 'world') + """) + await ds.invoke_startup() + await ds.add_query( + "data", + "trusted_delete_all", + "insert into docs(docs) values('delete-all')", + is_write=True, + is_trusted=True, + source="config", + ) + + response = await ds.client.post( + "/data/trusted_delete_all?_json=1", + actor={"id": "writer"}, + data={}, + ) + + assert response.status_code == 200 + assert response.json()["ok"] is True + assert ( + await db.execute("select count(*) from docs where docs match 'hello'") + ).first()[0] == 0 + + @pytest.mark.asyncio async def test_execute_write_create_table_uses_create_table_permission(): ds = Datasette( diff --git a/tests/test_utils.py b/tests/test_utils.py index 3fcb623e..e142bb5b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,7 @@ Tests for various datasette helper functions. from datasette.app import Datasette from datasette import utils from datasette.utils.asgi import Request -from datasette.utils.sqlite import sqlite3 +from datasette.utils.sqlite import sqlite3, sqlite_table_type import json import os import pathlib @@ -226,6 +226,49 @@ def test_detect_fts_different_table_names(table): conn.close() +@pytest.mark.parametrize("use_fallback", (False, True)) +def test_sqlite_table_type_detects_virtual_and_shadow_tables(monkeypatch, use_fallback): + if use_fallback: + monkeypatch.setattr("datasette.utils.sqlite.sqlite_version", lambda: (3, 25, 0)) + conn = utils.sqlite3.connect(":memory:") + try: + conn.executescript(""" + create table dogs(id integer primary key, name text); + create view dog_names as select name from dogs; + create virtual table search_index using fts5(title, body); + create virtual table boxes using rtree(id, minx, maxx, miny, maxy); + """) + + assert sqlite_table_type(conn, "dogs") == "table" + assert sqlite_table_type(conn, "dog_names") == "view" + assert sqlite_table_type(conn, "search_index") == "virtual" + assert sqlite_table_type(conn, "search_index_config") == "shadow" + assert sqlite_table_type(conn, "boxes") == "virtual" + assert sqlite_table_type(conn, "boxes_node") == "shadow" + assert sqlite_table_type(conn, "missing") is None + finally: + conn.close() + + +@pytest.mark.parametrize("use_fallback", (False, True)) +def test_sqlite_table_type_detects_attached_database_tables(monkeypatch, use_fallback): + if use_fallback: + monkeypatch.setattr("datasette.utils.sqlite.sqlite_version", lambda: (3, 25, 0)) + conn = utils.sqlite3.connect(":memory:") + try: + conn.executescript(""" + attach database ':memory:' as extra; + create table extra.cats(id integer primary key, name text); + create virtual table extra.cat_search using fts5(name); + """) + + assert sqlite_table_type(conn, "cats", schema="extra") == "table" + assert sqlite_table_type(conn, "cat_search", schema="extra") == "virtual" + assert sqlite_table_type(conn, "cat_search_data", schema="extra") == "shadow" + finally: + conn.close() + + @pytest.mark.parametrize( "url,expected", [ diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index df4b3625..979ff9e1 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -260,6 +260,53 @@ def test_analyze_create_virtual_table_operation(): } in [operation_dict(operation) for operation in analysis.operations] +def test_analyze_table_kind_for_regular_virtual_and_shadow_tables(): + conn = sqlite3.connect(":memory:") + try: + conn.executescript(""" + create table dogs (id integer primary key, name text); + create virtual table docs using fts5(title, body, content=''); + """) + + regular_analysis = analyze_sql_tables( + conn, + "insert into dogs (name) values ('Cleo')", + database_name="data", + ) + virtual_analysis = analyze_sql_tables( + conn, + "insert into docs(docs) values('delete-all')", + database_name="data", + ) + shadow_analysis = analyze_sql_tables( + conn, + "insert into docs_config(k, v) values ('x', 1)", + database_name="data", + ) + finally: + conn.close() + + regular_insert = next( + operation + for operation in regular_analysis.operations + if operation.operation == "insert" and operation.table == "dogs" + ) + virtual_insert = next( + operation + for operation in virtual_analysis.operations + if operation.operation == "insert" and operation.table == "docs" + ) + shadow_insert = next( + operation + for operation in shadow_analysis.operations + if operation.operation == "insert" and operation.table == "docs_config" + ) + + assert regular_insert.table_kind == "table" + assert virtual_insert.table_kind == "virtual" + assert shadow_insert.table_kind == "shadow" + + def test_analyze_create_table_as_select_function_is_not_internal(): conn = sqlite3.connect(":memory:") try: From aaf00e9ec22b77e53f291ccedcbf2f499cce9e2b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 08:42:06 -0700 Subject: [PATCH 751/767] Refactor hidden_table_names() to use new implemenatation Refs https://github.com/simonw/datasette/pull/2749#issuecomment-4565727978 --- datasette/database.py | 80 +------------------------------- datasette/utils/sqlite.py | 29 ++++++++++++ tests/test_internals_database.py | 9 +--- tests/test_utils.py | 12 ++++- 4 files changed, 43 insertions(+), 87 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index e7e9527e..10417670 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -26,7 +26,7 @@ from .utils import ( table_column_details, ) from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables -from .utils.sqlite import sqlite_version +from .utils.sqlite import sqlite_hidden_table_names from .inspect import inspect_hash connections = threading.local() @@ -702,83 +702,7 @@ class Database: t for t in db_config["tables"] if db_config["tables"][t].get("hidden") ] - if sqlite_version()[1] >= 37: - hidden_tables += [x[0] for x in await self.execute(""" - with shadow_tables as ( - select name - from pragma_table_list - where [type] = 'shadow' - order by name - ), - core_tables as ( - select name - from sqlite_master - WHERE name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') - OR substr(name, 1, 1) == '_' - ), - combined as ( - select name from shadow_tables - union all - select name from core_tables - ) - select name from combined order by 1 - """)] - else: - hidden_tables += [x[0] for x in await self.execute(""" - WITH base AS ( - SELECT name - FROM sqlite_master - WHERE name IN ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') - OR substr(name, 1, 1) == '_' - ), - fts_suffixes AS ( - SELECT column1 AS suffix - FROM (VALUES ('_data'), ('_idx'), ('_docsize'), ('_content'), ('_config')) - ), - fts5_names AS ( - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%USING FTS%' - ), - fts5_shadow_tables AS ( - SELECT - printf('%s%s', fts5_names.name, fts_suffixes.suffix) AS name - FROM fts5_names - JOIN fts_suffixes - ), - fts3_suffixes AS ( - SELECT column1 AS suffix - FROM (VALUES ('_content'), ('_segdir'), ('_segments'), ('_stat'), ('_docsize')) - ), - fts3_names AS ( - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%USING FTS3%' - OR sql LIKE '%VIRTUAL TABLE%USING FTS4%' - ), - fts3_shadow_tables AS ( - SELECT - printf('%s%s', fts3_names.name, fts3_suffixes.suffix) AS name - FROM fts3_names - JOIN fts3_suffixes - ), - final AS ( - SELECT name FROM base - UNION ALL - SELECT name FROM fts5_shadow_tables - UNION ALL - SELECT name FROM fts3_shadow_tables - ) - SELECT name FROM final ORDER BY 1 - """)] - # Also hide any FTS tables that have a content= argument - hidden_tables += [x[0] for x in await self.execute(""" - SELECT name - FROM sqlite_master - WHERE sql LIKE '%VIRTUAL TABLE%' - AND sql LIKE '%USING FTS%' - AND sql LIKE '%content=%' - """)] + hidden_tables += await self.execute_fn(sqlite_hidden_table_names) has_spatialite = await self.execute_fn(detect_spatialite) if has_spatialite: diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index 130c5f62..d3f52751 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -80,6 +80,28 @@ def sqlite_table_type( return _sqlite_table_type_from_schema(conn, table, schema=schema) +def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]: + schema_table = _sqlite_schema_table(schema) + try: + rows = conn.execute( + "select name, sql from {} where type = 'table'".format(schema_table) + ).fetchall() + except sqlite3.DatabaseError: + return [] + hidden_tables = [] + content_fts_tables = [] + for name, sql in rows: + if ( + name in {"sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"} + or name.startswith("_") + or sqlite_table_type(conn, name, schema=schema) == "shadow" + ): + hidden_tables.append(name) + elif _is_fts_content_virtual_table(sql): + content_fts_tables.append(name) + return sorted(hidden_tables) + content_fts_tables + + def _sqlite_table_type_from_schema( conn, table: str, @@ -150,3 +172,10 @@ def _virtual_table_module(sql: str | None) -> str | None: if match is None: return None return match.group(1).strip("\"'[]`").lower() + + +def _is_fts_content_virtual_table(sql: str | None) -> bool: + return ( + _virtual_table_module(sql) in {"fts3", "fts4", "fts5"} + and "content=" in sql.lower() + ) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index d6e130b4..88f9d571 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -8,7 +8,7 @@ from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues from datasette.database import DatasetteClosedError from datasette.database import _deliver_write_result -from datasette.utils.sqlite import sqlite3, sqlite_version +from datasette.utils.sqlite import sqlite3 from datasette.utils import Column import pytest import time @@ -798,14 +798,7 @@ async def test_in_memory_databases_forbid_writes(app_client): assert await db.table_names() == ["foo"] -def pragma_table_list_supported(): - return sqlite_version()[1] >= 37 - - @pytest.mark.asyncio -@pytest.mark.skipif( - not pragma_table_list_supported(), reason="Requires PRAGMA table_list support" -) async def test_hidden_tables(app_client): ds = app_client.ds db = ds.add_database(Database(ds, is_memory=True, is_mutable=True)) diff --git a/tests/test_utils.py b/tests/test_utils.py index e142bb5b..90013537 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,7 @@ Tests for various datasette helper functions. from datasette.app import Datasette from datasette import utils from datasette.utils.asgi import Request -from datasette.utils.sqlite import sqlite3, sqlite_table_type +from datasette.utils.sqlite import sqlite3, sqlite_hidden_table_names, sqlite_table_type import json import os import pathlib @@ -246,6 +246,16 @@ def test_sqlite_table_type_detects_virtual_and_shadow_tables(monkeypatch, use_fa assert sqlite_table_type(conn, "boxes") == "virtual" assert sqlite_table_type(conn, "boxes_node") == "shadow" assert sqlite_table_type(conn, "missing") is None + assert sqlite_hidden_table_names(conn) == [ + "boxes_node", + "boxes_parent", + "boxes_rowid", + "search_index_config", + "search_index_content", + "search_index_data", + "search_index_docsize", + "search_index_idx", + ] finally: conn.close() From 2785fd29deef505f132902dcee86284e39e3fdcb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 09:03:10 -0700 Subject: [PATCH 752/767] Fix tests I just broke --- datasette/utils/sql_analysis.py | 86 +++++++++++++++++++-------------- datasette/utils/sqlite.py | 2 +- tests/test_utils.py | 14 ++++++ 3 files changed, 65 insertions(+), 37 deletions(-) diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index a71fa315..b5d7ada8 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -193,6 +193,10 @@ _AUTHORIZER_ACTION_NAMES = { } +def _allow_authorizer_action(*args): + return sqlite3.SQLITE_OK + + def analyze_sql_tables( conn, sql: str, @@ -424,42 +428,59 @@ def analyze_sql_tables( ) return sqlite3.SQLITE_OK + table_kind_cache: dict[tuple[str | None, str], SQLTableKind | None] = {} + conn.set_authorizer(authorizer) try: explain_rows = conn.execute( "EXPLAIN " + sql, params if params is not None else {} ).fetchall() + # Passing None before these lookups leaves a failing callback installed + # on Python 3.10, so use a permissive callback until they are complete. + conn.set_authorizer(_allow_authorizer_action) + + if not operations: + vacuum_row = next((row for row in explain_rows if row[1] == "Vacuum"), None) + if vacuum_row is not None: + schema_by_index = { + row[0]: row[1] for row in conn.execute("PRAGMA database_list") + } + sqlite_schema = schema_by_index.get(vacuum_row[2]) + database = database_for_schema(sqlite_schema) + record( + "vacuum", + "database", + database=database, + table=None, + sqlite_schema=sqlite_schema, + target=database, + source=None, + ) + else: + record( + "unknown", + "statement", + database=database_name, + table=None, + sqlite_schema=None, + target=None, + source=None, + ) + + for key in operations: + if ( + key.target_type == "table" + and key.operation in {"read", "insert", "update", "delete"} + and key.table is not None + ): + cache_key = (key.sqlite_schema, key.table) + if cache_key not in table_kind_cache: + table_kind_cache[cache_key] = sqlite_table_type( + conn, key.table, schema=key.sqlite_schema + ) finally: conn.set_authorizer(None) - if not operations: - vacuum_row = next((row for row in explain_rows if row[1] == "Vacuum"), None) - if vacuum_row is not None: - schema_by_index = { - row[0]: row[1] for row in conn.execute("PRAGMA database_list") - } - sqlite_schema = schema_by_index.get(vacuum_row[2]) - database = database_for_schema(sqlite_schema) - record( - "vacuum", - "database", - database=database, - table=None, - sqlite_schema=sqlite_schema, - target=database, - source=None, - ) - else: - record( - "unknown", - "statement", - database=database_name, - table=None, - sqlite_schema=None, - target=None, - source=None, - ) - has_schema_operation = any( key.target_type in {"table", "index", "view", "trigger", "virtual-table"} and key.operation in {"create", "alter", "drop"} @@ -502,8 +523,6 @@ def analyze_sql_tables( return True return False - table_kind_cache: dict[tuple[str | None, str], SQLTableKind | None] = {} - def table_kind_for(key: OperationKey) -> SQLTableKind | None: if ( key.target_type != "table" @@ -511,12 +530,7 @@ def analyze_sql_tables( or key.table is None ): return None - cache_key = (key.sqlite_schema, key.table) - if cache_key not in table_kind_cache: - table_kind_cache[cache_key] = sqlite_table_type( - conn, key.table, schema=key.sqlite_schema - ) - return table_kind_cache[cache_key] + return table_kind_cache[(key.sqlite_schema, key.table)] return SQLAnalysis( operations=tuple( diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index d3f52751..5a7c6c38 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -16,7 +16,7 @@ _cached_sqlite_version = None SQLiteTableType = Literal["table", "view", "virtual", "shadow"] _VIRTUAL_TABLE_MODULE_RE = re.compile( r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)", - re.IGNORECASE, + re.IGNORECASE | re.DOTALL, ) _VIRTUAL_TABLE_SHADOW_SUFFIXES = { "fts3": ("_content", "_segdir", "_segments", "_stat", "_docsize"), diff --git a/tests/test_utils.py b/tests/test_utils.py index 90013537..e83eed7a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -279,6 +279,20 @@ def test_sqlite_table_type_detects_attached_database_tables(monkeypatch, use_fal conn.close() +def test_sqlite_hidden_table_names_hides_multiline_content_fts_table(): + conn = utils.sqlite3.connect(":memory:") + try: + conn.executescript(""" + create table searchable(id integer primary key, body text); + create virtual table searchable_fts + using fts5(body, content='searchable', content_rowid='id'); + """) + + assert "searchable_fts" in sqlite_hidden_table_names(conn) + finally: + conn.close() + + @pytest.mark.parametrize( "url,expected", [ From 8bd7e165f465fe057beace2b17d52c0a347819f8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 09:50:56 -0700 Subject: [PATCH 753/767] Refactored for code readability --- datasette/app.py | 5 +- datasette/stored_queries.py | 211 +------------------------ datasette/views/database.py | 3 +- datasette/views/query_helpers.py | 27 ++-- datasette/write_sql.py | 255 +++++++++++++++++++++++++++++++ tests/test_write_sql.py | 59 +++++++ 6 files changed, 339 insertions(+), 221 deletions(-) create mode 100644 datasette/write_sql.py create mode 100644 tests/test_write_sql.py diff --git a/datasette/app.py b/datasette/app.py index 56b89789..e7f34e69 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -42,7 +42,7 @@ from jinja2.exceptions import TemplateNotFound from .events import Event from .column_types import SQLiteType -from . import stored_queries +from . import stored_queries, write_sql from .views import Context from .views.database import ( database_download, @@ -1197,7 +1197,8 @@ class Datasette: async def ensure_query_write_permissions( self, database, sql, *, actor=None, params=None, analysis=None ): - return await stored_queries.ensure_query_write_permissions( + # Raise Forbidden or QueryWriteRejected if SQL should not run + return await write_sql.ensure_query_write_permissions( self, database, sql, actor=actor, params=params, analysis=analysis ) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index b5aea221..b6ac49b8 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -2,26 +2,13 @@ from __future__ import annotations from dataclasses import dataclass import json -from typing import Any, Iterable, TYPE_CHECKING +from typing import Any, Iterable -from .resources import DatabaseResource, TableResource -from .permissions import Resource -from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components -from .utils.asgi import Forbidden -from .utils.sql_analysis import Operation, SQLAnalysis - -if TYPE_CHECKING: - from .app import Datasette +from .utils import tilde_encode, urlsafe_components UNCHANGED = object() -class QueryWriteRejected(Exception): - def __init__(self, message: str): - self.message = message - super().__init__(message) - - QUERY_OPTION_FIELDS = ( "hide_sql", "fragment", @@ -593,197 +580,3 @@ async def list_queries( has_more=has_more, limit=limit, ) - - -@dataclass(frozen=True) -class PermissionRequirement: - action: str - resource: Resource - - -def row_mutation_requirements( - database: str, table: str -) -> tuple[PermissionRequirement, ...]: - resource = TableResource(database=database, table=table) - return tuple( - PermissionRequirement(action=action, resource=resource) - for action in ("insert-row", "update-row", "delete-row") - ) - - -def permission_requirements_for_operation( - operation: Operation, -) -> tuple[PermissionRequirement, ...]: - if ( - operation.operation == "read" - and operation.target_type == "table" - and operation.database is not None - and operation.table is not None - ): - return ( - PermissionRequirement( - action="view-table", - resource=TableResource( - database=operation.database, table=operation.table - ), - ), - ) - if ( - operation.operation in {"insert", "update"} - and operation.target_type == "table" - and operation.database is not None - and operation.table is not None - ): - return row_mutation_requirements( - database=operation.database, - table=operation.table, - ) - if ( - operation.operation == "delete" - and operation.target_type == "table" - and operation.database is not None - and operation.table is not None - ): - return ( - PermissionRequirement( - action="delete-row", - resource=TableResource( - database=operation.database, table=operation.table - ), - ), - ) - if operation.operation == "create" and operation.target_type == "table": - if operation.database is None: - return () - return ( - PermissionRequirement( - action="create-table", - resource=DatabaseResource(database=operation.database), - ), - ) - if ( - operation.operation == "alter" - and operation.target_type == "table" - and operation.database is not None - and operation.table is not None - ): - return ( - PermissionRequirement( - action="alter-table", - resource=TableResource( - database=operation.database, table=operation.table - ), - ), - ) - if ( - operation.operation == "drop" - and operation.target_type == "table" - and operation.database is not None - and operation.table is not None - ): - return ( - PermissionRequirement( - action="drop-table", - resource=TableResource( - database=operation.database, table=operation.table - ), - ), - ) - if ( - operation.operation in {"create", "drop"} - and operation.target_type == "index" - and operation.database is not None - and operation.table is not None - ): - return ( - PermissionRequirement( - action="alter-table", - resource=TableResource( - database=operation.database, table=operation.table - ), - ), - ) - return () - - -def operation_should_be_ignored(operation: Operation) -> bool: - return operation.internal or operation.operation == "select" - - -def operation_forbidden_message(operation: Operation) -> str | None: - if operation.operation == "vacuum": - return "VACUUM is not allowed in user-supplied SQL" - if operation.operation in {"insert", "update", "delete"}: - if operation.table_kind == "virtual": - return "Writes to virtual tables are not allowed in user-supplied SQL" - if operation.table_kind == "shadow": - return "Writes to shadow tables are not allowed in user-supplied SQL" - return None - - -def operation_is_write(operation: Operation) -> bool: - return operation.operation in { - "insert", - "update", - "delete", - "create", - "alter", - "drop", - "begin", - "commit", - "rollback", - "savepoint", - "attach", - "detach", - "pragma", - "analyze", - "reindex", - "vacuum", - "unknown", - } - - -async def ensure_query_write_permissions( - datasette: Datasette, - database: str, - sql: str, - *, - actor: dict[str, object] | None = None, - params: dict[str, object] | None = None, - analysis: SQLAnalysis | None = None, -) -> SQLAnalysis: - db = datasette.get_database(database) - if analysis is None: - if params is None: - params = {name: "" for name in named_parameters(sql)} - try: - analysis = await db.analyze_sql(sql, params) - except sqlite3.DatabaseError as ex: - raise Forbidden(f"Could not analyze query: {ex}") from ex - - for operation in analysis.operations: - if operation_should_be_ignored(operation): - continue - forbidden_message = operation_forbidden_message(operation) - if forbidden_message is not None: - raise QueryWriteRejected(forbidden_message) - permissions = permission_requirements_for_operation(operation) - if not permissions: - raise Forbidden( - "Unsupported SQL operation: {} {}".format( - operation.operation, operation.target_type - ) - ) - if operation.database != database: - raise Forbidden("Writable queries may not access attached databases") - for permission in permissions: - if not await datasette.allowed( - action=permission.action, - resource=permission.resource, - actor=actor, - ): - raise Forbidden( - f"Permission denied: need {permission.action} " - f"on {permission.resource}" - ) - return analysis diff --git a/datasette/views/database.py b/datasette/views/database.py index ae1cf375..b4a964f1 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -13,7 +13,8 @@ import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource -from datasette.stored_queries import QueryWriteRejected, stored_query_to_dict +from datasette.stored_queries import stored_query_to_dict +from datasette.write_sql import QueryWriteRejected from datasette.utils import ( add_cors_headers, await_me_maybe, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 92328ff3..712832e8 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -3,11 +3,14 @@ import re from datasette.resources import DatabaseResource from datasette.stored_queries import ( - QueryWriteRejected, StoredQuery, +) +from datasette.write_sql import ( + IgnoreWriteSqlOperation, + QueryWriteRejected, + RequireWriteSqlPermissions, + decision_for_write_sql_operation, operation_is_write, - operation_should_be_ignored, - permission_requirements_for_operation, ) from datasette.utils import ( named_parameters as derive_named_parameters, @@ -212,7 +215,9 @@ async def _analyze_user_query(datasette, db, sql, *, actor): def _display_operations(analysis: SQLAnalysis) -> list[Operation]: operations = [] for operation in analysis.operations: - if operation_should_be_ignored(operation): + if isinstance( + decision_for_write_sql_operation(operation), IgnoreWriteSqlOperation + ): continue operations.append(operation) return operations @@ -221,8 +226,12 @@ def _display_operations(analysis: SQLAnalysis) -> list[Operation]: def _analysis_rows(analysis: SQLAnalysis) -> list[dict[str, object]]: rows = [] for operation in _display_operations(analysis): - permissions = permission_requirements_for_operation(operation) - required_permission = ", ".join(permission.action for permission in permissions) + decision = decision_for_write_sql_operation(operation) + required_permission = ( + ", ".join(permission.action for permission in decision.permissions) + if isinstance(decision, RequireWriteSqlPermissions) + else "" + ) rows.append( { "operation": operation.operation, @@ -241,10 +250,10 @@ async def _analysis_rows_with_permissions( rows = _analysis_rows(analysis) is_write = _analysis_is_write(analysis) for row, operation in zip(rows, _display_operations(analysis)): - permissions = permission_requirements_for_operation(operation) - if permissions: + decision = decision_for_write_sql_operation(operation) + if isinstance(decision, RequireWriteSqlPermissions): row["allowed"] = True - for permission in permissions: + for permission in decision.permissions: if not await datasette.allowed( action=permission.action, resource=permission.resource, diff --git a/datasette/write_sql.py b/datasette/write_sql.py new file mode 100644 index 00000000..2e1b69af --- /dev/null +++ b/datasette/write_sql.py @@ -0,0 +1,255 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from .permissions import Resource +from .resources import DatabaseResource, TableResource +from .utils import named_parameters, sqlite3 +from .utils.asgi import Forbidden +from .utils.sql_analysis import Operation, SQLAnalysis + +if TYPE_CHECKING: + from .app import Datasette + + +class QueryWriteRejected(Exception): + def __init__(self, message: str): + self.message = message + super().__init__(message) + + +@dataclass(frozen=True) +class PermissionRequirement: + action: str + resource: Resource + + +PermissionRequirements = tuple[PermissionRequirement, ...] + + +class WriteSqlOperationDecision: + """What Datasette should do with one operation in user-supplied write SQL.""" + + +@dataclass(frozen=True) +class IgnoreWriteSqlOperation(WriteSqlOperationDecision): + reason: str + + +@dataclass(frozen=True) +class RequireWriteSqlPermissions(WriteSqlOperationDecision): + permissions: PermissionRequirements + + +@dataclass(frozen=True) +class RejectWriteSqlOperation(WriteSqlOperationDecision): + message: str + + +@dataclass(frozen=True) +class UnsupportedWriteSqlOperation(WriteSqlOperationDecision): + message: str + + +def row_mutation_requirements(database: str, table: str) -> PermissionRequirements: + resource = TableResource(database=database, table=table) + return tuple( + PermissionRequirement(action=action, resource=resource) + for action in ("insert-row", "update-row", "delete-row") + ) + + +def decision_for_write_sql_operation( + operation: Operation, +) -> WriteSqlOperationDecision: + unsupported_message = ( + f"Unsupported SQL operation: {operation.operation} {operation.target_type}" + ) + if operation.internal: + return IgnoreWriteSqlOperation("internal SQLite operation") + if operation.operation == "select": + return IgnoreWriteSqlOperation("select statement") + if operation.operation == "vacuum": + return RejectWriteSqlOperation("VACUUM is not allowed in user-supplied SQL") + if operation.operation in {"insert", "update", "delete"}: + if operation.table_kind == "virtual": + return RejectWriteSqlOperation( + "Writes to virtual tables are not allowed in user-supplied SQL" + ) + if operation.table_kind == "shadow": + return RejectWriteSqlOperation( + "Writes to shadow tables are not allowed in user-supplied SQL" + ) + if operation.operation == "function": + # SQL functions currently have no Datasette permission mapping. They are + # rejected by the user-supplied write SQL allow-list as unsupported. + return UnsupportedWriteSqlOperation(unsupported_message) + if ( + operation.operation == "read" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="view-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) + if ( + operation.operation in {"insert", "update"} + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + row_mutation_requirements( + database=operation.database, + table=operation.table, + ) + ) + if ( + operation.operation == "delete" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="delete-row", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) + if operation.operation == "create" and operation.target_type == "table": + if operation.database is None: + return UnsupportedWriteSqlOperation(unsupported_message) + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="create-table", + resource=DatabaseResource(database=operation.database), + ), + ) + ) + if ( + operation.operation == "alter" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="alter-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) + if ( + operation.operation == "drop" + and operation.target_type == "table" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="drop-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) + if ( + operation.operation in {"create", "drop"} + and operation.target_type == "index" + and operation.database is not None + and operation.table is not None + ): + return RequireWriteSqlPermissions( + ( + PermissionRequirement( + action="alter-table", + resource=TableResource( + database=operation.database, table=operation.table + ), + ), + ) + ) + return UnsupportedWriteSqlOperation(unsupported_message) + + +def operation_is_write(operation: Operation) -> bool: + return operation.operation in { + "insert", + "update", + "delete", + "create", + "alter", + "drop", + "begin", + "commit", + "rollback", + "savepoint", + "attach", + "detach", + "pragma", + "analyze", + "reindex", + "vacuum", + "unknown", + } + + +async def ensure_query_write_permissions( + datasette: Datasette, + database: str, + sql: str, + *, + actor: dict[str, object] | None = None, + params: dict[str, object] | None = None, + analysis: SQLAnalysis | None = None, +) -> SQLAnalysis: + db = datasette.get_database(database) + if analysis is None: + if params is None: + params = {name: "" for name in named_parameters(sql)} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise Forbidden(f"Could not analyze query: {ex}") from ex + + for operation in analysis.operations: + decision = decision_for_write_sql_operation(operation) + if isinstance(decision, IgnoreWriteSqlOperation): + continue + if isinstance(decision, RejectWriteSqlOperation): + raise QueryWriteRejected(decision.message) + if isinstance(decision, UnsupportedWriteSqlOperation): + raise Forbidden(decision.message) + permissions = decision.permissions + if operation.database != database: + raise Forbidden("Writable queries may not access attached databases") + for permission in permissions: + if not await datasette.allowed( + action=permission.action, + resource=permission.resource, + actor=actor, + ): + raise Forbidden( + f"Permission denied: need {permission.action} " + f"on {permission.resource}" + ) + return analysis diff --git a/tests/test_write_sql.py b/tests/test_write_sql.py new file mode 100644 index 00000000..cfaf0f53 --- /dev/null +++ b/tests/test_write_sql.py @@ -0,0 +1,59 @@ +from datasette.utils.sql_analysis import Operation +from datasette.write_sql import ( + IgnoreWriteSqlOperation, + RejectWriteSqlOperation, + RequireWriteSqlPermissions, + UnsupportedWriteSqlOperation, + WriteSqlOperationDecision, + decision_for_write_sql_operation, +) + + +def test_decision_for_write_sql_operation_ignores_internal_and_select_operations(): + internal_decision = decision_for_write_sql_operation( + Operation("read", "schema", None, None, "main", internal=True) + ) + select_decision = decision_for_write_sql_operation( + Operation("select", "statement", None, None, None) + ) + + assert isinstance(internal_decision, IgnoreWriteSqlOperation) + assert isinstance(internal_decision, WriteSqlOperationDecision) + assert isinstance(select_decision, IgnoreWriteSqlOperation) + assert isinstance(select_decision, WriteSqlOperationDecision) + + +def test_decision_for_write_sql_operation_requires_table_write_permissions(): + decision = decision_for_write_sql_operation( + Operation("insert", "table", "data", "dogs", None) + ) + + assert isinstance(decision, RequireWriteSqlPermissions) + assert [permission.action for permission in decision.permissions] == [ + "insert-row", + "update-row", + "delete-row", + ] + assert [str(permission.resource) for permission in decision.permissions] == [ + "data/dogs", + "data/dogs", + "data/dogs", + ] + + +def test_decision_for_write_sql_operation_rejects_vacuum(): + decision = decision_for_write_sql_operation( + Operation("vacuum", "statement", None, None, None) + ) + + assert isinstance(decision, RejectWriteSqlOperation) + assert decision.message == "VACUUM is not allowed in user-supplied SQL" + + +def test_decision_for_write_sql_operation_reports_unsupported_functions(): + decision = decision_for_write_sql_operation( + Operation("function", "function", None, None, None, target="upper") + ) + + assert isinstance(decision, UnsupportedWriteSqlOperation) + assert decision.message == "Unsupported SQL operation: function function" From 51dab16149f8b345d46cf517fa03b95fc1028234 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 10:22:16 -0700 Subject: [PATCH 754/767] Allow SQL functions in SQL write queries Closes #2751 --- datasette/write_sql.py | 4 +- docs/authentication.rst | 2 +- docs/json_api.rst | 2 +- docs/sql_queries.rst | 2 +- tests/test_queries.py | 83 +++++++++++++++++++++++++++++++++++++---- tests/test_write_sql.py | 13 ++++++- 6 files changed, 91 insertions(+), 15 deletions(-) diff --git a/datasette/write_sql.py b/datasette/write_sql.py index 2e1b69af..cdc0c6d3 100644 --- a/datasette/write_sql.py +++ b/datasette/write_sql.py @@ -82,9 +82,7 @@ def decision_for_write_sql_operation( "Writes to shadow tables are not allowed in user-supplied SQL" ) if operation.operation == "function": - # SQL functions currently have no Datasette permission mapping. They are - # rejected by the user-supplied write SQL allow-list as unsupported. - return UnsupportedWriteSqlOperation(unsupported_message) + return IgnoreWriteSqlOperation("SQL function") if ( operation.operation == "read" and operation.target_type == "table" diff --git a/docs/authentication.rst b/docs/authentication.rst index f720c12f..a0891900 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1425,7 +1425,7 @@ See also :ref:`the default_allow_sql setting `. execute-write-sql ----------------- -Actor is allowed to run arbitrary writable SQL queries against a specific database, subject to table-level write permissions such as ``insert-row``, ``update-row`` and ``delete-row``. +Actor is allowed to run arbitrary writable SQL queries against a specific database, subject to table-level write permissions such as ``insert-row``, ``update-row`` and ``delete-row``. SQL functions are allowed and are not separately restricted by Datasette permissions. ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) diff --git a/docs/json_api.rst b/docs/json_api.rst index fffc16d7..d502299e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -531,7 +531,7 @@ The request body must include a ``"sql"`` string. Named SQL parameters can be pr The SQL must be writable. Read-only ``select`` queries should use the regular :ref:`custom SQL query API ` instead. -Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. +Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. SQL functions are allowed and are not separately restricted by Datasette permissions. A successful response includes a message, the SQLite ``rowcount`` and a summary of the operations that were executed: diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index f593a534..d427ea2b 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -140,7 +140,7 @@ Datasette stores both configured queries and user-created queries in the ``queri Stored queries created by users default to private. Private stored queries can only be viewed, updated or deleted by the actor that created them. Broad ``view-query``, ``update-query`` or ``delete-query`` permission grants still do not allow other actors to access another actor's private stored queries. -Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. +Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. SQL functions are allowed and are not separately restricted by Datasette permissions. .. _trusted_stored_queries: .. _trusted_saved_queries: diff --git a/tests/test_queries.py b/tests/test_queries.py index 73f8f3cf..9c3ebcc8 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1414,6 +1414,11 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): actor={"id": "root"}, params={"sql": "insert into dogs (name) values (:name)"}, ) + function_response = await ds.client.get( + "/data/-/execute-write/analyze", + actor={"id": "root"}, + params={"sql": "insert into dogs (name) values (upper(:name))"}, + ) read_only_response = await ds.client.get( "/data/-/execute-write/analyze", actor={"id": "root"}, @@ -1438,6 +1443,22 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): ] assert "params" not in data + assert function_response.status_code == 200 + function_data = function_response.json() + assert function_data["ok"] is True + assert function_data["parameters"] == ["name"] + assert function_data["execute_disabled"] is False + assert function_data["analysis_rows"] == [ + { + "operation": "insert", + "database": "data", + "table": "dogs", + "required_permission": "insert-row, update-row, delete-row", + "source": None, + "allowed": True, + } + ] + assert read_only_response.status_code == 200 read_only_data = read_only_response.json() assert read_only_data["ok"] is False @@ -1970,7 +1991,7 @@ async def test_execute_write_create_table_as_select_requires_view_table_on_sourc @pytest.mark.asyncio -async def test_execute_write_rejects_function_operations(): +async def test_execute_write_allows_function_operations(): ds = Datasette( memory=True, default_deny=True, @@ -1998,17 +2019,65 @@ async def test_execute_write_rejects_function_operations(): await db.execute_write("create table dogs (id integer primary key, name text)") await ds.invoke_startup() - denied_response = await ds.client.post( + response = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, json={"sql": "insert into dogs (name) values (upper('cleo'))"}, ) - assert denied_response.status_code == 403 - assert denied_response.json()["errors"] == [ - "Unsupported SQL operation: function function" - ] - assert (await db.execute("select name from dogs")).dicts() == [] + assert response.status_code == 200 + assert response.json()["ok"] is True + assert (await db.execute("select name from dogs")).dicts() == [{"name": "CLEO"}] + + +@pytest.mark.asyncio +async def test_untrusted_stored_write_query_allows_function_operations(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "view-query": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "dogs": { + "permissions": { + "insert-row": {"id": "writer"}, + "update-row": {"id": "writer"}, + "delete-row": {"id": "writer"}, + } + } + }, + } + } + }, + ) + db = ds.add_memory_database("stored_query_function_operation", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "insert_dog", + "insert into dogs (name) values (upper(:name))", + is_write=True, + is_trusted=False, + source="user", + owner_id="writer", + ) + + response = await ds.client.post( + "/data/insert_dog?_json=1", + actor={"id": "writer"}, + data={"name": "cleo"}, + ) + + assert response.status_code == 200 + assert response.json()["ok"] is True + assert (await db.execute("select name from dogs")).dicts() == [{"name": "CLEO"}] @pytest.mark.asyncio diff --git a/tests/test_write_sql.py b/tests/test_write_sql.py index cfaf0f53..6d95c3c4 100644 --- a/tests/test_write_sql.py +++ b/tests/test_write_sql.py @@ -50,10 +50,19 @@ def test_decision_for_write_sql_operation_rejects_vacuum(): assert decision.message == "VACUUM is not allowed in user-supplied SQL" -def test_decision_for_write_sql_operation_reports_unsupported_functions(): +def test_decision_for_write_sql_operation_ignores_functions(): decision = decision_for_write_sql_operation( Operation("function", "function", None, None, None, target="upper") ) + assert isinstance(decision, IgnoreWriteSqlOperation) + assert decision.reason == "SQL function" + + +def test_decision_for_write_sql_operation_reports_unsupported_operations(): + decision = decision_for_write_sql_operation( + Operation("unknown", "unknown", None, None, None) + ) + assert isinstance(decision, UnsupportedWriteSqlOperation) - assert decision.message == "Unsupported SQL operation: function function" + assert decision.message == "Unsupported SQL operation: unknown unknown" From b2b20b36c52ea446fb05fe688b636b83d187e6a6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 10:24:40 -0700 Subject: [PATCH 755/767] Document write SQL analyzer restrictions Expand the unreleased changelog with the deny-by-default operation analysis model, SQL function handling, and the VACUUM and virtual/shadow table restrictions for user-supplied write SQL. Clarify the /-/execute-write JSON API documentation with the same restrictions and DDL permission requirements. --- docs/changelog.rst | 2 ++ docs/json_api.rst | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2ba713ee..a4be98b1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,8 @@ Write SQL UI - New "Write to this database" interface at ``//-/execute-write`` for running arbitrary writable SQL against mutable databases. The form extracts named parameters, analyzes the SQL, shows the table operations that will be attempted and links to a newly inserted row when a single-row insert succeeds. (:issue:`2742`) - Added the new :ref:`execute-write-sql ` permission for running arbitrary writable SQL. Execution is also gated by table-level permissions such as :ref:`insert-row `, :ref:`update-row ` and :ref:`delete-row `, and writes to attached databases are rejected. (:issue:`2742`) +- The write SQL analyzer now uses a deny-by-default model for unsupported operations. Reads from source tables require :ref:`view-table ` permission, schema changes require :ref:`create-table `, :ref:`alter-table ` or :ref:`drop-table ` as appropriate, and row mutation statements require the full ``insert-row``, ``update-row`` and ``delete-row`` permission set. SQL functions are allowed and are not separately permission-gated. (:issue:`2748`) +- User-supplied write SQL now rejects ``VACUUM`` and writes to SQLite virtual tables or shadow tables. These restrictions also apply to untrusted stored write queries; trusted configured stored queries continue to skip these filters. (:issue:`2748`) Plugin API changes ~~~~~~~~~~~~~~~~~~ diff --git a/docs/json_api.rst b/docs/json_api.rst index d502299e..db19afc2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -531,7 +531,9 @@ The request body must include a ``"sql"`` string. Named SQL parameters can be pr The SQL must be writable. Read-only ``select`` queries should use the regular :ref:`custom SQL query API ` instead. -Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. SQL functions are allowed and are not separately restricted by Datasette permissions. +Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. Schema changes require ``create-table``, ``alter-table`` or ``drop-table`` permissions as appropriate. + +Unsupported SQL operations are rejected by default. ``VACUUM`` is not allowed in arbitrary write SQL, and writes to SQLite virtual tables or shadow tables are rejected. SQL functions are allowed and are not separately restricted by Datasette permissions. A successful response includes a message, the SQLite ``rowcount`` and a summary of the operations that were executed: From cbe9594a3dcac1f91a6baa7ac99a138c22a71a8a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 11:00:04 -0700 Subject: [PATCH 756/767] Use SQLiteTableType directly in SQL analysis Remove the redundant SQLTableKind alias from the write SQL analysis model. Operation.table_kind and the analyzer cache now use the SQLite metadata classification type directly, making the source of table-kind values clearer. --- datasette/utils/sql_analysis.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py index b5d7ada8..0a3a947c 100644 --- a/datasette/utils/sql_analysis.py +++ b/datasette/utils/sql_analysis.py @@ -42,7 +42,6 @@ SQLTargetType = Literal[ SQLTableOperation = Literal["read", "insert", "update", "delete"] SQLSchemaOperation = Literal["create", "drop"] SQLSchemaTargetType = Literal["index", "table", "trigger", "view", "virtual-table"] -SQLTableKind = SQLiteTableType @dataclass(frozen=True) @@ -52,7 +51,7 @@ class Operation: database: str | None table: str | None sqlite_schema: str | None - table_kind: SQLTableKind | None = None + table_kind: SQLiteTableType | None = None target: str | None = None columns: tuple[str, ...] = () source: str | None = None @@ -428,7 +427,7 @@ def analyze_sql_tables( ) return sqlite3.SQLITE_OK - table_kind_cache: dict[tuple[str | None, str], SQLTableKind | None] = {} + table_kind_cache: dict[tuple[str | None, str], SQLiteTableType | None] = {} conn.set_authorizer(authorizer) try: @@ -523,7 +522,7 @@ def analyze_sql_tables( return True return False - def table_kind_for(key: OperationKey) -> SQLTableKind | None: + def table_kind_for(key: OperationKey) -> SQLiteTableType | None: if ( key.target_type != "table" or key.operation not in {"read", "insert", "update", "delete"} From 17f45b884b4b4844e9f0cce0fef402e888c690f0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 12:06:57 -0700 Subject: [PATCH 757/767] Clarify ignored write SQL operation tests Split the combined ignored-operation decision test into separate internal-operation and select-statement cases. Assert the decision reason for each case instead of checking the shared base class, so the tests document why those operations are ignored. --- tests/test_write_sql.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/test_write_sql.py b/tests/test_write_sql.py index 6d95c3c4..75d6b6e1 100644 --- a/tests/test_write_sql.py +++ b/tests/test_write_sql.py @@ -4,23 +4,26 @@ from datasette.write_sql import ( RejectWriteSqlOperation, RequireWriteSqlPermissions, UnsupportedWriteSqlOperation, - WriteSqlOperationDecision, decision_for_write_sql_operation, ) -def test_decision_for_write_sql_operation_ignores_internal_and_select_operations(): - internal_decision = decision_for_write_sql_operation( +def test_decision_for_write_sql_operation_ignores_internal_operations(): + decision = decision_for_write_sql_operation( Operation("read", "schema", None, None, "main", internal=True) ) - select_decision = decision_for_write_sql_operation( + + assert isinstance(decision, IgnoreWriteSqlOperation) + assert decision.reason == "internal SQLite operation" + + +def test_decision_for_write_sql_operation_ignores_select_statement_operations(): + decision = decision_for_write_sql_operation( Operation("select", "statement", None, None, None) ) - assert isinstance(internal_decision, IgnoreWriteSqlOperation) - assert isinstance(internal_decision, WriteSqlOperationDecision) - assert isinstance(select_decision, IgnoreWriteSqlOperation) - assert isinstance(select_decision, WriteSqlOperationDecision) + assert isinstance(decision, IgnoreWriteSqlOperation) + assert decision.reason == "select statement" def test_decision_for_write_sql_operation_requires_table_write_permissions(): From 0b7c26c6c8bf4827c02aba9707b1db0eb63aeaa5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 12:09:02 -0700 Subject: [PATCH 758/767] Refactored write decision tests --- tests/test_write_sql.py | 71 ---------------- tests/test_write_sql_operation_decisions.py | 94 +++++++++++++++++++++ 2 files changed, 94 insertions(+), 71 deletions(-) delete mode 100644 tests/test_write_sql.py create mode 100644 tests/test_write_sql_operation_decisions.py diff --git a/tests/test_write_sql.py b/tests/test_write_sql.py deleted file mode 100644 index 75d6b6e1..00000000 --- a/tests/test_write_sql.py +++ /dev/null @@ -1,71 +0,0 @@ -from datasette.utils.sql_analysis import Operation -from datasette.write_sql import ( - IgnoreWriteSqlOperation, - RejectWriteSqlOperation, - RequireWriteSqlPermissions, - UnsupportedWriteSqlOperation, - decision_for_write_sql_operation, -) - - -def test_decision_for_write_sql_operation_ignores_internal_operations(): - decision = decision_for_write_sql_operation( - Operation("read", "schema", None, None, "main", internal=True) - ) - - assert isinstance(decision, IgnoreWriteSqlOperation) - assert decision.reason == "internal SQLite operation" - - -def test_decision_for_write_sql_operation_ignores_select_statement_operations(): - decision = decision_for_write_sql_operation( - Operation("select", "statement", None, None, None) - ) - - assert isinstance(decision, IgnoreWriteSqlOperation) - assert decision.reason == "select statement" - - -def test_decision_for_write_sql_operation_requires_table_write_permissions(): - decision = decision_for_write_sql_operation( - Operation("insert", "table", "data", "dogs", None) - ) - - assert isinstance(decision, RequireWriteSqlPermissions) - assert [permission.action for permission in decision.permissions] == [ - "insert-row", - "update-row", - "delete-row", - ] - assert [str(permission.resource) for permission in decision.permissions] == [ - "data/dogs", - "data/dogs", - "data/dogs", - ] - - -def test_decision_for_write_sql_operation_rejects_vacuum(): - decision = decision_for_write_sql_operation( - Operation("vacuum", "statement", None, None, None) - ) - - assert isinstance(decision, RejectWriteSqlOperation) - assert decision.message == "VACUUM is not allowed in user-supplied SQL" - - -def test_decision_for_write_sql_operation_ignores_functions(): - decision = decision_for_write_sql_operation( - Operation("function", "function", None, None, None, target="upper") - ) - - assert isinstance(decision, IgnoreWriteSqlOperation) - assert decision.reason == "SQL function" - - -def test_decision_for_write_sql_operation_reports_unsupported_operations(): - decision = decision_for_write_sql_operation( - Operation("unknown", "unknown", None, None, None) - ) - - assert isinstance(decision, UnsupportedWriteSqlOperation) - assert decision.message == "Unsupported SQL operation: unknown unknown" diff --git a/tests/test_write_sql_operation_decisions.py b/tests/test_write_sql_operation_decisions.py new file mode 100644 index 00000000..cc19f701 --- /dev/null +++ b/tests/test_write_sql_operation_decisions.py @@ -0,0 +1,94 @@ +import pytest + +from datasette.utils.sql_analysis import Operation +from datasette.write_sql import ( + IgnoreWriteSqlOperation, + RejectWriteSqlOperation, + RequireWriteSqlPermissions, + UnsupportedWriteSqlOperation, + decision_for_write_sql_operation, +) + + +@pytest.mark.parametrize( + ("operation", "reason"), + ( + pytest.param( + Operation("read", "schema", None, None, "main", internal=True), + "internal SQLite operation", + id="internal", + ), + pytest.param( + Operation("select", "statement", None, None, None), + "select statement", + id="select-statement", + ), + pytest.param( + Operation("function", "function", None, None, None, target="upper"), + "SQL function", + id="function", + ), + ), +) +def test_decision_for_write_sql_operation_ignores_operations(operation, reason): + decision = decision_for_write_sql_operation(operation) + + assert isinstance(decision, IgnoreWriteSqlOperation) + assert decision.reason == reason + + +@pytest.mark.parametrize("operation", ("insert", "update")) +def test_decision_for_write_sql_operation_requires_table_write_permissions(operation): + decision = decision_for_write_sql_operation( + Operation(operation, "table", "data", "dogs", None) + ) + + assert isinstance(decision, RequireWriteSqlPermissions) + assert [permission.action for permission in decision.permissions] == [ + "insert-row", + "update-row", + "delete-row", + ] + assert [str(permission.resource) for permission in decision.permissions] == [ + "data/dogs", + "data/dogs", + "data/dogs", + ] + + +@pytest.mark.parametrize( + ("operation", "message"), + ( + pytest.param( + Operation("vacuum", "statement", None, None, None), + "VACUUM is not allowed in user-supplied SQL", + id="vacuum", + ), + pytest.param( + Operation("insert", "table", "data", "docs", None, table_kind="virtual"), + "Writes to virtual tables are not allowed in user-supplied SQL", + id="virtual-table", + ), + pytest.param( + Operation( + "insert", "table", "data", "docs_data", None, table_kind="shadow" + ), + "Writes to shadow tables are not allowed in user-supplied SQL", + id="shadow-table", + ), + ), +) +def test_decision_for_write_sql_operation_rejects_operations(operation, message): + decision = decision_for_write_sql_operation(operation) + + assert isinstance(decision, RejectWriteSqlOperation) + assert decision.message == message + + +def test_decision_for_write_sql_operation_reports_unsupported_operations(): + decision = decision_for_write_sql_operation( + Operation("unknown", "unknown", None, None, None) + ) + + assert isinstance(decision, UnsupportedWriteSqlOperation) + assert decision.message == "Unsupported SQL operation: unknown unknown" From cd838daef4d066e584b047164d8e2a5e96909511 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 15:22:21 -0700 Subject: [PATCH 759/767] Refactor tests a bit --- tests/test_queries.py | 449 +++++++++++++++++++++--------------------- 1 file changed, 225 insertions(+), 224 deletions(-) diff --git a/tests/test_queries.py b/tests/test_queries.py index 9c3ebcc8..216cb211 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1700,8 +1700,22 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.parametrize( + "database_name, sql", + ( + ( + "execute_write_insert_or_replace", + "insert or replace into users(id, email) values (3, 'b@example.com')", + ), + ( + "execute_write_update_or_replace", + "update or replace users set email = 'b@example.com' where id = 1", + ), + ), + ids=("insert-or-replace", "update-or-replace"), +) @pytest.mark.asyncio -async def test_execute_write_insert_or_replace_requires_delete_row_permission(): +async def test_execute_write_replace_requires_delete_row_permission(database_name, sql): ds = Datasette( memory=True, default_deny=True, @@ -1725,7 +1739,7 @@ async def test_execute_write_insert_or_replace_requires_delete_row_permission(): } }, ) - db = ds.add_memory_database("execute_write_insert_or_replace", name="data") + db = ds.add_memory_database(database_name, name="data") await db.execute_write( "create table users (id integer primary key, email text unique)" ) @@ -1738,64 +1752,7 @@ async def test_execute_write_insert_or_replace_requires_delete_row_permission(): denied_response = await ds.client.post( "/data/-/execute-write", actor={"id": "writer"}, - json={ - "sql": ( - "insert or replace into users(id, email) " "values (3, 'b@example.com')" - ) - }, - ) - - assert denied_response.status_code == 403 - assert denied_response.json()["errors"] == [ - "Permission denied: need delete-row on data/users" - ] - assert (await db.execute("select id, email from users order by id")).dicts() == [ - {"id": 1, "email": "a@example.com"}, - {"id": 2, "email": "b@example.com"}, - ] - - -@pytest.mark.asyncio -async def test_execute_write_update_or_replace_requires_delete_row_permission(): - ds = Datasette( - memory=True, - default_deny=True, - config={ - "databases": { - "data": { - "permissions": { - "view-database": {"id": "writer"}, - "execute-write-sql": {"id": "writer"}, - }, - "tables": { - "users": { - "permissions": { - "insert-row": {"id": "writer"}, - "update-row": {"id": "writer"}, - "view-table": {"id": "writer"}, - } - } - }, - } - } - }, - ) - db = ds.add_memory_database("execute_write_update_or_replace", name="data") - await db.execute_write( - "create table users (id integer primary key, email text unique)" - ) - await db.execute_write( - "insert into users (id, email) values " - "(1, 'a@example.com'), (2, 'b@example.com')" - ) - await ds.invoke_startup() - - denied_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "writer"}, - json={ - "sql": "update or replace users set email = 'b@example.com' where id = 1" - }, + json={"sql": sql}, ) assert denied_response.status_code == 403 @@ -2262,74 +2219,71 @@ async def test_trusted_stored_write_query_skips_vacuum_filtering(): assert response.json()["ok"] is True +@pytest.mark.parametrize( + ( + "database_name", + "setup_sqls", + "write_sql", + "expected_error", + "verification_sql", + "expected_count", + ), + ( + ( + "execute_write_virtual_table_control", + ( + "create virtual table docs using fts5(title, body, content='')", + "insert into docs(rowid, title, body) values (1, 'hello', 'world')", + ), + "insert into docs(docs) values('delete-all')", + "Writes to virtual tables are not allowed in user-supplied SQL", + "select count(*) from docs where docs match 'hello'", + 1, + ), + ( + "execute_write_virtual_table_insert", + ("create virtual table docs using fts5(title, body)",), + "insert into docs(rowid, title, body) values (1, 'a', 'b')", + "Writes to virtual tables are not allowed in user-supplied SQL", + "select count(*) from docs", + 0, + ), + ( + "execute_write_shadow_table_insert", + ("create virtual table docs using fts5(title, body)",), + "insert into docs_config(k, v) values ('x', 1)", + "Writes to shadow tables are not allowed in user-supplied SQL", + "select count(*) from docs_config", + 1, + ), + ), + ids=("control-insert", "virtual-table", "shadow-table"), +) @pytest.mark.asyncio -async def test_execute_write_rejects_virtual_table_control_insert(): +async def test_execute_write_rejects_virtual_and_shadow_table_writes( + database_name, + setup_sqls, + write_sql, + expected_error, + verification_sql, + expected_count, +): ds = Datasette(memory=True, default_deny=True) ds.root_enabled = True - db = ds.add_memory_database("execute_write_virtual_table_control", name="data") - await db.execute_write(""" - create virtual table docs using fts5(title, body, content='') - """) - await db.execute_write(""" - insert into docs(rowid, title, body) values (1, 'hello', 'world') - """) + db = ds.add_memory_database(database_name, name="data") + for setup_sql in setup_sqls: + await db.execute_write(setup_sql) await ds.invoke_startup() denied_response = await ds.client.post( "/data/-/execute-write", actor={"id": "root"}, - json={"sql": "insert into docs(docs) values('delete-all')"}, + json={"sql": write_sql}, ) assert denied_response.status_code == 403 - assert denied_response.json()["errors"] == [ - "Writes to virtual tables are not allowed in user-supplied SQL" - ] - assert ( - await db.execute("select count(*) from docs where docs match 'hello'") - ).first()[0] == 1 - - -@pytest.mark.asyncio -async def test_execute_write_rejects_regular_virtual_table_insert(): - ds = Datasette(memory=True, default_deny=True) - ds.root_enabled = True - db = ds.add_memory_database("execute_write_virtual_table_insert", name="data") - await db.execute_write("create virtual table docs using fts5(title, body)") - await ds.invoke_startup() - - denied_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "root"}, - json={"sql": "insert into docs(rowid, title, body) values (1, 'a', 'b')"}, - ) - - assert denied_response.status_code == 403 - assert denied_response.json()["errors"] == [ - "Writes to virtual tables are not allowed in user-supplied SQL" - ] - assert (await db.execute("select count(*) from docs")).first()[0] == 0 - - -@pytest.mark.asyncio -async def test_execute_write_rejects_shadow_table_insert(): - ds = Datasette(memory=True, default_deny=True) - ds.root_enabled = True - db = ds.add_memory_database("execute_write_shadow_table_insert", name="data") - await db.execute_write("create virtual table docs using fts5(title, body)") - await ds.invoke_startup() - - denied_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "root"}, - json={"sql": "insert into docs_config(k, v) values ('x', 1)"}, - ) - - assert denied_response.status_code == 403 - assert denied_response.json()["errors"] == [ - "Writes to shadow tables are not allowed in user-supplied SQL" - ] - assert (await db.execute("select count(*) from docs_config")).first()[0] == 1 + assert denied_response.json()["errors"] == [expected_error] + assert (await db.execute(verification_sql)).first()[0] == expected_count @pytest.mark.asyncio @@ -2482,8 +2436,69 @@ async def test_execute_write_create_table_uses_create_table_permission(): assert not await db.table_exists("should_not_exist") +@pytest.mark.parametrize( + ( + "database_name", + "allowed_actor", + "allowed_sql", + "denied_sql", + "expected_error", + "setup_sqls", + "expected_state", + ), + ( + ( + "execute_write_alter_table", + "alterer", + "alter table dogs add column age integer", + "alter table cats add column age integer", + "Permission denied: need alter-table on data/cats", + (), + "alter-table", + ), + ( + "execute_write_create_index", + "alterer", + "create index idx_dogs_name on dogs(name)", + "create index idx_cats_name on cats(name)", + "Permission denied: need alter-table on data/cats", + (), + "create-index", + ), + ( + "execute_write_drop_index", + "alterer", + "drop index idx_dogs_name", + "drop index idx_cats_name", + "Permission denied: need alter-table on data/cats", + ( + "create index idx_dogs_name on dogs(name)", + "create index idx_cats_name on cats(name)", + ), + "drop-index", + ), + ( + "execute_write_drop_table", + "dropper", + "drop table dogs", + "drop table cats", + "Permission denied: need drop-table on data/cats", + (), + "drop-table", + ), + ), + ids=("alter-table", "create-index", "drop-index", "drop-table"), +) @pytest.mark.asyncio -async def test_execute_write_alter_and_drop_table_use_schema_permissions(): +async def test_execute_write_schema_operations_use_schema_permissions( + database_name, + allowed_actor, + allowed_sql, + denied_sql, + expected_error, + setup_sqls, + expected_state, +): ds = Datasette( memory=True, default_deny=True, @@ -2513,73 +2528,53 @@ async def test_execute_write_alter_and_drop_table_use_schema_permissions(): }, }, ) - db = ds.add_memory_database("execute_write_alter_drop_table", name="data") + db = ds.add_memory_database(database_name, name="data") await db.execute_write("create table dogs (id integer primary key, name text)") await db.execute_write("create table cats (id integer primary key, name text)") + for setup_sql in setup_sqls: + await db.execute_write(setup_sql) await ds.invoke_startup() - alter_allowed_response = await ds.client.post( + async def index_exists(index_name): + row = ( + await db.execute( + "select 1 from sqlite_master where type = 'index' and name = ?", + [index_name], + ) + ).first() + return row is not None + + allowed_response = await ds.client.post( "/data/-/execute-write", - actor={"id": "alterer"}, - json={"sql": "alter table dogs add column age integer"}, + actor={"id": allowed_actor}, + json={"sql": allowed_sql}, ) - alter_row_permission_response = await ds.client.post( + denied_response = await ds.client.post( "/data/-/execute-write", actor={"id": "row-writer"}, - json={"sql": "alter table cats add column age integer"}, + json={"sql": denied_sql}, ) - assert alter_allowed_response.status_code == 200 - assert "age" in [column.name for column in await db.table_column_details("dogs")] - assert alter_row_permission_response.status_code == 403 - assert alter_row_permission_response.json()["errors"] == [ - "Permission denied: need alter-table on data/cats" - ] - assert "age" not in [ - column.name for column in await db.table_column_details("cats") - ] + assert allowed_response.status_code == 200 + assert denied_response.status_code == 403 + assert denied_response.json()["errors"] == [expected_error] - create_index_allowed_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "alterer"}, - json={"sql": "create index idx_dogs_name on dogs(name)"}, - ) - create_index_row_permission_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "row-writer"}, - json={"sql": "create index idx_cats_name on cats(name)"}, - ) - drop_index_allowed_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "alterer"}, - json={"sql": "drop index idx_dogs_name"}, - ) - - assert create_index_allowed_response.status_code == 200 - assert create_index_row_permission_response.status_code == 403 - assert create_index_row_permission_response.json()["errors"] == [ - "Permission denied: need alter-table on data/cats" - ] - assert drop_index_allowed_response.status_code == 200 - - drop_allowed_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "dropper"}, - json={"sql": "drop table dogs"}, - ) - drop_row_permission_response = await ds.client.post( - "/data/-/execute-write", - actor={"id": "row-writer"}, - json={"sql": "drop table cats"}, - ) - - assert drop_allowed_response.status_code == 200 - assert not await db.table_exists("dogs") - assert drop_row_permission_response.status_code == 403 - assert drop_row_permission_response.json()["errors"] == [ - "Permission denied: need drop-table on data/cats" - ] - assert await db.table_exists("cats") + if expected_state == "alter-table": + assert "age" in [ + column.name for column in await db.table_column_details("dogs") + ] + assert "age" not in [ + column.name for column in await db.table_column_details("cats") + ] + elif expected_state == "create-index": + assert await index_exists("idx_dogs_name") + assert not await index_exists("idx_cats_name") + elif expected_state == "drop-index": + assert not await index_exists("idx_dogs_name") + assert await index_exists("idx_cats_name") + elif expected_state == "drop-table": + assert not await db.table_exists("dogs") + assert await db.table_exists("cats") @pytest.mark.asyncio @@ -2644,8 +2639,9 @@ async def test_execute_write_post_rejects_read_only_sql(): ] +@pytest.mark.parametrize("action", ("view-query", "update-query", "delete-query")) @pytest.mark.asyncio -async def test_query_owner_gets_update_delete_and_writable_view_defaults(): +async def test_query_owner_gets_update_delete_and_writable_view_defaults(action): ds = Datasette(memory=True, default_deny=True) ds.add_memory_database("query_owner_defaults", name="data") await ds.invoke_startup() @@ -2658,21 +2654,35 @@ async def test_query_owner_gets_update_delete_and_writable_view_defaults(): owner_id="alice", ) - for action in ("view-query", "update-query", "delete-query"): - assert await ds.allowed( - action=action, - resource=QueryResource("data", "insert_dog"), - actor={"id": "alice"}, - ) - assert not await ds.allowed( - action=action, - resource=QueryResource("data", "insert_dog"), - actor={"id": "bob"}, - ) + assert await ds.allowed( + action=action, + resource=QueryResource("data", "insert_dog"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action=action, + resource=QueryResource("data", "insert_dog"), + actor={"id": "bob"}, + ) +@pytest.mark.parametrize( + "action, path_suffix, request_json, expected_public_title", + ( + ( + "update-query", + "-/update", + {"update": {"title": "Bob can edit public queries"}}, + "Bob can edit public queries", + ), + ("delete-query", "-/delete", {}, None), + ), + ids=("update-query", "delete-query"), +) @pytest.mark.asyncio -async def test_private_query_restricts_broad_update_delete_permissions(): +async def test_private_query_restricts_broad_update_delete_permissions( + action, path_suffix, request_json, expected_public_title +): ds = Datasette( memory=True, default_deny=True, @@ -2706,50 +2716,41 @@ async def test_private_query_restricts_broad_update_delete_permissions(): owner_id="alice", ) - for action in ("update-query", "delete-query"): - assert await ds.allowed( - action=action, - resource=QueryResource("data", "alice_private"), - actor={"id": "alice"}, - ) - assert not await ds.allowed( - action=action, - resource=QueryResource("data", "alice_private"), - actor={"id": "bob"}, - ) - assert await ds.allowed( - action=action, - resource=QueryResource("data", "alice_public"), - actor={"id": "bob"}, - ) - - private_update_response = await ds.client.post( - "/data/alice_private/-/update", - actor={"id": "bob"}, - json={"update": {"title": "Nope"}}, + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), + actor={"id": "alice"}, ) - private_delete_response = await ds.client.post( - "/data/alice_private/-/delete", + assert not await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), actor={"id": "bob"}, - json={}, ) - public_update_response = await ds.client.post( - "/data/alice_public/-/update", + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_public"), actor={"id": "bob"}, - json={"update": {"title": "Bob can edit public queries"}}, - ) - public_delete_response = await ds.client.post( - "/data/alice_public/-/delete", - actor={"id": "bob"}, - json={}, ) - assert private_update_response.status_code == 403 - assert private_delete_response.status_code == 403 - assert public_update_response.status_code == 200 - assert public_delete_response.status_code == 200 + private_response = await ds.client.post( + "/data/alice_private/{}".format(path_suffix), + actor={"id": "bob"}, + json=request_json, + ) + public_response = await ds.client.post( + "/data/alice_public/{}".format(path_suffix), + actor={"id": "bob"}, + json=request_json, + ) + + assert private_response.status_code == 403 + assert public_response.status_code == 200 assert await ds.get_query("data", "alice_private") is not None - assert await ds.get_query("data", "alice_public") is None + public_query = await ds.get_query("data", "alice_public") + if expected_public_title is None: + assert public_query is None + else: + assert public_query.title == expected_public_title @pytest.mark.asyncio From b6e9b189905f6a03136e5998fdf39e1944a1e2a8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 15:37:48 -0700 Subject: [PATCH 760/767] datasette.yml can no longer set a query to private Private means it has an owner, and the config does not let you say who the owner is - plus configured queries should not be possible to edit or delete in the UI so having an owner makes even less sense. You can still make configured queries visible to specific people using regular view-query permissions. --- datasette/stored_queries.py | 1 - tests/test_queries.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index b6ac49b8..a6123daa 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -109,7 +109,6 @@ async def save_queries_from_config(datasette: Any) -> None: fragment=query_config.get("fragment"), parameters=query_config.get("params"), is_write=bool(query_config.get("write")), - is_private=bool(query_config.get("is_private")), is_trusted=bool(query_config.get("is_trusted", True)), source="config", on_success_message=query_config.get("on_success_message"), diff --git a/tests/test_queries.py b/tests/test_queries.py index 216cb211..2aa5142b 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -191,6 +191,8 @@ async def test_config_queries_imported_to_internal_table(): "title": "Configured query", "description_html": "

Configured HTML

", "params": ["name"], + # Configured queries are always public; this is ignored. + "is_private": True, "on_success_message_sql": "select 'Hello ' || :name", } } From 74324cb8492be8aa8597e58fb6f690158128e6fc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 15:46:27 -0700 Subject: [PATCH 761/767] Improved docs for user-facing SQL query pages - /database-name/-/execute-write - /-/queries --- docs/authentication.rst | 4 ++-- docs/pages.rst | 27 +++++++++++++++++++++++++++ docs/sql_queries.rst | 2 ++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index a0891900..5d831da0 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1413,7 +1413,7 @@ Actor is allowed to drop a database table. execute-sql ----------- -Actor is allowed to run arbitrary read-only SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100 +Actor is allowed to run arbitrary read-only SQL queries against a specific database using the :ref:`custom SQL query page `, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100 ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) @@ -1425,7 +1425,7 @@ See also :ref:`the default_allow_sql setting `. execute-write-sql ----------------- -Actor is allowed to run arbitrary writable SQL queries against a specific database, subject to table-level write permissions such as ``insert-row``, ``update-row`` and ``delete-row``. SQL functions are allowed and are not separately restricted by Datasette permissions. +Actor is allowed to run arbitrary writable SQL queries against a specific database using the :ref:`write SQL queries page `, subject to table-level write permissions such as ``insert-row``, ``update-row`` and ``delete-row``. SQL functions are allowed and are not separately restricted by Datasette permissions. ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) diff --git a/docs/pages.rst b/docs/pages.rst index e57c15e6..a8ff7c37 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -62,6 +62,11 @@ The following tables are hidden by default: Queries ======= +.. _pages_custom_sql_queries: + +Custom SQL queries +------------------ + The ``/database-name/-/query`` page can be used to execute an arbitrary SQL query against that database, if the :ref:`actions_execute_sql` permission is enabled. This query is passed as the ``?sql=`` query string parameter. This means you can link directly to a query by constructing the following URL: @@ -72,6 +77,28 @@ Each configured :ref:`stored query ` has its own page, at ``/dat In both cases adding a ``.json`` extension to the URL will return the results as JSON. +.. _pages_execute_write: + +Write SQL queries +----------------- + +The ``/database-name/-/execute-write`` page can be used to execute SQL statements that write to a mutable database, if the :ref:`actions_execute_write_sql` permission is enabled. + +This page extracts named parameters from the SQL, shows the tables that will be affected and lists the permissions required before the query can be executed. It also includes templates for common ``INSERT``, ``UPDATE`` and ``DELETE`` statements. + +Datasette checks additional permissions based on the operations in the SQL. Row changes require the relevant table-level permissions such as :ref:`actions_insert_row`, :ref:`actions_update_row` and :ref:`actions_delete_row`; reads from source tables require :ref:`actions_view_table`; and schema changes require permissions such as :ref:`actions_create_table`, :ref:`actions_alter_table` or :ref:`actions_drop_table`. + +Use the :ref:`ExecuteWriteView` JSON API to execute writable SQL programmatically. + +.. _pages_stored_query_browser: + +Stored query browsers +--------------------- + +The ``/-/queries`` page lists stored queries across every database visible to the current actor. The ``/database-name/-/queries`` page lists stored queries for a single database. + +These pages support search, pagination and filters for read-only or writable queries and private or public queries. Adding a ``.json`` extension to either URL returns the same list as JSON. + .. _TableView: Table diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index d427ea2b..c0ba67f0 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -7,6 +7,8 @@ Datasette treats SQLite database files as read-only and immutable. This means it The easiest way to execute custom SQL against Datasette is through the web UI. The database index page includes a SQL editor that lets you run any SELECT query you like. You can also construct queries using the filter interface on the tables page, then click "View and edit SQL" to open that query in the custom SQL editor. +For mutable databases, actors with the appropriate permissions can use the :ref:`write SQL page ` to execute SQL statements that insert, update or delete rows. + Note that this interface is only available if the :ref:`actions_execute_sql` permission is allowed. See :ref:`authentication_permissions_execute_sql`. Any Datasette SQL query is reflected in the URL of the page, allowing you to bookmark them, share them with others and navigate through previous queries using your browser back button. From 6a998610eef6e69d439a654dd31087023d285452 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 15:52:51 -0700 Subject: [PATCH 762/767] datasette inspect now counts 10,000+ tables correctly (#2752) Closes #2712 Refs https://github.com/simonw/datasette/pull/2721#issuecomment-4568966383 --- datasette/cli.py | 7 ++++--- docs/changelog.rst | 1 + tests/test_cli.py | 18 +++++++++++++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 93aa22ef..90a33e80 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -21,6 +21,7 @@ from .app import ( SQLITE_LIMIT_ATTACHED, pm, ) +from .inspect import inspect_tables from .utils import ( LoadExtension, StartupError, @@ -154,14 +155,14 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - counts = await database.table_counts(limit=3600 * 1000) + tables = await database.execute_fn(lambda conn: inspect_tables(conn, {})) data[name] = { "hash": database.hash, "size": database.size, "file": database.path, "tables": { - table_name: {"count": table_count} - for table_name, table_count in counts.items() + table_name: {"count": table["count"]} + for table_name, table in tables.items() }, } return data diff --git a/docs/changelog.rst b/docs/changelog.rst index a4be98b1..3882cc12 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,6 +37,7 @@ Bug fixes ~~~~~~~~~ - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) +- The ``datasette inspect`` command now correctly records row counts for tables with more than 10,000 rows. (:issue:`2712`) .. _v1_0_a30: diff --git a/tests/test_cli.py b/tests/test_cli.py index 1d3a2b28..f86d6909 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -35,12 +35,28 @@ def test_inspect_cli(app_client): assert expected_count == database["tables"][table_name]["count"] +def test_inspect_cli_counts_all_rows(tmp_path): + db_path = tmp_path / "big.db" + conn = sqlite3.connect(db_path) + with conn: + conn.execute("create table t (id integer primary key)") + conn.executemany("insert into t (id) values (?)", ((i,) for i in range(10002))) + conn.close() + + runner = CliRunner() + result = runner.invoke(cli, ["inspect", str(db_path)]) + assert result.exit_code == 0, result.output + data = json.loads(result.output) + + assert data["big"]["tables"]["t"]["count"] == 10002 + + def test_inspect_cli_writes_to_file(app_client): runner = CliRunner() result = runner.invoke( cli, ["inspect", "fixtures.db", "--inspect-file", "foo.json"] ) - assert 0 == result.exit_code, result.output + assert result.exit_code == 0, result.output with open("foo.json") as fp: data = json.load(fp) assert ["fixtures"] == list(data.keys()) From e5b6166fa35558920342e74f5ec13078957e87bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 16:19:39 -0700 Subject: [PATCH 763/767] Nicer UI around Execute Write SQL denied Refs https://github.com/simonw/datasette/issues/2753#issuecomment-4569117665 --- datasette/templates/execute_write.html | 82 ++++++++++++++++++++------ datasette/views/execute_write.py | 17 +++--- datasette/views/query_helpers.py | 20 +++++-- tests/test_queries.py | 75 ++++++++++++++++++++++- 4 files changed, 160 insertions(+), 34 deletions(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 6b626f8d..ee251111 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -40,6 +40,26 @@ border-radius: 0.25rem; min-width: 13rem; } +.execute-write-submit-row { + align-items: center; + display: flex; + flex-wrap: wrap; + gap: 0.45rem 0.75rem; +} +.execute-write-submit-row [hidden] { + display: none; +} +form.sql.core input[data-execute-write-submit]:disabled { + background: #d0d7de; + border-color: #b6c0cc; + color: #5f6975; + cursor: not-allowed; + opacity: 1; +} +.execute-write-disabled-reason { + color: #4f5b6d; + font-size: 0.85rem; +} {% include "_execute_write_analysis_styles.html" %} {% include "_sql_parameter_styles.html" %} @@ -119,9 +139,10 @@ {% endif %}
-

- - {% if save_query_base_url %}Save this query{% endif %} +

+ + {{ execute_disabled_reason or "" }} + {% if save_query_url %}Save this query{% endif %}

@@ -143,25 +164,55 @@ window.addEventListener("DOMContentLoaded", () => { const submitButton = form ? form.querySelector("[data-execute-write-submit]") : null; - const saveQueryLink = form + const submitDisabledReason = form + ? form.querySelector("[data-execute-write-disabled-reason]") + : null; + const submitRow = form + ? form.querySelector(".execute-write-submit-row") + : null; + let saveQueryLink = form ? form.querySelector("[data-save-query-link]") : null; + function updateSubmitState(data) { + if (submitButton) { + submitButton.disabled = data.execute_disabled; + } + if (!submitDisabledReason) { + return; + } + const reason = data.execute_disabled_reason || ""; + submitDisabledReason.textContent = reason; + submitDisabledReason.hidden = !reason; + } + function updateSaveQueryLink(data) { - if (!saveQueryLink) { + if (!submitRow || !submitRow.dataset.saveQueryBaseUrl) { return; } const sql = window.editor ? window.editor.state.doc.toString() : executeWriteSqlInput.value; if (!sql.trim() || !data.ok || data.execute_disabled) { - saveQueryLink.hidden = true; + if (saveQueryLink) { + saveQueryLink.remove(); + saveQueryLink = null; + } return; } - const url = new URL(saveQueryLink.dataset.saveQueryBaseUrl, window.location.href); + if (!saveQueryLink) { + saveQueryLink = document.createElement("a"); + saveQueryLink.className = "save-query"; + saveQueryLink.setAttribute("data-save-query-link", ""); + saveQueryLink.textContent = "Save this query"; + submitRow.appendChild(saveQueryLink); + } + const url = new URL( + submitRow.dataset.saveQueryBaseUrl, + window.location.href + ); url.searchParams.set("sql", sql); saveQueryLink.href = url.pathname + url.search + url.hash; - saveQueryLink.hidden = false; } window.datasetteSqlParameters.setupSqlParameterRefresh({ @@ -170,9 +221,7 @@ window.addEventListener("DOMContentLoaded", () => { allowExpand: true, onData(data) { window.datasetteSqlAnalysis.renderAnalysis(analysisSection, data); - if (submitButton) { - submitButton.disabled = data.execute_disabled; - } + updateSubmitState(data); updateSaveQueryLink(data); }, onError(error) { @@ -180,12 +229,11 @@ window.addEventListener("DOMContentLoaded", () => { analysis_error: error.message, analysis_rows: [], }); - if (submitButton) { - submitButton.disabled = true; - } - if (saveQueryLink) { - saveQueryLink.hidden = true; - } + updateSubmitState({ + execute_disabled: true, + execute_disabled_reason: error.message, + }); + updateSaveQueryLink({ ok: false, execute_disabled: true }); }, }); }); diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index 57c4d78e..7b693978 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -14,6 +14,7 @@ from .query_helpers import ( _coerce_execute_write_payload, _derived_query_parameters, _execute_write_analysis_data, + _execute_write_disabled_reason, _inserted_row_url, _json_or_form_payload, _prepare_execute_write, @@ -80,13 +81,12 @@ class ExecuteWriteView(BaseView): ) save_query_base_url = None save_query_url = None + execute_disabled_reason = _execute_write_disabled_reason( + sql, analysis_error, analysis_rows + ) if allow_save_query: save_query_base_url = self.ds.urls.database(db.name) + "/-/queries/store" - if ( - sql - and analysis_error is None - and not any(row["allowed"] is False for row in analysis_rows) - ): + if not execute_disabled_reason: save_query_url = save_query_base_url + "?" + urlencode({"sql": sql}) response = await self.render( @@ -103,11 +103,8 @@ class ExecuteWriteView(BaseView): "execution_message": execution_message, "execution_links": execution_links, "execution_ok": execution_ok, - "execute_disabled": bool( - (not sql) - or analysis_error - or any(row["allowed"] is False for row in analysis_rows) - ), + "execute_disabled": bool(execute_disabled_reason), + "execute_disabled_reason": execute_disabled_reason, "table_columns": table_columns, "write_template_tables": write_template_tables, "save_query_url": save_query_url, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 712832e8..f30a30bc 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -268,6 +268,16 @@ async def _analysis_rows_with_permissions( return rows +def _execute_write_disabled_reason(sql, analysis_error, analysis_rows): + if not (sql and sql.strip()): + return "Enter writable SQL before executing." + if analysis_error: + return analysis_error + if any(row.get("allowed") is False for row in analysis_rows): + return "You do not have permission for every operation listed above." + return None + + def _coerce_execute_write_payload(data, is_json): if not isinstance(data, dict): raise QueryValidationError("JSON must be a dictionary") @@ -358,16 +368,16 @@ async def _execute_write_analysis_data(datasette, db, sql, actor): ) except (QueryValidationError, sqlite3.DatabaseError) as ex: analysis_error = getattr(ex, "message", str(ex)) + execute_disabled_reason = _execute_write_disabled_reason( + sql, analysis_error, analysis_rows + ) return { "ok": analysis_error is None, "parameters": parameter_names, "analysis_error": analysis_error, "analysis_rows": analysis_rows, - "execute_disabled": bool( - (not sql) - or analysis_error - or any(row["allowed"] is False for row in analysis_rows) - ), + "execute_disabled": bool(execute_disabled_reason), + "execute_disabled_reason": execute_disabled_reason, } diff --git a/tests/test_queries.py b/tests/test_queries.py index 2aa5142b..87ecacde 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1374,6 +1374,10 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'addEventListener("paste"' in response.text assert "setupSqlParameterRefresh" in response.text assert "datasetteSqlAnalysis.renderAnalysis" in response.text + assert "input[data-execute-write-submit]:disabled" in response.text + assert ( + 'data-execute-write-disabled-reason aria-live="polite" hidden' in response.text + ) assert '' in response.text assert '' in response.text assert "" in response.text @@ -1390,7 +1394,9 @@ async def test_execute_write_get_prepopulates_without_executing(): ) assert '' in empty_response.text assert 'executeWriteSqlInput.value = "\\n\\n\\n";' in empty_response.text - assert "hidden>Save this query" in empty_response.text + assert "Enter writable SQL before executing." in empty_response.text + assert 'data-save-query-base-url="/data/-/queries/store"' in empty_response.text + assert 'Save this query" in read_only_response.text + assert ( + '' + ) in read_only_response.text + assert 'data-save-query-base-url="/data/-/queries/store"' in read_only_response.text + assert '' + ) in response.text + assert ( + '' + "You do not have permission for every operation listed above." + ) in response.text + assert 'no' in response.text + assert 'data-save-query-base-url="/data/-/queries/store"' in response.text + assert ' Date: Thu, 28 May 2026 16:20:28 -0700 Subject: [PATCH 764/767] //-/query.json and changelog docs --- docs/changelog.rst | 3 ++- docs/json_api.rst | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3882cc12..3501aa60 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,12 +17,13 @@ Stored queries - Users with :ref:`store-query ` and :ref:`execute-sql ` permission can create stored queries from the SQL query page or the new ``GET //-/queries/store`` form. (:issue:`2735`) - The database page now shows a count and preview of stored queries, capped at five, and links to new paginated query browsers at ``/-/queries`` and ``//-/queries``. Those browsers support search. (:issue:`2735`) - Stored queries created by users default to private and untrusted. Private stored queries can only be viewed, updated or deleted by their owner, even if another actor has broad ``view-query``, ``update-query`` or ``delete-query`` permission. Untrusted stored queries execute using the permissions of the actor running them. See :ref:`stored_queries` and :ref:`trusted_stored_queries` for details. (:issue:`2735`) +- Configured queries from ``datasette.yaml`` are trusted by default, so they can execute with ``view-query`` permission alone. They can opt out of that behavior using ``is_trusted: false`` but cannot be made private; private queries are only available for user-created stored queries. (:issue:`2735`) - New ``store-query``, ``update-query`` and ``delete-query`` permissions, plus updated semantics for :ref:`view-query `. Trusted stored queries can still execute with ``view-query`` alone; untrusted read queries also require :ref:`execute-sql ` and untrusted writable queries require :ref:`execute-write-sql ` plus the relevant table-level write permissions. (:issue:`2735`) Write SQL UI ~~~~~~~~~~~~ -- New "Write to this database" interface at ``//-/execute-write`` for running arbitrary writable SQL against mutable databases. The form extracts named parameters, analyzes the SQL, shows the table operations that will be attempted and links to a newly inserted row when a single-row insert succeeds. (:issue:`2742`) +- New "Write to this database" interface at ``//-/execute-write`` for running arbitrary writable SQL against mutable databases. The form extracts named parameters, analyzes the SQL, shows the table operations that will be attempted, includes starter templates for ``INSERT``, ``UPDATE`` and ``DELETE`` statements and links to a newly inserted row when a single-row insert succeeds. This is also available as a :ref:`JSON API `. (:issue:`2742`) - Added the new :ref:`execute-write-sql ` permission for running arbitrary writable SQL. Execution is also gated by table-level permissions such as :ref:`insert-row `, :ref:`update-row ` and :ref:`delete-row `, and writes to attached databases are rejected. (:issue:`2742`) - The write SQL analyzer now uses a deny-by-default model for unsupported operations. Reads from source tables require :ref:`view-table ` permission, schema changes require :ref:`create-table `, :ref:`alter-table ` or :ref:`drop-table ` as appropriate, and row mutation statements require the full ``insert-row``, ``update-row`` and ``delete-row`` permission set. SQL functions are allowed and are not separately permission-gated. (:issue:`2748`) - User-supplied write SQL now rejects ``VACUUM`` and writes to SQLite virtual tables or shadow tables. These restrictions also apply to untrusted stored write queries; trusted configured stored queries continue to skip these filters. (:issue:`2748`) diff --git a/docs/json_api.rst b/docs/json_api.rst index db19afc2..4bd76717 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -50,6 +50,25 @@ The ``"truncated"`` key lets you know if the query was truncated. This can happe For table pages, an additional key ``"next"`` may be present. This indicates that the next page in the pagination set can be retrieved using ``?_next=VALUE``. +.. _json_api_custom_sql: + +Executing custom SQL +-------------------- + +Actors with the :ref:`actions_execute_sql` permission can execute read-only SQL against a database using ``/-/query.json``: + +:: + + GET //-/query.json?sql=select+*+from+dogs + +Values for named SQL parameters can be provided as additional query string parameters: + +:: + + GET //-/query.json?sql=select+*+from+dogs+where+name=:name&name=Cleo + +The response uses the same default representation described above. + .. _json_api_shapes: Different shapes @@ -529,7 +548,7 @@ The request body must include a ``"sql"`` string. Named SQL parameters can be pr } } -The SQL must be writable. Read-only ``select`` queries should use the regular :ref:`custom SQL query API ` instead. +The SQL must be writable. Read-only ``select`` queries should use the regular :ref:`custom SQL query JSON API ` instead. Datasette analyzes the SQL before executing it. The actor must have ``execute-write-sql`` permission for the database, and must also have any permissions required by the operations in the SQL. For example, inserts and updates against a table require ``insert-row``, ``update-row`` and ``delete-row`` permissions for that table. Reads performed as part of the write, such as ``insert into dogs select ... from other_table``, require ``view-table`` permission on the source table. Schema changes require ``create-table``, ``alter-table`` or ``drop-table`` permissions as appropriate. From 9e377e8b90b27ae21d3263d0bfe8d3808e2c6133 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 28 May 2026 20:01:48 -0700 Subject: [PATCH 765/767] Only show valid SQL write templates Closes #2753 Demo: https://github.com/simonw/datasette/issues/2753#issuecomment-4570071413 --- datasette/templates/execute_write.html | 130 ++++------------- datasette/views/execute_write.py | 192 ++++++++++++++++++++++++- tests/test_queries.py | 117 ++++++++++++++- 3 files changed, 331 insertions(+), 108 deletions(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index ee251111..394261de 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -89,16 +89,18 @@ form.sql.core input[data-execute-write-submit]:disabled {

- - - + {% for operation in write_template_operations %} + + {% endfor %}

+ {% else %} +

You don't currently have permission to insert, edit or delete from any tables.

{% endif %}

@@ -242,119 +244,43 @@ window.addEventListener("DOMContentLoaded", () => { {% if write_template_tables %} {% endif %} diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index 7b693978..cff20847 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -1,3 +1,4 @@ +import re from urllib.parse import urlencode from datasette.resources import DatabaseResource @@ -22,6 +23,187 @@ from .query_helpers import ( _wants_json, ) +WRITE_TEMPLATE_LABELS = { + "insert": "Insert row", + "update": "Update rows", + "delete": "Delete rows", +} +WRITE_TEMPLATE_OPERATIONS = tuple(WRITE_TEMPLATE_LABELS) + + +def _parameter_names(columns): + seen = set() + names = {} + for column in columns: + base = re.sub(r"[^a-z0-9_]+", "_", column.lower()) + base = base.strip("_") or "value" + if base[0].isdigit(): + base = "p_{}".format(base) + name = base + index = 2 + while name in seen: + name = "{}_{}".format(base, index) + index += 1 + seen.add(name) + names[column] = name + return names + + +def _quote_identifier(identifier): + return '"{}"'.format(identifier.replace('"', '""')) + + +def _preferred_where_column(table, columns): + lower_table_id = "{}_id".format(table.lower()) + return ( + next((column for column in columns if column.lower() == "id"), None) + or next( + (column for column in columns if column.lower() == lower_table_id), None + ) + or columns[0] + ) + + +def _auto_incrementing_primary_key(columns): + primary_keys = [column for column in columns if column.is_pk] + if len(primary_keys) != 1: + return None + primary_key = primary_keys[0] + if primary_key.type and primary_key.type.lower() == "integer": + return primary_key.name + return None + + +def _insert_template_sql(table, columns): + column_names = [column.name for column in columns] + auto_pk = _auto_incrementing_primary_key(columns) + insert_columns = [column for column in column_names if column != auto_pk] + if not insert_columns: + return "insert into {}\ndefault values".format(_quote_identifier(table)) + names = _parameter_names(insert_columns) + return "\n".join( + ( + "insert into {} (".format(_quote_identifier(table)), + ",\n".join( + " {}".format(_quote_identifier(column)) for column in insert_columns + ), + ")", + "values (", + ",\n".join(" :{}".format(names[column]) for column in insert_columns), + ")", + ) + ) + + +def _update_template_sql(table, columns): + column_names = [column.name for column in columns] + names = _parameter_names(column_names) + where_column = _preferred_where_column(table, column_names) + set_columns = [column for column in column_names if column != where_column] + if not set_columns: + return "\n".join( + ( + "update {}".format(_quote_identifier(table)), + "set {} = :new_{}".format( + _quote_identifier(where_column), names[where_column] + ), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), + ) + ) + return "\n".join( + ( + "update {}".format(_quote_identifier(table)), + "set " + + ",\n".join( + "{}{} = :{}".format( + " " if index else "", + _quote_identifier(column), + names[column], + ) + for index, column in enumerate(set_columns) + ), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), + ) + ) + + +def _delete_template_sql(table, columns): + column_names = [column.name for column in columns] + names = _parameter_names(column_names) + where_column = _preferred_where_column(table, column_names) + return "\n".join( + ( + "delete from {}".format(_quote_identifier(table)), + "where {} = :{}".format( + _quote_identifier(where_column), names[where_column] + ), + ) + ) + + +def _template_sqls_for_table(table, columns): + return { + "insert": _insert_template_sql(table, columns), + "update": _update_template_sql(table, columns), + "delete": _delete_template_sql(table, columns), + } + + +async def _template_sql_allowed(datasette, db, sql, actor): + params = {parameter: "" for parameter in _derived_query_parameters(sql)} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError: + return False + if not _analysis_is_write(analysis): + return False + analysis_rows = await _analysis_rows_with_permissions(datasette, analysis, actor) + return _execute_write_disabled_reason(sql, None, analysis_rows) is None + + +async def _write_template_tables( + datasette, db, table_columns, hidden_table_names, actor +): + write_template_tables = {} + for table in table_columns: + if table in hidden_table_names or not table_columns[table]: + continue + column_details = [ + column + for column in await db.table_column_details(table) + if not column.hidden + ] + if not column_details: + continue + templates = {} + for operation, sql in _template_sqls_for_table(table, column_details).items(): + if await _template_sql_allowed(datasette, db, sql, actor): + templates[operation] = sql + if templates: + write_template_tables[table] = { + "templates": templates, + } + return write_template_tables + + +def _write_template_operations(write_template_tables): + operations = [] + for operation in WRITE_TEMPLATE_OPERATIONS: + if any( + operation in table["templates"] for table in write_template_tables.values() + ): + operations.append( + { + "name": operation, + "label": WRITE_TEMPLATE_LABELS[operation], + } + ) + return operations + class ExecuteWriteView(BaseView): name = "execute-write" @@ -47,11 +229,10 @@ class ExecuteWriteView(BaseView): analysis_rows = [] table_columns = await _table_columns(self.ds, db.name) hidden_table_names = set(await db.hidden_table_names()) - write_template_tables = { - table: columns - for table, columns in table_columns.items() - if columns and table not in hidden_table_names - } + write_template_tables = await _write_template_tables( + self.ds, db, table_columns, hidden_table_names, request.actor + ) + write_template_operations = _write_template_operations(write_template_tables) if sql and analysis_error is None: try: parameter_names = _derived_query_parameters(sql) @@ -107,6 +288,7 @@ class ExecuteWriteView(BaseView): "execute_disabled_reason": execute_disabled_reason, "table_columns": table_columns, "write_template_tables": write_template_tables, + "write_template_operations": write_template_operations, "save_query_url": save_query_url, "save_query_base_url": save_query_base_url, }, diff --git a/tests/test_queries.py b/tests/test_queries.py index 87ecacde..89167a1d 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1,4 +1,6 @@ import json +import re +from html import unescape import pytest @@ -8,6 +10,19 @@ from datasette.stored_queries import StoredQuery, StoredQueryPage from datasette.utils.asgi import Forbidden +def _template_option_attributes(html, table): + match = re.search(r'' in response.text + assert '
Required permissioninsert