From 7b41521b338304bd4065a9ec0db9b4c651ec6fed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 5 Apr 2023 16:25:29 -0700 Subject: [PATCH 01/19] WIP new JSON for queries, refs #2049 --- datasette/app.py | 13 +- datasette/views/database.py | 329 ++++++++++++++++++++++++++++++++++++ 2 files changed, 340 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index d7dace67..966a6faf 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -33,7 +33,12 @@ from jinja2.environment import Template from jinja2.exceptions import TemplateNotFound from .views.base import ureg -from .views.database import DatabaseDownload, DatabaseView, TableCreateView +from .views.database import ( + DatabaseDownload, + DatabaseView, + TableCreateView, + database_view, +) from .views.index import IndexView from .views.special import ( JsonDataView, @@ -1365,8 +1370,12 @@ class Datasette: r"/-/patterns$", ) add_route(DatabaseDownload.as_view(self), r"/(?P[^\/\.]+)\.db$") + # add_route( + # DatabaseView.as_view(self), r"/(?P[^\/\.]+)(\.(?P\w+))?$" + # ) add_route( - DatabaseView.as_view(self), r"/(?P[^\/\.]+)(\.(?P\w+))?$" + wrap_view(database_view, self), + r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") add_route( diff --git a/datasette/views/database.py b/datasette/views/database.py index dda82510..7b90a2db 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -757,3 +757,332 @@ async def _table_columns(datasette, database_name): for view_name in await db.view_names(): table_columns[view_name] = [] return table_columns + + +async def database_view(request, datasette): + return await database_view_impl(request, datasette) + + +async def database_view_impl( + request, + datasette, + canned_query=None, + _size=None, + named_parameters=None, + write=False, +): + db = await datasette.resolve_database(request) + database = db.name + params = {key: request.args.get(key) for key in request.args} + sql = "" + if "sql" in params: + sql = params.pop("sql") + _shape = None + if "_shape" in params: + _shape = params.pop("_shape") + + private = False + if canned_query: + # Respect canned query permissions + visible, private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-query", (database, canned_query)), + ("view-database", database), + "view-instance", + ], + ) + if not visible: + raise Forbidden("You do not have permission to view this query") + + else: + await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + + # Extract any :named parameters + named_parameters = named_parameters or await derive_named_parameters(db, sql) + named_parameter_values = { + named_parameter: params.get(named_parameter) or "" + for named_parameter in named_parameters + if not named_parameter.startswith("_") + } + + # Set to blank string if missing from params + for named_parameter in named_parameters: + if named_parameter not in params and not named_parameter.startswith("_"): + params[named_parameter] = "" + + extra_args = {} + if params.get("_timelimit"): + extra_args["custom_time_limit"] = int(params["_timelimit"]) + if _size: + extra_args["page_size"] = _size + + templates = [f"query-{to_css_class(database)}.html", "query.html"] + if canned_query: + templates.insert( + 0, + f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", + ) + + query_error = None + + # Execute query - as write or as read + if write: + raise NotImplementedError("Write queries not yet implemented") + # if request.method == "POST": + # # If database is immutable, return an error + # if not db.is_mutable: + # raise Forbidden("Database is immutable") + # body = await request.post_body() + # body = body.decode("utf-8").strip() + # if body.startswith("{") and body.endswith("}"): + # params = json.loads(body) + # # But we want key=value strings + # for key, value in params.items(): + # params[key] = str(value) + # else: + # params = dict(parse_qsl(body, keep_blank_values=True)) + # # Should we return JSON? + # should_return_json = ( + # request.headers.get("accept") == "application/json" + # or request.args.get("_json") + # or params.get("_json") + # ) + # if canned_query: + # params_for_query = MagicParameters(params, request, self.ds) + # else: + # params_for_query = params + # ok = None + # try: + # cursor = await self.ds.databases[database].execute_write( + # sql, params_for_query + # ) + # message = metadata.get( + # "on_success_message" + # ) or "Query executed, {} row{} affected".format( + # cursor.rowcount, "" if cursor.rowcount == 1 else "s" + # ) + # message_type = self.ds.INFO + # redirect_url = metadata.get("on_success_redirect") + # ok = True + # except Exception as e: + # message = metadata.get("on_error_message") or str(e) + # message_type = self.ds.ERROR + # redirect_url = metadata.get("on_error_redirect") + # ok = False + # if should_return_json: + # return Response.json( + # { + # "ok": ok, + # "message": message, + # "redirect": redirect_url, + # } + # ) + # else: + # self.ds.add_message(request, message, message_type) + # return self.redirect(request, redirect_url or request.path) + # else: + + # async def extra_template(): + # return { + # "request": request, + # "db_is_immutable": not db.is_mutable, + # "path_with_added_args": path_with_added_args, + # "path_with_removed_args": path_with_removed_args, + # "named_parameter_values": named_parameter_values, + # "canned_query": canned_query, + # "success_message": request.args.get("_success") or "", + # "canned_write": True, + # } + + # return ( + # { + # "database": database, + # "rows": [], + # "truncated": False, + # "columns": [], + # "query": {"sql": sql, "params": params}, + # "private": private, + # }, + # extra_template, + # templates, + # ) + + # Not a write + if canned_query: + params_for_query = MagicParameters(params, request, datasette) + else: + params_for_query = params + try: + results = await datasette.execute( + database, sql, params_for_query, truncate=True, **extra_args + ) + columns = [r[0] for r in results.description] + except sqlite3.DatabaseError as e: + query_error = e + results = None + columns = [] + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + + return Response.json( + { + "ok": True, + "rows": [dict(r) for r in results], + # "columns": columns, + # "database": database, + # "params": params, + # "sql": sql, + # "_shape": _shape, + # "named_parameters": named_parameters, + # "named_parameter_values": named_parameter_values, + # "extra_args": extra_args, + # "templates": templates, + } + ) + + async def extra_template(): + display_rows = [] + truncate_cells = datasette.setting("truncate_cells_html") + for row in results.rows if results else []: + display_row = [] + for column, value in zip(results.columns, row): + display_value = value + # Let the plugins have a go + # pylint: disable=no-member + plugin_display_value = None + for candidate in pm.hook.render_cell( + row=row, + value=value, + column=column, + table=None, + database=database, + datasette=self.ds, + request=request, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value is not None: + display_value = plugin_display_value + else: + if value in ("", None): + display_value = Markup(" ") + elif is_url(str(display_value).strip()): + display_value = markupsafe.Markup( + '{truncated_url}'.format( + url=markupsafe.escape(value.strip()), + truncated_url=markupsafe.escape( + truncate_url(value.strip(), truncate_cells) + ), + ) + ) + elif isinstance(display_value, bytes): + blob_url = path_with_format( + request=request, + format="blob", + extra_qs={ + "_blob_column": column, + "_blob_hash": hashlib.sha256(display_value).hexdigest(), + }, + ) + formatted = format_bytes(len(value)) + display_value = markupsafe.Markup( + '<Binary: {:,} byte{}>'.format( + blob_url, + ' title="{}"'.format(formatted) + if "bytes" not in formatted + else "", + len(value), + "" if len(value) == 1 else "s", + ) + ) + else: + display_value = str(value) + if truncate_cells and len(display_value) > truncate_cells: + display_value = display_value[:truncate_cells] + "\u2026" + display_row.append(display_value) + display_rows.append(display_row) + + # Show 'Edit SQL' button only if: + # - User is allowed to execute SQL + # - SQL is an approved SELECT statement + # - No magic parameters, so no :_ in the SQL string + edit_sql_url = None + is_validated_sql = False + try: + validate_sql_select(sql) + is_validated_sql = True + except InvalidSql: + pass + if allow_execute_sql and is_validated_sql and ":_" not in sql: + edit_sql_url = ( + self.ds.urls.database(database) + + "?" + + urlencode( + { + **{ + "sql": sql, + }, + **named_parameter_values, + } + ) + ) + + show_hide_hidden = "" + if metadata.get("hide_sql"): + if bool(params.get("_show_sql")): + show_hide_link = path_with_removed_args(request, {"_show_sql"}) + show_hide_text = "hide" + show_hide_hidden = '' + else: + show_hide_link = path_with_added_args(request, {"_show_sql": 1}) + show_hide_text = "show" + else: + if bool(params.get("_hide_sql")): + show_hide_link = path_with_removed_args(request, {"_hide_sql"}) + show_hide_text = "show" + show_hide_hidden = '' + else: + show_hide_link = path_with_added_args(request, {"_hide_sql": 1}) + show_hide_text = "hide" + hide_sql = show_hide_text == "show" + return { + "display_rows": display_rows, + "custom_sql": True, + "named_parameter_values": named_parameter_values, + "editable": editable, + "canned_query": canned_query, + "edit_sql_url": edit_sql_url, + "metadata": metadata, + "settings": self.ds.settings_dict(), + "request": request, + "show_hide_link": self.ds.urls.path(show_hide_link), + "show_hide_text": show_hide_text, + "show_hide_hidden": markupsafe.Markup(show_hide_hidden), + "hide_sql": hide_sql, + "table_columns": await _table_columns(self.ds, database) + if allow_execute_sql + else {}, + } + + return ( + { + "ok": not query_error, + "database": database, + "query_name": canned_query, + "rows": results.rows if results else [], + "truncated": results.truncated if results else False, + "columns": columns, + "query": {"sql": sql, "params": params}, + "error": str(query_error) if query_error else None, + "private": private, + "allow_execute_sql": allow_execute_sql, + }, + extra_template, + templates, + 400 if query_error else 200, + ) From 40dc5f5c501c4c32120148d94ec7c9d130141571 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Apr 2023 17:04:26 -0700 Subject: [PATCH 02/19] WIP --- datasette/views/database.py | 227 +++++++++++++++++++++++++++++++++--- datasette/views/table.py | 1 - 2 files changed, 212 insertions(+), 16 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 7b90a2db..d097c933 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -12,6 +12,7 @@ import markupsafe from datasette.utils import ( add_cors_headers, await_me_maybe, + call_with_supported_arguments, derive_named_parameters, format_bytes, tilde_decode, @@ -763,6 +764,119 @@ async def database_view(request, datasette): return await database_view_impl(request, datasette) +async def database_index_view(request, datasette, db): + database = db.name + visible, private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-database", database), + "view-instance", + ], + ) + if not visible: + raise Forbidden("You do not have permission to view this database") + + metadata = (datasette.metadata("databases") or {}).get(database, {}) + datasette.update_with_inherited_metadata(metadata) + + table_counts = await db.table_counts(5) + hidden_table_names = set(await db.hidden_table_names()) + all_foreign_keys = await db.get_all_foreign_keys() + + views = [] + for view_name in await db.view_names(): + view_visible, view_private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-table", (database, view_name)), + ("view-database", database), + "view-instance", + ], + ) + if view_visible: + views.append( + { + "name": view_name, + "private": view_private, + } + ) + + tables = [] + for table in table_counts: + table_visible, table_private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-table", (database, table)), + ("view-database", database), + "view-instance", + ], + ) + if not table_visible: + continue + table_columns = await db.table_columns(table) + tables.append( + { + "name": table, + "columns": table_columns, + "primary_keys": await db.primary_keys(table), + "count": table_counts[table], + "hidden": table in hidden_table_names, + "fts_table": await db.fts_table(table), + "foreign_keys": all_foreign_keys[table], + "private": table_private, + } + ) + + tables.sort(key=lambda t: (t["hidden"], t["name"])) + canned_queries = [] + for query in (await datasette.get_canned_queries(database, request.actor)).values(): + query_visible, query_private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-query", (database, query["name"])), + ("view-database", database), + "view-instance", + ], + ) + if query_visible: + canned_queries.append(dict(query, private=query_private)) + + async def database_actions(): + links = [] + for hook in pm.hook.database_actions( + datasette=datasette, + database=database, + actor=request.actor, + request=request, + ): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + + attached_databases = [d.name for d in await db.attached_databases()] + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + return Response.json( + { + "database": db.name, + "private": private, + "path": datasette.urls.database(database), + "size": db.size, + "tables": tables, + "hidden_count": len([t for t in tables if t["hidden"]]), + "views": views, + "queries": canned_queries, + "allow_execute_sql": allow_execute_sql, + "table_columns": await _table_columns(datasette, database) + if allow_execute_sql + else {}, + } + ) + + async def database_view_impl( request, datasette, @@ -798,6 +912,12 @@ async def database_view_impl( else: await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + # If there's no sql, show the database index page + if not sql: + return await database_index_view(request, datasette, db) + + validate_sql_select(sql) + # Extract any :named parameters named_parameters = named_parameters or await derive_named_parameters(db, sql) named_parameter_values = { @@ -909,6 +1029,7 @@ async def database_view_impl( # ) # Not a write + rows = [] if canned_query: params_for_query = MagicParameters(params, request, datasette) else: @@ -918,6 +1039,7 @@ async def database_view_impl( database, sql, params_for_query, truncate=True, **extra_args ) columns = [r[0] for r in results.description] + rows = list(results.rows) except sqlite3.DatabaseError as e: query_error = e results = None @@ -927,21 +1049,96 @@ async def database_view_impl( request.actor, "execute-sql", database ) - return Response.json( - { - "ok": True, - "rows": [dict(r) for r in results], - # "columns": columns, - # "database": database, - # "params": params, - # "sql": sql, - # "_shape": _shape, - # "named_parameters": named_parameters, - # "named_parameter_values": named_parameter_values, - # "extra_args": extra_args, - # "templates": templates, - } - ) + format_ = request.url_vars.get("format") or "html" + + if format_ == "csv": + raise NotImplementedError("CSV format not yet implemented") + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=None, + database=db.name, + table=None, + request=request, + view_name="table", # TODO: should this be "query"? + # These will be deprecated in Datasette 1.0: + args=request.args, + data={ + "rows": rows, + }, # TODO what should this be? + ) + result = await await_me_maybe(result) + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(database)}.html", "query.html"] + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + r = Response.html( + await datasette.render_template( + template, + dict( + data, + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + # TODO: review up all of these hacks: + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=any(c["sortable"] for c in data["display_columns"]), + allow_execute_sql=await datasette.permission_allowed( + request.actor, "execute-sql", resolved.db.name + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="table", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + # if next_url: + # r.headers["link"] = f'<{next_url}>; rel="next"' + return r async def extra_template(): display_rows = [] diff --git a/datasette/views/table.py b/datasette/views/table.py index c102c103..e367a075 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -9,7 +9,6 @@ import markupsafe from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette import tracer -from datasette.renderer import json_renderer from datasette.utils import ( add_cors_headers, await_me_maybe, From 026429fadd7a1f4f85ebfda1bbfe882938f455f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 20:47:03 -0700 Subject: [PATCH 03/19] Work in progress on query view, refs #2049 --- datasette/views/database.py | 420 ++++++++++++++++++++++++++++++++++++ setup.py | 2 +- tests/test_cli_serve_get.py | 9 +- 3 files changed, 426 insertions(+), 5 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index d097c933..33b6702b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1,3 +1,4 @@ +from asyncinject import Registry import os import hashlib import itertools @@ -877,6 +878,416 @@ async def database_index_view(request, datasette, db): ) +async def query_view( + request, + datasette, + canned_query=None, + _size=None, + named_parameters=None, + write=False, +): + print("query_view") + db = await datasette.resolve_database(request) + database = db.name + # TODO: Why do I do this? Is it to eliminate multi-args? + # It's going to break ?_extra=...&_extra=... + params = {key: request.args.get(key) for key in request.args} + sql = "" + if "sql" in params: + sql = params.pop("sql") + + # TODO: Behave differently for canned query here: + await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + + _shape = None + if "_shape" in params: + _shape = params.pop("_shape") + + # ?_shape=arrays - "rows" is the default option, shown above + # ?_shape=objects - "rows" is a list of JSON key/value objects + # ?_shape=array - an JSON array of objects + # ?_shape=array&_nl=on - a newline-separated list of JSON objects + # ?_shape=arrayfirst - a flat JSON array containing just the first value from each row + # ?_shape=object - a JSON object keyed using the primary keys of the rows + async def _results(_sql, _params): + # Returns (results, error (can be None)) + try: + return await db.execute(_sql, _params, truncate=True), None + except Exception as e: + return None, e + + async def shape_arrays(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return { + "ok": True, + "rows": [list(r) for r in results.rows], + "truncated": results.truncated, + } + + async def shape_objects(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return { + "ok": True, + "rows": [dict(r) for r in results.rows], + "truncated": results.truncated, + } + + async def shape_array(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return [dict(r) for r in results.rows] + + shape_fn = { + "arrays": shape_arrays, + "objects": shape_objects, + "array": shape_array, + # "arrayfirst": shape_arrayfirst, + # "object": shape_object, + }[_shape or "objects"] + + registry = Registry.from_dict( + { + "_results": _results, + "_shape": shape_fn, + }, + parallel=False, + ) + + results = await registry.resolve_multi( + ["_shape"], + results={ + "_sql": sql, + "_params": params, + }, + ) + + # If "shape" does not include "rows" we return that as the response + if "rows" not in results["_shape"]: + return Response.json(results["_shape"]) + + output = results["_shape"] + output.update(dict((k, v) for k, v in results.items() if not k.startswith("_"))) + + response = Response.json(output) + + assert False + + import pdb + + pdb.set_trace() + + if isinstance(output, dict) and output.get("ok") is False: + # TODO: Other error codes? + + response.status_code = 400 + + if datasette.cors: + add_cors_headers(response.headers) + + return response + + # registry = Registry( + # extra_count, + # extra_facet_results, + # extra_facets_timed_out, + # extra_suggested_facets, + # facet_instances, + # extra_human_description_en, + # extra_next_url, + # extra_columns, + # extra_primary_keys, + # run_display_columns_and_rows, + # extra_display_columns, + # extra_display_rows, + # extra_debug, + # extra_request, + # extra_query, + # extra_metadata, + # extra_extras, + # extra_database, + # extra_table, + # extra_database_color, + # extra_table_actions, + # extra_filters, + # extra_renderers, + # extra_custom_table_templates, + # extra_sorted_facet_results, + # extra_table_definition, + # extra_view_definition, + # extra_is_view, + # extra_private, + # extra_expandable_columns, + # extra_form_hidden_args, + # ) + + results = await registry.resolve_multi( + ["extra_{}".format(extra) for extra in extras] + ) + data = { + "ok": True, + "next": next_value and str(next_value) or None, + } + data.update( + { + key.replace("extra_", ""): value + for key, value in results.items() + if key.startswith("extra_") and key.replace("extra_", "") in extras + } + ) + raw_sqlite_rows = rows[:page_size] + data["rows"] = [dict(r) for r in raw_sqlite_rows] + + private = False + if canned_query: + # Respect canned query permissions + visible, private = await datasette.check_visibility( + request.actor, + permissions=[ + ("view-query", (database, canned_query)), + ("view-database", database), + "view-instance", + ], + ) + if not visible: + raise Forbidden("You do not have permission to view this query") + + else: + await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) + + # If there's no sql, show the database index page + if not sql: + return await database_index_view(request, datasette, db) + + validate_sql_select(sql) + + # Extract any :named parameters + named_parameters = named_parameters or await derive_named_parameters(db, sql) + named_parameter_values = { + named_parameter: params.get(named_parameter) or "" + for named_parameter in named_parameters + if not named_parameter.startswith("_") + } + + # Set to blank string if missing from params + for named_parameter in named_parameters: + if named_parameter not in params and not named_parameter.startswith("_"): + params[named_parameter] = "" + + extra_args = {} + if params.get("_timelimit"): + extra_args["custom_time_limit"] = int(params["_timelimit"]) + if _size: + extra_args["page_size"] = _size + + templates = [f"query-{to_css_class(database)}.html", "query.html"] + if canned_query: + templates.insert( + 0, + f"query-{to_css_class(database)}-{to_css_class(canned_query)}.html", + ) + + query_error = None + + # Execute query - as write or as read + if write: + raise NotImplementedError("Write queries not yet implemented") + # if request.method == "POST": + # # If database is immutable, return an error + # if not db.is_mutable: + # raise Forbidden("Database is immutable") + # body = await request.post_body() + # body = body.decode("utf-8").strip() + # if body.startswith("{") and body.endswith("}"): + # params = json.loads(body) + # # But we want key=value strings + # for key, value in params.items(): + # params[key] = str(value) + # else: + # params = dict(parse_qsl(body, keep_blank_values=True)) + # # Should we return JSON? + # should_return_json = ( + # request.headers.get("accept") == "application/json" + # or request.args.get("_json") + # or params.get("_json") + # ) + # if canned_query: + # params_for_query = MagicParameters(params, request, self.ds) + # else: + # params_for_query = params + # ok = None + # try: + # cursor = await self.ds.databases[database].execute_write( + # sql, params_for_query + # ) + # message = metadata.get( + # "on_success_message" + # ) or "Query executed, {} row{} affected".format( + # cursor.rowcount, "" if cursor.rowcount == 1 else "s" + # ) + # message_type = self.ds.INFO + # redirect_url = metadata.get("on_success_redirect") + # ok = True + # except Exception as e: + # message = metadata.get("on_error_message") or str(e) + # message_type = self.ds.ERROR + # redirect_url = metadata.get("on_error_redirect") + # ok = False + # if should_return_json: + # return Response.json( + # { + # "ok": ok, + # "message": message, + # "redirect": redirect_url, + # } + # ) + # else: + # self.ds.add_message(request, message, message_type) + # return self.redirect(request, redirect_url or request.path) + # else: + + # async def extra_template(): + # return { + # "request": request, + # "db_is_immutable": not db.is_mutable, + # "path_with_added_args": path_with_added_args, + # "path_with_removed_args": path_with_removed_args, + # "named_parameter_values": named_parameter_values, + # "canned_query": canned_query, + # "success_message": request.args.get("_success") or "", + # "canned_write": True, + # } + + # return ( + # { + # "database": database, + # "rows": [], + # "truncated": False, + # "columns": [], + # "query": {"sql": sql, "params": params}, + # "private": private, + # }, + # extra_template, + # templates, + # ) + + # Not a write + rows = [] + if canned_query: + params_for_query = MagicParameters(params, request, datasette) + else: + params_for_query = params + try: + results = await datasette.execute( + database, sql, params_for_query, truncate=True, **extra_args + ) + columns = [r[0] for r in results.description] + rows = list(results.rows) + except sqlite3.DatabaseError as e: + query_error = e + results = None + columns = [] + + allow_execute_sql = await datasette.permission_allowed( + request.actor, "execute-sql", database + ) + + format_ = request.url_vars.get("format") or "html" + + if format_ == "csv": + raise NotImplementedError("CSV format not yet implemented") + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=None, + database=db.name, + table=None, + request=request, + view_name="table", # TODO: should this be "query"? + # These will be deprecated in Datasette 1.0: + args=request.args, + data={ + "rows": rows, + }, # TODO what should this be? + ) + result = await await_me_maybe(result) + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(database)}.html", "query.html"] + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + r = Response.html( + await datasette.render_template( + template, + dict( + data, + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + # TODO: review up all of these hacks: + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=any(c["sortable"] for c in data["display_columns"]), + allow_execute_sql=await datasette.permission_allowed( + request.actor, "execute-sql", resolved.db.name + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="table", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + # if next_url: + # r.headers["link"] = f'<{next_url}>; rel="next"' + return r + + async def database_view_impl( request, datasette, @@ -887,10 +1298,19 @@ async def database_view_impl( ): db = await datasette.resolve_database(request) database = db.name + + if request.args.get("sql", "").strip(): + return await query_view( + request, datasette, canned_query, _size, named_parameters, write + ) + + # Index page shows the tables/views/canned queries for this database + params = {key: request.args.get(key) for key in request.args} sql = "" if "sql" in params: sql = params.pop("sql") + _shape = None if "_shape" in params: _shape = params.pop("_shape") diff --git a/setup.py b/setup.py index d41e428a..b591869e 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ setup( "mergedeep>=1.1.1", "itsdangerous>=1.1", "sqlite-utils>=3.30", - "asyncinject>=0.5", + "asyncinject>=0.6", ], entry_points=""" [console_scripts] diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index ac44e1e2..e484a6db 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -1,6 +1,7 @@ from datasette.cli import cli, serve from datasette.plugins import pm from click.testing import CliRunner +from unittest.mock import ANY import textwrap import json @@ -35,11 +36,11 @@ def test_serve_with_get(tmp_path_factory): ], ) assert 0 == result.exit_code, result.output - assert { - "database": "_memory", + assert json.loads(result.output) == { + "ok": True, + "rows": [{"sqlite_version()": ANY}], "truncated": False, - "columns": ["sqlite_version()"], - }.items() <= json.loads(result.output).items() + } # The plugin should have created hello.txt assert (plugins_dir / "hello.txt").read_text() == "hello" From a706f34b923c5c503c1fa8e6a260925c61cec3fb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 22:07:05 -0700 Subject: [PATCH 04/19] Remove debug lines --- datasette/views/database.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 33b6702b..cb31a762 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -975,12 +975,6 @@ async def query_view( response = Response.json(output) - assert False - - import pdb - - pdb.set_trace() - if isinstance(output, dict) and output.get("ok") is False: # TODO: Other error codes? From 8b86fb7fb4840b0351c25cf6ff40ca9f17a8fee7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:50:12 -0700 Subject: [PATCH 05/19] Better debugging --- datasette/utils/testing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index d4990784..7747f7ce 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -16,6 +16,9 @@ class TestResponse: def status(self): return self.httpx_response.status_code + def __repr__(self): + return "".format(self.httpx_response.url, self.status) + # Supports both for test-writing convenience @property def status_code(self): From 3304fd43a2492739e9803f513c4954a5fd1e374d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:51:05 -0700 Subject: [PATCH 06/19] refresh_schemas() on database view --- datasette/views/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index cb31a762..30016188 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -886,7 +886,6 @@ async def query_view( named_parameters=None, write=False, ): - print("query_view") db = await datasette.resolve_database(request) database = db.name # TODO: Why do I do this? Is it to eliminate multi-args? @@ -1290,6 +1289,7 @@ async def database_view_impl( named_parameters=None, write=False, ): + await datasette.refresh_schemas() db = await datasette.resolve_database(request) database = db.name From 6f903d5a98c59f8a602ef7de776ea631b7145ba8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:51:19 -0700 Subject: [PATCH 07/19] Fixed a test --- tests/test_api.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 780e9fa5..e700ad7b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -643,9 +643,6 @@ async def test_custom_sql(ds_client): "/fixtures.json?sql=select+content+from+simple_primary_key&_shape=objects" ) data = response.json() - assert {"sql": "select content from simple_primary_key", "params": {}} == data[ - "query" - ] assert [ {"content": "hello"}, {"content": "world"}, @@ -653,8 +650,6 @@ async def test_custom_sql(ds_client): {"content": "RENDER_CELL_DEMO"}, {"content": "RENDER_CELL_ASYNC"}, ] == data["rows"] - assert ["content"] == data["columns"] - assert "fixtures" == data["database"] assert not data["truncated"] From fdb141f6225ddf7209dc6abff0c1691bbb675465 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 May 2023 17:51:29 -0700 Subject: [PATCH 08/19] shape_arrayfirst for query view --- datasette/views/database.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 30016188..eac57cd3 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -941,11 +941,17 @@ async def query_view( return {"ok": False, "error": str(error)} return [dict(r) for r in results.rows] + async def shape_arrayfirst(_results): + results, error = _results + if error: + return {"ok": False, "error": str(error)} + return [r[0] for r in results.rows] + shape_fn = { "arrays": shape_arrays, "objects": shape_objects, "array": shape_array, - # "arrayfirst": shape_arrayfirst, + "arrayfirst": shape_arrayfirst, # "object": shape_object, }[_shape or "objects"] From 6ae5312158540aa12c4fc714404c0f3c1a72e4d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 May 2023 18:44:07 -0700 Subject: [PATCH 09/19] ?sql=... now displays HTML --- datasette/views/database.py | 158 +++++++++++++++++++++++++++++++++--- 1 file changed, 145 insertions(+), 13 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index eac57cd3..455ebd1f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -12,10 +12,12 @@ import markupsafe from datasette.utils import ( add_cors_headers, + append_querystring, await_me_maybe, call_with_supported_arguments, derive_named_parameters, format_bytes, + path_with_replaced_args, tilde_decode, to_css_class, validate_sql_select, @@ -887,6 +889,145 @@ async def query_view( write=False, ): db = await datasette.resolve_database(request) + + format_ = request.url_vars.get("format") or "html" + force_shape = None + if format_ == "html": + force_shape = "arrays" + + data = await query_view_data( + request, + datasette, + canned_query=canned_query, + _size=_size, + named_parameters=named_parameters, + write=write, + force_shape=force_shape, + ) + if format_ == "csv": + raise NotImplementedError("CSV format not yet implemented") + elif format_ in datasette.renderers.keys(): + # Dispatch request to the correct output format renderer + # (CSV is not handled here due to streaming) + result = call_with_supported_arguments( + datasette.renderers[format_][0], + datasette=datasette, + columns=columns, + rows=rows, + sql=sql, + query_name=None, + database=db.name, + table=None, + request=request, + view_name="table", # TODO: should this be "query"? + # These will be deprecated in Datasette 1.0: + args=request.args, + data={ + "rows": rows, + }, # TODO what should this be? + ) + result = await await_me_maybe(result) + if result is None: + raise NotFound("No data") + if isinstance(result, dict): + r = Response( + body=result.get("body"), + status=result.get("status_code") or 200, + content_type=result.get("content_type", "text/plain"), + headers=result.get("headers"), + ) + elif isinstance(result, Response): + r = result + # if status_code is not None: + # # Over-ride the status code + # r.status = status_code + else: + assert False, f"{result} should be dict or Response" + elif format_ == "html": + headers = {} + templates = [f"query-{to_css_class(db.name)}.html", "query.html"] + template = datasette.jinja_env.select_template(templates) + alternate_url_json = datasette.absolute_url( + request, + datasette.urls.path(path_with_format(request=request, format="json")), + ) + headers.update( + { + "Link": '{}; rel="alternate"; type="application/json+datasette"'.format( + alternate_url_json + ) + } + ) + metadata = (datasette.metadata("databases") or {}).get(db.name, {}) + datasette.update_with_inherited_metadata(metadata) + + r = Response.html( + await datasette.render_template( + template, + dict( + data, + database=db.name, + database_color=lambda database: "ff0000", + metadata=metadata, + display_rows=data["rows"], + renderers={}, + query={ + "sql": request.args.get("sql"), + }, + editable=True, + append_querystring=append_querystring, + path_with_replaced_args=path_with_replaced_args, + fix_path=datasette.urls.path, + settings=datasette.settings_dict(), + # TODO: review up all of these hacks: + alternate_url_json=alternate_url_json, + datasette_allow_facet=( + "true" if datasette.setting("allow_facet") else "false" + ), + is_sortable=False, + allow_execute_sql=await datasette.permission_allowed( + request.actor, "execute-sql", db.name + ), + query_ms=1.2, + select_templates=[ + f"{'*' if template_name == template.name else ''}{template_name}" + for template_name in templates + ], + ), + request=request, + view_name="table", + ), + headers=headers, + ) + else: + assert False, "Invalid format: {}".format(format_) + # if next_url: + # r.headers["link"] = f'<{next_url}>; rel="next"' + return r + + response = Response.json(data) + + if isinstance(data, dict) and data.get("ok") is False: + # TODO: Other error codes? + + response.status_code = 400 + + if datasette.cors: + add_cors_headers(response.headers) + + return response + + +async def query_view_data( + request, + datasette, + canned_query=None, + _size=None, + named_parameters=None, + write=False, + force_shape=None, +): + db = await datasette.resolve_database(request) database = db.name # TODO: Why do I do this? Is it to eliminate multi-args? # It's going to break ?_extra=...&_extra=... @@ -898,11 +1039,11 @@ async def query_view( # TODO: Behave differently for canned query here: await datasette.ensure_permissions(request.actor, [("execute-sql", database)]) - _shape = None + _shape = force_shape if "_shape" in params: _shape = params.pop("_shape") - # ?_shape=arrays - "rows" is the default option, shown above + # ?_shape=arrays # ?_shape=objects - "rows" is a list of JSON key/value objects # ?_shape=array - an JSON array of objects # ?_shape=array&_nl=on - a newline-separated list of JSON objects @@ -921,6 +1062,7 @@ async def query_view( return {"ok": False, "error": str(error)} return { "ok": True, + "columns": [r[0] for r in results.description], "rows": [list(r) for r in results.rows], "truncated": results.truncated, } @@ -978,17 +1120,7 @@ async def query_view( output = results["_shape"] output.update(dict((k, v) for k, v in results.items() if not k.startswith("_"))) - response = Response.json(output) - - if isinstance(output, dict) and output.get("ok") is False: - # TODO: Other error codes? - - response.status_code = 400 - - if datasette.cors: - add_cors_headers(response.headers) - - return response + return output # registry = Registry( # extra_count, From bbbfdb034ca02c1aa3f02de0e8edf12d8190175c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 20:36:10 -0700 Subject: [PATCH 10/19] Add setuptools to dependencies Refs #2065 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b591869e..1139ff31 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ setup( "itsdangerous>=1.1", "sqlite-utils>=3.30", "asyncinject>=0.6", + "setuptools", ], entry_points=""" [console_scripts] From 3a7be0c5b1911554b7f7298ef459259f24a1a9d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 21:20:38 -0700 Subject: [PATCH 11/19] Hopeful fix for Python 3.7 httpx failure, refs #2066 --- tests/test_csv.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/test_csv.py b/tests/test_csv.py index c43e528b..ed83d685 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -6,6 +6,7 @@ from .fixtures import ( # noqa app_client_with_cors, app_client_with_trace, ) +import urllib.parse EXPECTED_TABLE_CSV = """id,content 1,hello @@ -154,11 +155,24 @@ async def test_csv_with_non_ascii_characters(ds_client): def test_max_csv_mb(app_client_csv_max_mb_one): + # This query deliberately generates a really long string + # should be 100*100*100*2 = roughly 2MB response = app_client_csv_max_mb_one.get( - ( - "/fixtures.csv?sql=select+'{}'+" - "from+compound_three_primary_keys&_stream=1&_size=max" - ).format("abcdefg" * 10000) + "/fixtures.csv?" + + urllib.parse.urlencode( + { + "sql": """ + select group_concat('ab', '') + from json_each(json_array({lots})), + json_each(json_array({lots})), + json_each(json_array({lots})) + """.format( + lots=", ".join(str(i) for i in range(100)) + ), + "_stream": 1, + "_size": "max", + } + ), ) # It's a 200 because we started streaming before we knew the error assert response.status == 200 From 305655c816a70fdfa729b37529d8c51490d4f4d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 26 Apr 2023 22:07:35 -0700 Subject: [PATCH 12/19] Add pip as a dependency too, for Rye - refs #2065 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1139ff31..0eedadb2 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ setup( "sqlite-utils>=3.30", "asyncinject>=0.6", "setuptools", + "pip", ], entry_points=""" [console_scripts] From 01353c7ee83ba2c58b42bdc35f3b26dfc06159ac Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 7 May 2023 11:44:27 -0700 Subject: [PATCH 13/19] Build docs with 3.11 on ReadTheDocs Inspired by https://github.com/simonw/sqlite-utils/issues/540 --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e157fb9c..5b30e75a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.11" sphinx: configuration: docs/conf.py From 59c52b5874045d884bac8bac35a4ecbfdde4e73c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 May 2023 09:24:28 -0700 Subject: [PATCH 14/19] Action: Deploy a Datasette branch preview to Vercel Closes #2070 --- .github/workflows/deploy-branch-preview.yml | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/deploy-branch-preview.yml diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml new file mode 100644 index 00000000..872aff71 --- /dev/null +++ b/.github/workflows/deploy-branch-preview.yml @@ -0,0 +1,35 @@ +name: Deploy a Datasette branch preview to Vercel + +on: + workflow_dispatch: + inputs: + branch: + description: "Branch to deploy" + required: true + type: string + +jobs: + deploy-branch-preview: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: | + pip install datasette-publish-vercel + - name: Deploy the preview + env: + VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} + run: | + export BRANCH="${{ github.event.inputs.branch }}" + wget https://latest.datasette.io/fixtures.db + datasette publish vercel fixtures.db \ + --branch $BRANCH \ + --project "datasette-preview-$BRANCH" \ + --token $VERCEL_TOKEN \ + --scope datasette \ + --about "Preview of $BRANCH" \ + --about_url "https://github.com/simonw/datasette/tree/$BRANCH" From 3f39fba7ea74e54673cce8a71d6c1baeaa84c513 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 11:35:34 -0700 Subject: [PATCH 15/19] datasette.utils.check_callable(obj) - refs #2078 --- datasette/utils/callable.py | 25 ++++++++++++++++++++ tests/test_utils_callable.py | 46 ++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 datasette/utils/callable.py create mode 100644 tests/test_utils_callable.py diff --git a/datasette/utils/callable.py b/datasette/utils/callable.py new file mode 100644 index 00000000..5b8a30ac --- /dev/null +++ b/datasette/utils/callable.py @@ -0,0 +1,25 @@ +import asyncio +import types +from typing import NamedTuple, Any + + +class CallableStatus(NamedTuple): + is_callable: bool + is_async_callable: bool + + +def check_callable(obj: Any) -> CallableStatus: + if not callable(obj): + return CallableStatus(False, False) + + if isinstance(obj, type): + # It's a class + return CallableStatus(True, False) + + if isinstance(obj, types.FunctionType): + return CallableStatus(True, asyncio.iscoroutinefunction(obj)) + + if hasattr(obj, "__call__"): + return CallableStatus(True, asyncio.iscoroutinefunction(obj.__call__)) + + assert False, "obj {} is somehow callable with no __call__ method".format(repr(obj)) diff --git a/tests/test_utils_callable.py b/tests/test_utils_callable.py new file mode 100644 index 00000000..d1d0aac5 --- /dev/null +++ b/tests/test_utils_callable.py @@ -0,0 +1,46 @@ +from datasette.utils.callable import check_callable +import pytest + + +class AsyncClass: + async def __call__(self): + pass + + +class NotAsyncClass: + def __call__(self): + pass + + +class ClassNoCall: + pass + + +async def async_func(): + pass + + +def non_async_func(): + pass + + +@pytest.mark.parametrize( + "obj,expected_is_callable,expected_is_async_callable", + ( + (async_func, True, True), + (non_async_func, True, False), + (AsyncClass(), True, True), + (NotAsyncClass(), True, False), + (ClassNoCall(), False, False), + (AsyncClass, True, False), + (NotAsyncClass, True, False), + (ClassNoCall, True, False), + ("", False, False), + (1, False, False), + (str, True, False), + ), +) +def test_check_callable(obj, expected_is_callable, expected_is_async_callable): + status = check_callable(obj) + assert status.is_callable == expected_is_callable + assert status.is_async_callable == expected_is_async_callable From 080577106167046428c946c9978631dd59e10181 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 11:49:40 -0700 Subject: [PATCH 16/19] Rename callable.py to check_callable.py, refs #2078 --- datasette/utils/{callable.py => check_callable.py} | 0 tests/{test_utils_callable.py => test_utils_check_callable.py} | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename datasette/utils/{callable.py => check_callable.py} (100%) rename tests/{test_utils_callable.py => test_utils_check_callable.py} (94%) diff --git a/datasette/utils/callable.py b/datasette/utils/check_callable.py similarity index 100% rename from datasette/utils/callable.py rename to datasette/utils/check_callable.py diff --git a/tests/test_utils_callable.py b/tests/test_utils_check_callable.py similarity index 94% rename from tests/test_utils_callable.py rename to tests/test_utils_check_callable.py index d1d0aac5..4f72f9ff 100644 --- a/tests/test_utils_callable.py +++ b/tests/test_utils_check_callable.py @@ -1,4 +1,4 @@ -from datasette.utils.callable import check_callable +from datasette.utils.check_callable import check_callable import pytest From 94882aa72b5ef4afb267447de27f0ec2140400d8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 15:05:58 -0700 Subject: [PATCH 17/19] --cors Access-Control-Max-Age: 3600, closes #2079 --- datasette/utils/__init__.py | 1 + docs/json_api.rst | 18 +++++++++++++++++- tests/test_api.py | 2 ++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 925c6560..c388673d 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1141,6 +1141,7 @@ def add_cors_headers(headers): headers["Access-Control-Allow-Headers"] = "Authorization, Content-Type" headers["Access-Control-Expose-Headers"] = "Link" headers["Access-Control-Allow-Methods"] = "GET, POST, HEAD, OPTIONS" + headers["Access-Control-Max-Age"] = "3600" _TILDE_ENCODING_SAFE = frozenset( diff --git a/docs/json_api.rst b/docs/json_api.rst index 7b130c58..c273c2a8 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -454,12 +454,28 @@ Enabling CORS ------------- If you start Datasette with the ``--cors`` option, each JSON endpoint will be -served with the following additional HTTP headers:: +served with the following additional HTTP headers: + +.. [[[cog + from datasette.utils import add_cors_headers + import textwrap + headers = {} + add_cors_headers(headers) + output = "\n".join("{}: {}".format(k, v) for k, v in headers.items()) + cog.out("\n::\n\n") + cog.out(textwrap.indent(output, ' ')) + cog.out("\n\n") +.. ]]] + +:: Access-Control-Allow-Origin: * Access-Control-Allow-Headers: Authorization, Content-Type Access-Control-Expose-Headers: Link Access-Control-Allow-Methods: GET, POST, HEAD, OPTIONS + Access-Control-Max-Age: 3600 + +.. [[[end]]] This allows JavaScript running on any domain to make cross-origin requests to interact with the Datasette API. diff --git a/tests/test_api.py b/tests/test_api.py index e700ad7b..e7d8d849 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -936,6 +936,7 @@ def test_cors( assert ( response.headers["Access-Control-Allow-Methods"] == "GET, POST, HEAD, OPTIONS" ) + assert response.headers["Access-Control-Max-Age"] == "3600" # Same request to app_client_two_attached_databases_one_immutable # should not have those headers - I'm using that fixture because # regular app_client doesn't have immutable fixtures.db which means @@ -946,6 +947,7 @@ def test_cors( assert "Access-Control-Allow-Headers" not in response.headers assert "Access-Control-Expose-Headers" not in response.headers assert "Access-Control-Allow-Methods" not in response.headers + assert "Access-Control-Max-Age" not in response.headers @pytest.mark.parametrize( From 8ea00e038d7d43d173cebdca21a31e774ff33f1c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 May 2023 17:18:43 -0700 Subject: [PATCH 18/19] New View base class (#2080) * New View base class, closes #2078 * Use new View subclass for PatternPortfolioView --- datasette/app.py | 40 +++++++++++++++++- datasette/views/base.py | 37 +++++++++++++++++ datasette/views/special.py | 19 +++++---- tests/test_base_view.py | 84 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 10 deletions(-) create mode 100644 tests/test_base_view.py diff --git a/datasette/app.py b/datasette/app.py index 966a6faf..8bc6518f 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -17,6 +17,7 @@ import secrets import sys import threading import time +import types import urllib.parse from concurrent import futures from pathlib import Path @@ -1366,7 +1367,7 @@ class Datasette: r"/-/allow-debug$", ) add_route( - PatternPortfolioView.as_view(self), + wrap_view(PatternPortfolioView, self), r"/-/patterns$", ) add_route(DatabaseDownload.as_view(self), r"/(?P[^\/\.]+)\.db$") @@ -1682,7 +1683,42 @@ def _cleaner_task_str(task): return _cleaner_task_str_re.sub("", s) -def wrap_view(view_fn, datasette): +def wrap_view(view_fn_or_class, datasette): + is_function = isinstance(view_fn_or_class, types.FunctionType) + if is_function: + return wrap_view_function(view_fn_or_class, datasette) + else: + if not isinstance(view_fn_or_class, type): + raise ValueError("view_fn_or_class must be a function or a class") + return wrap_view_class(view_fn_or_class, datasette) + + +def wrap_view_class(view_class, datasette): + async def async_view_for_class(request, send): + instance = view_class() + if inspect.iscoroutinefunction(instance.__call__): + return await async_call_with_supported_arguments( + instance.__call__, + scope=request.scope, + receive=request.receive, + send=send, + request=request, + datasette=datasette, + ) + else: + return call_with_supported_arguments( + instance.__call__, + scope=request.scope, + receive=request.receive, + send=send, + request=request, + datasette=datasette, + ) + + return async_view_for_class + + +def wrap_view_function(view_fn, datasette): @functools.wraps(view_fn) async def async_view_fn(request, send): if inspect.iscoroutinefunction(view_fn): diff --git a/datasette/views/base.py b/datasette/views/base.py index 927d1aff..94645cd8 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -53,6 +53,43 @@ class DatasetteError(Exception): self.message_is_html = message_is_html +class View: + async def head(self, request, datasette): + if not hasattr(self, "get"): + return await self.method_not_allowed(request) + response = await self.get(request, datasette) + response.body = "" + return response + + async def method_not_allowed(self, request): + if ( + request.path.endswith(".json") + or request.headers.get("content-type") == "application/json" + ): + response = Response.json( + {"ok": False, "error": "Method not allowed"}, status=405 + ) + else: + response = Response.text("Method not allowed", status=405) + return response + + async def options(self, request, datasette): + response = Response.text("ok") + response.headers["allow"] = ", ".join( + method.upper() + for method in ("head", "get", "post", "put", "patch", "delete") + if hasattr(self, method) + ) + return response + + async def __call__(self, request, datasette): + try: + handler = getattr(self, request.method.lower()) + except AttributeError: + return await self.method_not_allowed(request) + return await handler(request, datasette) + + class BaseView: ds = None has_json_alternate = True diff --git a/datasette/views/special.py b/datasette/views/special.py index 1aeb4be6..03e085d6 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -6,7 +6,7 @@ from datasette.utils import ( tilde_encode, tilde_decode, ) -from .base import BaseView +from .base import BaseView, View import secrets import urllib @@ -57,13 +57,16 @@ class JsonDataView(BaseView): ) -class PatternPortfolioView(BaseView): - name = "patterns" - has_json_alternate = False - - async def get(self, request): - await self.ds.ensure_permissions(request.actor, ["view-instance"]) - return await self.render(["patterns.html"], request=request) +class PatternPortfolioView(View): + async def get(self, request, datasette): + await datasette.ensure_permissions(request.actor, ["view-instance"]) + return Response.html( + await datasette.render_template( + "patterns.html", + request=request, + view_name="patterns", + ) + ) class AuthTokenView(BaseView): diff --git a/tests/test_base_view.py b/tests/test_base_view.py new file mode 100644 index 00000000..2cd4d601 --- /dev/null +++ b/tests/test_base_view.py @@ -0,0 +1,84 @@ +from datasette.views.base import View +from datasette import Request, Response +from datasette.app import Datasette +import json +import pytest + + +class GetView(View): + async def get(self, request, datasette): + return Response.json( + { + "absolute_url": datasette.absolute_url(request, "/"), + "request_path": request.path, + } + ) + + +class GetAndPostView(GetView): + async def post(self, request, datasette): + return Response.json( + { + "method": request.method, + "absolute_url": datasette.absolute_url(request, "/"), + "request_path": request.path, + } + ) + + +@pytest.mark.asyncio +async def test_get_view(): + v = GetView() + datasette = Datasette() + response = await v(Request.fake("/foo"), datasette) + assert json.loads(response.body) == { + "absolute_url": "http://localhost/", + "request_path": "/foo", + } + # Try a HEAD request + head_response = await v(Request.fake("/foo", method="HEAD"), datasette) + assert head_response.body == "" + assert head_response.status == 200 + # And OPTIONS + options_response = await v(Request.fake("/foo", method="OPTIONS"), datasette) + assert options_response.body == "ok" + assert options_response.status == 200 + assert options_response.headers["allow"] == "HEAD, GET" + # And POST + post_response = await v(Request.fake("/foo", method="POST"), datasette) + assert post_response.body == "Method not allowed" + assert post_response.status == 405 + # And POST with .json extension + post_json_response = await v(Request.fake("/foo.json", method="POST"), datasette) + assert json.loads(post_json_response.body) == { + "ok": False, + "error": "Method not allowed", + } + assert post_json_response.status == 405 + + +@pytest.mark.asyncio +async def test_post_view(): + v = GetAndPostView() + datasette = Datasette() + response = await v(Request.fake("/foo"), datasette) + assert json.loads(response.body) == { + "absolute_url": "http://localhost/", + "request_path": "/foo", + } + # Try a HEAD request + head_response = await v(Request.fake("/foo", method="HEAD"), datasette) + assert head_response.body == "" + assert head_response.status == 200 + # And OPTIONS + options_response = await v(Request.fake("/foo", method="OPTIONS"), datasette) + assert options_response.body == "ok" + assert options_response.status == 200 + assert options_response.headers["allow"] == "HEAD, GET, POST" + # And POST + post_response = await v(Request.fake("/foo", method="POST"), datasette) + assert json.loads(post_response.body) == { + "method": "POST", + "absolute_url": "http://localhost/", + "request_path": "/foo", + } From 68223784167fdec4e7ebfca56002a6548ba7b423 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Jun 2023 19:00:33 -0700 Subject: [PATCH 19/19] Prototype of rst_docs_for_dataclass mechanism, refs #1510 --- datasette/context.py | 97 +++++++++++++++++++++++++++++++++++++++ docs/conf.py | 7 ++- docs/index.rst | 1 + docs/jsoncontext.py | 28 +++++++++++ docs/template_context.rst | 29 ++++++++++++ 5 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 datasette/context.py create mode 100644 docs/jsoncontext.py create mode 100644 docs/template_context.rst diff --git a/datasette/context.py b/datasette/context.py new file mode 100644 index 00000000..c5927d1d --- /dev/null +++ b/datasette/context.py @@ -0,0 +1,97 @@ +from typing import List, Dict, Optional, Any +from dataclasses import dataclass, field + + +def doc(documentation): + return field(metadata={"doc": documentation}) + + +def is_builtin_type(obj): + return isinstance( + obj, + tuple( + x.__class__ + for x in (int, float, str, bool, bytes, list, tuple, dict, set, frozenset) + ), + ) + + +def rst_docs_for_dataclass(klass: Any) -> str: + """Generate reStructuredText (reST) docs for a dataclass.""" + docs = [] + + # Class name and docstring + docs.append(klass.__name__) + docs.append("-" * len(klass.__name__)) + docs.append("") + if klass.__doc__: + docs.append(klass.__doc__) + docs.append("") + + # Dataclass fields + docs.append("Fields") + docs.append("~~~~~~") + docs.append("") + + for name, field_info in klass.__dataclass_fields__.items(): + if is_builtin_type(field_info.type): + # + type_name = field_info.type.__name__ + else: + # List[str] + type_name = str(field_info.type).replace("typing.", "") + docs.append(f':{name} - ``{type_name}``: {field_info.metadata.get("doc", "")}') + + return "\n".join(docs) + + +@dataclass +class ForeignKey: + incoming: List[Dict] + outgoing: List[Dict] + + +@dataclass +class Table: + "A table is a useful thing" + name: str = doc("The name of the table") + columns: List[str] = doc("List of column names in the table") + primary_keys: List[str] = doc("List of column names that are primary keys") + count: int = doc("Number of rows in the table") + hidden: bool = doc( + "Should this table default to being hidden in the main database UI?" + ) + fts_table: Optional[str] = doc( + "If this table has FTS support, the accompanying FTS table name" + ) + foreign_keys: ForeignKey = doc("List of foreign keys for this table") + private: bool = doc("Private tables are not visible to signed-out anonymous users") + + +@dataclass +class View: + name: str + private: bool + + +@dataclass +class Query: + title: str + sql: str + name: str + private: bool + + +@dataclass +class Database: + content: str + private: bool + path: str + size: int + tables: List[Table] + hidden_count: int + views: List[View] + queries: List[Query] + allow_execute_sql: bool + table_columns: Dict[str, List[str]] + query_ms: float diff --git a/docs/conf.py b/docs/conf.py index c25d8a95..5423fa2a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,12 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["sphinx.ext.extlinks", "sphinx.ext.autodoc", "sphinx_copybutton"] +extensions = [ + "sphinx.ext.extlinks", + "sphinx.ext.autodoc", + "sphinx_copybutton", + "jsoncontext", +] extlinks = { "issue": ("https://github.com/simonw/datasette/issues/%s", "#%s"), diff --git a/docs/index.rst b/docs/index.rst index 5a9cc7ed..254ed3da 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -57,6 +57,7 @@ Contents settings introspection custom_templates + template_context plugins writing_plugins plugin_hooks diff --git a/docs/jsoncontext.py b/docs/jsoncontext.py new file mode 100644 index 00000000..d759f89f --- /dev/null +++ b/docs/jsoncontext.py @@ -0,0 +1,28 @@ +from docutils import nodes +from sphinx.util.docutils import SphinxDirective +from importlib import import_module +import json + + +class JSONContextDirective(SphinxDirective): + required_arguments = 1 + + def run(self): + module_path, class_name = self.arguments[0].rsplit(".", 1) + try: + module = import_module(module_path) + dataclass = getattr(module, class_name) + except ImportError: + warning = f"Unable to import {self.arguments[0]}" + return [nodes.error(None, nodes.paragraph(text=warning))] + + doc = json.dumps( + dataclass.__annotations__, indent=4, sort_keys=True, default=repr + ) + doc_node = nodes.literal_block(text=doc) + + return [doc_node] + + +def setup(app): + app.add_directive("jsoncontext", JSONContextDirective) diff --git a/docs/template_context.rst b/docs/template_context.rst new file mode 100644 index 00000000..f1de4f9e --- /dev/null +++ b/docs/template_context.rst @@ -0,0 +1,29 @@ +.. _template_context: + +Template context +================ + +This page describes the variables made available to templates used by Datasette to render different pages of the application. + + +.. [[[cog + from datasette.context import rst_docs_for_dataclass, Table + cog.out(rst_docs_for_dataclass(Table)) +.. ]]] +Table +----- + +A table is a useful thing + +Fields +~~~~~~ + +:name - ``str``: The name of the table +:columns - ``List[str]``: List of column names in the table +:primary_keys - ``List[str]``: List of column names that are primary keys +:count - ``int``: Number of rows in the table +:hidden - ``bool``: Should this table default to being hidden in the main database UI? +:fts_table - ``Optional[str]``: If this table has FTS support, the accompanying FTS table name +:foreign_keys - ``ForeignKey``: List of foreign keys for this table +:private - ``bool``: Private tables are not visible to signed-out anonymous users +.. [[[end]]]