From dfd4ad558b74defbe23b01196260b087f9a56813 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 25 Feb 2024 12:54:16 -0800 Subject: [PATCH 001/334] New design for table and database action menus Closes #2281 --- datasette/static/app.css | 45 +++++++++++++++++++++-------- datasette/templates/database.html | 47 +++++++++++++++++------------- datasette/templates/patterns.html | 48 +++++++++++++++++++------------ datasette/templates/table.html | 46 ++++++++++++++++------------- 4 files changed, 116 insertions(+), 70 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 80dfc677..5453a3d4 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -163,28 +163,22 @@ h6, } .page-header { - display: flex; - align-items: center; padding-left: 10px; border-left: 10px solid #666; margin-bottom: 0.75rem; margin-top: 1rem; } .page-header h1 { - display: inline; margin: 0; font-size: 2rem; padding-right: 0.2em; } -.page-header details { - display: inline-flex; -} -.page-header details > summary { + +.page-action-menu details > summary { list-style: none; - display: inline-flex; cursor: pointer; } -.page-header details > summary::-webkit-details-marker { +.page-action-menu details > summary::-webkit-details-marker { display: none; } @@ -364,13 +358,40 @@ details .nav-menu-inner { } /* Table/database actions menu */ -.page-header { +.page-action-menu { position: relative; } +.actions-menu-links { + display: inline; +} .actions-menu-links .dropdown-menu { position: absolute; top: calc(100% + 10px); - left: -10px; + left: 0; +} +.page-action-menu .icon-text { + display: inline-flex; + align-items: center; + border-radius: .25rem; + padding: 5px 12px 3px 7px; + color: #fff; + font-weight: 400; + font-size: 0.8em; + background: linear-gradient(180deg, #007bff 0%, #4E79C7 100%); + border-color: #007bff; +} +.page-action-menu .icon-text span { + /* Nudge text up a bit */ + position: relative; + top: -2px; +} +.page-action-menu .icon-text:hover { + cursor: pointer; +} +.page-action-menu .icon { + width: 18px; + height: 18px; + margin-right: 4px; } /* Components ============================================================== */ @@ -536,7 +557,7 @@ form input[type=submit], form button[type=button] { form input[type=submit] { color: #fff; - background-color: #007bff; + background: linear-gradient(180deg, #007bff 0%, #4E79C7 100%); border-color: #007bff; -webkit-appearance: button; } diff --git a/datasette/templates/database.html b/datasette/templates/database.html index ee4dd705..6c0cebcd 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -12,27 +12,34 @@ {% block content %}

{{ metadata.title or database }}{% if private %} 🔒{% endif %}

- {% set links = database_actions() %}{% if links %} -
- - Table actions - - - -
- {% if links %} - - {% endif %} +
+{% set links = database_actions() %}{% if links %} +
+
+ +
+ + Database actions + + + + Database actions
-
{% endif %} -
+ +
+
+ {% if links %} + + {% endif %} +
+
+
+{% endif %} + {{ top_database() }} diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index 9905df2c..33db3d1a 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -96,18 +96,24 @@

fixtures

+
+
- - Table actions - - - + +
+ + Database actions + + + + Database actions +
+
+
@@ -158,18 +164,24 @@

roadside_attraction_characteristics

+
+
- - Table actions - - - + +
+ + Database actions + + + + Table actions +
+
+
diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 5aee6319..0c2be672 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -23,27 +23,33 @@ {% block content %}

{{ metadata.get("title") or table }}{% if is_view %} (view){% endif %}{% if private %} 🔒{% endif %}

- {% set links = table_actions() %}{% if links %} -
- - Table actions - - - -
- {% if links %} - - {% endif %} -
-
{% endif %}
+{% set links = table_actions() %}{% if links %} +
+
+ +
+ + Table actions + + + + Table actions +
+ +
+
+ {% if links %} + + {% endif %} +
+
+
+{% endif %} {{ top_table() }} From c863443ea11ea9b9f0e264ab38f103691509fab9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Feb 2024 13:24:47 -0800 Subject: [PATCH 002/334] Documentation for derive_named_parameters() Closes #2284 Refs https://github.com/simonw/datasette-write/issues/7#issuecomment-1967593883 --- datasette/utils/__init__.py | 12 ++++++++++-- docs/internals.rst | 9 +++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index e3637f7a..4adfcc8d 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -18,12 +18,14 @@ import time import types import secrets import shutil -from typing import Iterable, Tuple +from typing import Iterable, List, Tuple import urllib import yaml from .shutil_backport import copytree from .sqlite import sqlite3, supports_table_xinfo +if typing.TYPE_CHECKING: + from datasette.database import Database # From https://www.sqlite.org/lang_keywords.html reserved_words = set( @@ -1130,7 +1132,13 @@ class StartupError(Exception): _re_named_parameter = re.compile(":([a-zA-Z0-9_]+)") -async def derive_named_parameters(db, sql): +@documented +async def derive_named_parameters(db: "Database", sql: str) -> List[str]: + """ + Given a SQL statement, return a list of named parameters that are used in the statement + + e.g. for ``select * from foo where id=:id`` this would return ``["id"]`` + """ explain = "explain {}".format(sql.strip().rstrip(";")) possible_params = _re_named_parameter.findall(sql) try: diff --git a/docs/internals.rst b/docs/internals.rst index 6ca62423..e9f8b391 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1256,6 +1256,15 @@ Utility function for calling ``await`` on a return value if it is awaitable, oth .. autofunction:: datasette.utils.await_me_maybe +.. _internals_utils_derive_named_parameters: + +derive_named_parameters(db, sql) +-------------------------------- + +Derive the list of named parameters referenced in a SQL query, using an ``explain`` query executed against the provided database. + +.. autofunction:: datasette.utils.derive_named_parameters + .. _internals_tilde_encoding: Tilde encoding From f99c2f5f8cd1550442f69def0b27f8ec799d6bd8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Feb 2024 16:07:37 -0800 Subject: [PATCH 003/334] ?column_notcontains= table filter, closes #2287 --- datasette/filters.py | 7 +++++++ docs/json_api.rst | 3 +++ tests/test_filters.py | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/datasette/filters.py b/datasette/filters.py index 4d9580d8..585d4865 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -281,6 +281,13 @@ class Filters: '{c} contains "{v}"', format="%{}%", ), + TemplatedFilter( + "notcontains", + "does not contain", + '"{c}" not like :{p}', + '{c} does not contain "{v}"', + format="%{}%", + ), TemplatedFilter( "endswith", "ends with", diff --git a/docs/json_api.rst b/docs/json_api.rst index 366f74b2..4b39a048 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -237,6 +237,9 @@ You can filter the data returned by the table based on column values using a que ``?column__contains=value`` Rows where the string column contains the specified value (``column like "%value%"`` in SQL). +``?column__notcontains=value`` + Rows where the string column does not contain the specified value (``column not like "%value%"`` in SQL). + ``?column__endswith=value`` Rows where the string column ends with the specified value (``column like "%value"`` in SQL). diff --git a/tests/test_filters.py b/tests/test_filters.py index 5b2e9636..a3fada98 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -7,6 +7,11 @@ import pytest "args,expected_where,expected_params", [ ((("name_english__contains", "foo"),), ['"name_english" like :p0'], ["%foo%"]), + ( + (("name_english__notcontains", "foo"),), + ['"name_english" not like :p0'], + ["%foo%"], + ), ( (("foo", "bar"), ("bar__contains", "baz")), ['"bar" like :p0', '"foo" = :p1'], From 6ec0081f5d827a71c96972e634a2e887edaf5392 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Feb 2024 21:55:16 -0800 Subject: [PATCH 004/334] `query_actions` plugin hook * New query_actions plugin hook, closes #2283 --- datasette/hookspecs.py | 5 ++++ datasette/templates/query.html | 27 ++++++++++++++++++ datasette/views/database.py | 23 +++++++++++++++ docs/plugin_hooks.rst | 52 ++++++++++++++++++++++++++++++++++ tests/fixtures.py | 1 + tests/plugins/my_plugin.py | 18 ++++++++++++ tests/test_plugins.py | 25 ++++++++++++++++ 7 files changed, 151 insertions(+) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index b473f398..1141ca75 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -145,6 +145,11 @@ def table_actions(datasette, actor, database, table, request): """Links for the table actions menu""" +@hookspec +def query_actions(datasette, actor, database, query_name, request, sql, params): + """Links for the query and canned query actions menu""" + + @hookspec def database_actions(datasette, actor, database, request): """Links for the database actions menu""" diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 1815e592..b5991772 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -29,6 +29,33 @@ {% endif %}

{{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

+{% set links = query_actions() %}{% if links %} +
+
+ +
+ + Query actions + + + + Query actions +
+ +
+
+ {% if links %} + + {% endif %} +
+
+
+{% endif %} + {% if canned_query %}{{ top_canned_query() }}{% else %}{{ top_query() }}{% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 56fc6f8c..851ae21f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -9,6 +9,7 @@ import os import re import sqlite_utils import textwrap +from typing import List from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted @@ -256,6 +257,11 @@ class QueryContext: top_canned_query: callable = field( metadata={"help": "Callable to render the top_canned_query slot"} ) + query_actions: callable = field( + metadata={ + "help": "Callable returning a list of links for the query action menu" + } + ) async def get_tables(datasette, request, db): @@ -694,6 +700,22 @@ class QueryView(View): ) ) + async def query_actions(): + query_actions = [] + for hook in pm.hook.query_actions( + datasette=datasette, + actor=request.actor, + database=database, + query_name=canned_query["name"] if canned_query else None, + request=request, + sql=sql, + params=params, + ): + extra_links = await await_me_maybe(hook) + if extra_links: + query_actions.extend(extra_links) + return query_actions + r = Response.html( await datasette.render_template( template, @@ -749,6 +771,7 @@ class QueryView(View): database=database, query_name=canned_query["name"] if canned_query else None, ), + query_actions=query_actions, ), request=request, view_name="database", diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 5372ea5e..3ada41e2 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1520,6 +1520,58 @@ This example adds a new table action if the signed in user is ``"root"``: Example: `datasette-graphql `_ +.. _plugin_hook_query_actions: + +query_actions(datasette, actor, database, query_name, request, sql, params) +--------------------------------------------------------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``database`` - string + The name of the database. + +``query_name`` - string or None + The name of the canned query, or ``None`` if this is an arbitrary SQL query. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``sql`` - string + The SQL query being executed + +``params`` - dictionary + The parameters passed to the SQL query, if any. + +This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the canned query and arbitrary SQL query pages. + +This example adds a new query action linking to a page for explaining a query: + +.. code-block:: python + + from datasette import hookimpl + import urllib + + + @hookimpl + def query_actions(datasette, database, sql): + return [ + { + "href": datasette.urls.database(database) + + "/-/explain?" + + urllib.parse.urlencode( + { + "sql": sql, + } + ), + "label": "Explain this query", + }, + ] + + .. _plugin_hook_database_actions: database_actions(datasette, actor, database, request) diff --git a/tests/fixtures.py b/tests/fixtures.py index bb979d79..c3c77fce 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -46,6 +46,7 @@ EXPECTED_PLUGINS = [ "permission_allowed", "prepare_connection", "prepare_jinja2_environment", + "query_actions", "register_facet_classes", "register_magic_parameters", "register_permissions", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 9d1f86bc..f96441cb 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -7,6 +7,7 @@ from datasette.utils.asgi import asgi_send_json, Response import base64 import pint import json +import urllib ureg = pint.UnitRegistry() @@ -390,6 +391,23 @@ def table_actions(datasette, database, table, actor): ] +@hookimpl +def query_actions(datasette, database, query_name, sql): + args = { + "sql": sql, + } + if query_name: + args["query_name"] = query_name + return [ + { + "href": datasette.urls.database(database) + + "/-/explain?" + + urllib.parse.urlencode(args), + "label": "Explain this query", + }, + ] + + @hookimpl def database_actions(datasette, database, actor, request): if actor: diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 40d01c71..86208371 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -945,6 +945,31 @@ async def test_hook_table_actions(ds_client, table_or_view): ] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path,expected_url", + ( + ("/fixtures?sql=select+1", "/fixtures/-/explain?sql=select+1"), + ( + "/fixtures/pragma_cache_size", + "/fixtures/-/explain?sql=PRAGMA+cache_size%3B&query_name=pragma_cache_size", + ), + ), +) +async def test_hook_query_actions(ds_client, path, expected_url): + def get_table_actions_links(html): + soup = Soup(html, "html.parser") + details = soup.find("details", {"class": "actions-menu-links"}) + if details is None: + return [] + return [{"label": a.text, "href": a["href"]} for a in details.select("a")] + + response = await ds_client.get(path) + assert response.status_code == 200 + links = get_table_actions_links(response.text) + assert links == [{"label": "Explain this query", "href": expected_url}] + + @pytest.mark.asyncio async def test_hook_database_actions(ds_client): def get_table_actions_links(html): From 57c1ce0e8b17deae33eb9033383c6ecfc041104a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Feb 2024 14:25:50 -0800 Subject: [PATCH 005/334] Reset column menu on every click, closes #2289 --- datasette/static/table.js | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/static/table.js b/datasette/static/table.js index 4f81b2e5..909eebf3 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -88,6 +88,7 @@ const initDatasetteTable = function (manager) { function onTableHeaderClick(ev) { ev.preventDefault(); ev.stopPropagation(); + menu.innerHTML = DROPDOWN_HTML; var th = ev.target; while (th.nodeName != "TH") { th = th.parentNode; From 86335dc722d31dbf44c6d4bbffd7c5d2d11b1290 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 29 Feb 2024 14:35:28 -0800 Subject: [PATCH 006/334] Release 1.0a12 Refs #2281, #2283, #2287, #2289 --- datasette/version.py | 2 +- docs/changelog.rst | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index c27e1e06..b2cd10bf 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a11" +__version__ = "1.0a12" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 492b2921..240df141 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v1_0_a12: + +1.0a12 (2024-02-29) +------------------- + +- New :ref:`query_actions() ` plugin hook, similar to :ref:`table_actions() ` and :ref:`database_actions() `. Can be used to add a menu of actions to the canned query or arbitrary SQL query page. (:issue:`2283`) +- New design for the button that opens the query, table and database actions menu. (:issue:`2281`) +- "does not contain" table filter for finding rows that do not contain a string. (:issue:`2287`) +- Fixed a bug in the :ref:`javascript_plugins_makeColumnActions` JavaScript plugin mechanism where the column action menu was not fully reset in between each interaction. (:issue:`2289`) + .. _v1_0_a11: 1.0a11 (2024-02-19) From 5de6797d4a4333653568dd3db24c4df7b7502ba3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Mar 2024 18:06:38 -0800 Subject: [PATCH 007/334] Better demo plugin for query_actions, refs #2293 --- docs/plugin_hooks.rst | 4 ++-- tests/plugins/my_plugin.py | 8 ++++++-- tests/test_plugins.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 3ada41e2..62062bfd 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1561,10 +1561,10 @@ This example adds a new query action linking to a page for explaining a query: return [ { "href": datasette.urls.database(database) - + "/-/explain?" + + "?" + urllib.parse.urlencode( { - "sql": sql, + "sql": "explain " + sql, } ), "label": "Explain this query", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index f96441cb..650cc57d 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -401,8 +401,12 @@ def query_actions(datasette, database, query_name, sql): return [ { "href": datasette.urls.database(database) - + "/-/explain?" - + urllib.parse.urlencode(args), + + "?" + + urllib.parse.urlencode( + { + "sql": "explain " + sql, + } + ), "label": "Explain this query", }, ] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 86208371..4620af51 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -949,10 +949,10 @@ async def test_hook_table_actions(ds_client, table_or_view): @pytest.mark.parametrize( "path,expected_url", ( - ("/fixtures?sql=select+1", "/fixtures/-/explain?sql=select+1"), + ("/fixtures?sql=select+1", "/fixtures?sql=explain+select+1"), ( "/fixtures/pragma_cache_size", - "/fixtures/-/explain?sql=PRAGMA+cache_size%3B&query_name=pragma_cache_size", + "/fixtures?sql=explain+PRAGMA+cache_size%3B", ), ), ) From 4d24bf6b34c0d9ed6e6831f68f625bee7f447b85 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Mar 2024 18:14:55 -0800 Subject: [PATCH 008/334] Don't explain an explain even in the demo, refs #2293 --- docs/plugin_hooks.rst | 6 ++++-- tests/plugins/my_plugin.py | 8 +++----- tests/test_plugins.py | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 62062bfd..ecc8f058 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1557,7 +1557,10 @@ This example adds a new query action linking to a page for explaining a query: @hookimpl - def query_actions(datasette, database, sql): + def query_actions(datasette, database, query_name, sql): + # Don't explain an explain + if sql.lower().startswith("explain"): + return return [ { "href": datasette.urls.database(database) @@ -1571,7 +1574,6 @@ This example adds a new query action linking to a page for explaining a query: }, ] - .. _plugin_hook_database_actions: database_actions(datasette, actor, database, request) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 650cc57d..01324213 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -393,11 +393,9 @@ def table_actions(datasette, database, table, actor): @hookimpl def query_actions(datasette, database, query_name, sql): - args = { - "sql": sql, - } - if query_name: - args["query_name"] = query_name + # Don't explain an explain + if sql.lower().startswith("explain"): + return return [ { "href": datasette.urls.database(database) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 4620af51..d1da16fa 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -954,6 +954,8 @@ async def test_hook_table_actions(ds_client, table_or_view): "/fixtures/pragma_cache_size", "/fixtures?sql=explain+PRAGMA+cache_size%3B", ), + # Don't attempt to explain an explain + ("/fixtures?sql=explain+select+1", None), ), ) async def test_hook_query_actions(ds_client, path, expected_url): @@ -967,7 +969,10 @@ async def test_hook_query_actions(ds_client, path, expected_url): response = await ds_client.get(path) assert response.status_code == 200 links = get_table_actions_links(response.text) - assert links == [{"label": "Explain this query", "href": expected_url}] + if expected_url is None: + assert links == [] + else: + assert links == [{"label": "Explain this query", "href": expected_url}] @pytest.mark.asyncio From c6e8a4a76cedb7d178bf20a37fc331f8a5cfb6b9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 Mar 2024 19:34:57 -0800 Subject: [PATCH 009/334] margin-bottom on .page-action-menu, refs #2286 --- datasette/static/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 5453a3d4..b3223abf 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -360,6 +360,7 @@ details .nav-menu-inner { /* Table/database actions menu */ .page-action-menu { position: relative; + margin-bottom: 0.5em; } .actions-menu-links { display: inline; From 090dff542bee7f716088835b3cc633e9e04b985d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 6 Mar 2024 22:54:06 -0500 Subject: [PATCH 010/334] Action menu descriptions * Refactor tests to extract get_actions_links() helper * Table, database and query action menu items now support optional descriptions Closes #2294 --- datasette/static/app.css | 7 ++++ datasette/templates/database.html | 6 ++- datasette/templates/query.html | 6 ++- datasette/templates/table.html | 6 ++- docs/plugin_hooks.rst | 4 +- tests/plugins/my_plugin.py | 1 + tests/test_plugins.py | 64 ++++++++++++++++--------------- 7 files changed, 59 insertions(+), 35 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index b3223abf..e4a0ee10 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -841,6 +841,13 @@ svg.dropdown-menu-icon { .dropdown-menu a:hover { background-color: #eee; } +.dropdown-menu .dropdown-description { + margin: 0; + color: #666; + font-size: 0.8em; + max-width: 80vw; + white-space: normal; +} .dropdown-menu .hook { display: block; position: absolute; diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 6c0cebcd..02e6fb3d 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -31,7 +31,11 @@ {% if links %} {% endif %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index b5991772..09a29118 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -47,7 +47,11 @@ {% if links %} {% endif %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 0c2be672..1d328366 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -42,7 +42,11 @@ {% if links %} {% endif %} diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index ecc8f058..4f77d75c 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1493,7 +1493,7 @@ table_actions(datasette, actor, database, table, request) ``request`` - :ref:`internals_request` or None The current HTTP request. This can be ``None`` if the request object is not available. -This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items. +This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items, with optional ``"description": "..."`` keys describing each action in more detail. It can alternatively return an ``async def`` awaitable function which returns a list of menu items. @@ -1515,6 +1515,7 @@ This example adds a new table action if the signed in user is ``"root"``: ) ), "label": "Edit schema for this table", + "description": "Add, remove, rename or alter columns for this table.", } ] @@ -1571,6 +1572,7 @@ This example adds a new query action linking to a page for explaining a query: } ), "label": "Explain this query", + "description": "Get a summary of how SQLite executes the query", }, ] diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 01324213..5f000537 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -406,6 +406,7 @@ def query_actions(datasette, database, query_name, sql): } ), "label": "Explain this query", + "description": "Runs a SQLite explain", }, ] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index d1da16fa..9f69e4fa 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -925,26 +925,36 @@ async def test_hook_menu_links(ds_client): @pytest.mark.asyncio @pytest.mark.parametrize("table_or_view", ["facetable", "simple_view"]) async def test_hook_table_actions(ds_client, table_or_view): - def get_table_actions_links(html): - soup = Soup(html, "html.parser") - details = soup.find("details", {"class": "actions-menu-links"}) - if details is None: - return [] - return [{"label": a.text, "href": a["href"]} for a in details.select("a")] - response = await ds_client.get(f"/fixtures/{table_or_view}") - assert get_table_actions_links(response.text) == [] + assert get_actions_links(response.text) == [] response_2 = await ds_client.get(f"/fixtures/{table_or_view}?_bot=1&_hello=BOB") assert sorted( - get_table_actions_links(response_2.text), key=lambda link: link["label"] + get_actions_links(response_2.text), key=lambda link: link["label"] ) == [ - {"label": "Database: fixtures", "href": "/"}, - {"label": "From async BOB", "href": "/"}, - {"label": f"Table: {table_or_view}", "href": "/"}, + {"label": "Database: fixtures", "href": "/", "description": None}, + {"label": "From async BOB", "href": "/", "description": None}, + {"label": f"Table: {table_or_view}", "href": "/", "description": None}, ] +def get_actions_links(html): + soup = Soup(html, "html.parser") + details = soup.find("details", {"class": "actions-menu-links"}) + if details is None: + return [] + links = [] + for a_el in details.select("a"): + description = None + if a_el.find("p") is not None: + description = a_el.find("p").text.strip() + a_el.find("p").extract() + label = a_el.text.strip() + href = a_el["href"] + links.append({"label": label, "href": href, "description": description}) + return links + + @pytest.mark.asyncio @pytest.mark.parametrize( "path,expected_url", @@ -959,37 +969,29 @@ async def test_hook_table_actions(ds_client, table_or_view): ), ) async def test_hook_query_actions(ds_client, path, expected_url): - def get_table_actions_links(html): - soup = Soup(html, "html.parser") - details = soup.find("details", {"class": "actions-menu-links"}) - if details is None: - return [] - return [{"label": a.text, "href": a["href"]} for a in details.select("a")] - response = await ds_client.get(path) assert response.status_code == 200 - links = get_table_actions_links(response.text) + links = get_actions_links(response.text) if expected_url is None: assert links == [] else: - assert links == [{"label": "Explain this query", "href": expected_url}] + assert links == [ + { + "label": "Explain this query", + "href": expected_url, + "description": "Runs a SQLite explain", + } + ] @pytest.mark.asyncio async def test_hook_database_actions(ds_client): - def get_table_actions_links(html): - soup = Soup(html, "html.parser") - details = soup.find("details", {"class": "actions-menu-links"}) - if details is None: - return [] - return [{"label": a.text, "href": a["href"]} for a in details.select("a")] - response = await ds_client.get("/fixtures") - assert get_table_actions_links(response.text) == [] + assert get_actions_links(response.text) == [] response_2 = await ds_client.get("/fixtures?_bot=1&_hello=BOB") - assert get_table_actions_links(response_2.text) == [ - {"label": "Database: fixtures - BOB", "href": "/"}, + assert get_actions_links(response_2.text) == [ + {"label": "Database: fixtures - BOB", "href": "/", "description": None}, ] From a395256c8c10f19bb006e0fce8f5eddd2b645757 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Mar 2024 00:03:20 -0500 Subject: [PATCH 011/334] Allow-list select * from pragma_table_list() Refs https://github.com/simonw/datasette/issues/2104#issuecomment-1982352475 --- datasette/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 4adfcc8d..9c0bbfa3 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -246,6 +246,7 @@ allowed_pragmas = ( "schema_version", "table_info", "table_xinfo", + "table_list", ) disallawed_sql_res = [ ( From 7818e8b9d15a3d50c16f080dc7fe4b5e8eb3d241 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Mar 2024 00:03:42 -0500 Subject: [PATCH 012/334] Hide tables starting with an _, refs #2104 --- datasette/database.py | 1 + docs/pages.rst | 15 +++++++++++++++ tests/test_api.py | 15 +++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/datasette/database.py b/datasette/database.py index 4e590d3a..ffe94ea7 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -469,6 +469,7 @@ class Database: and ( sql like '%VIRTUAL TABLE%USING FTS%' ) or name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4') + or name like '\\_%' escape '\\' """ ) ).rows diff --git a/docs/pages.rst b/docs/pages.rst index 2ce05428..1c3e2c1e 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -40,6 +40,21 @@ The JSON version of this page provides programmatic access to the underlying dat * `fivethirtyeight.datasettes.com/fivethirtyeight.json `_ * `global-power-plants.datasettes.com/global-power-plants.json `_ +.. _DatabaseView_hidden: + +Hidden tables +------------- + +Some tables listed on the database page are treated as hidden. Hidden tables are not completely invisible - they can be accessed through the "hidden tables" link at the bottom of the page. They are hidden because they represent low-level implementation details which are generally not useful to end-users of Datasette. + +The following tables are hidden by default: + +- Any table with a name that starts with an underscore - this is a Datasette convention to help plugins easily hide their own internal tables. +- Tables that have been configured as ``"hidden": true`` using :ref:`metadata_hiding_tables`. +- ``*_fts`` tables that implement SQLite full-text search indexes. +- Tables relating to the inner workings of the SpatiaLite SQLite extension. +- ``sqlite_stat`` tables used to store statistics used by the query optimizer. + .. _TableView: Table diff --git a/tests/test_api.py b/tests/test_api.py index 7a25b55e..4ad55d72 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1018,6 +1018,21 @@ async def test_hidden_sqlite_stat1_table(): ) +@pytest.mark.asyncio +async def test_hide_tables_starting_with_underscore(): + ds = Datasette() + db = ds.add_memory_database("test_hide_tables_starting_with_underscore") + await db.execute_write("create table normal (id integer primary key, name text)") + await db.execute_write("create table _hidden (id integer primary key, name text)") + data = ( + await ds.client.get( + "/test_hide_tables_starting_with_underscore.json?_show_hidden=1" + ) + ).json() + tables = [(t["name"], t["hidden"]) for t in data["tables"]] + assert tables == [("normal", False), ("_hidden", True)] + + @pytest.mark.asyncio @pytest.mark.parametrize("db_name", ("foo", r"fo%o", "f~/c.d")) async def test_tilde_encoded_database_names(db_name): From 7b32d5f7d83e8373e52ceef6198a6737b2c10f71 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 7 Mar 2024 00:11:14 -0500 Subject: [PATCH 013/334] datasette-create-view as example of query_actions hook --- docs/plugin_hooks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4f77d75c..91db80f8 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1576,6 +1576,8 @@ This example adds a new query action linking to a page for explaining a query: }, ] +Example: `datasette-create-view `_ + .. _plugin_hook_database_actions: database_actions(datasette, actor, database, request) From daf5ca02ca9df56de1f920a6bd20e81dbad2686c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 13:44:07 -0700 Subject: [PATCH 014/334] homepage_actions() plugin hook, closes #2298 --- datasette/hookspecs.py | 5 +++++ datasette/templates/database.html | 1 - datasette/templates/index.html | 31 +++++++++++++++++++++++++++++ datasette/views/index.py | 18 ++++++++++++++++- docs/plugin_hooks.rst | 33 +++++++++++++++++++++++++++++++ tests/fixtures.py | 1 + tests/plugins/my_plugin.py | 12 +++++++++++ tests/test_plugins.py | 19 ++++++++++++++++++ 8 files changed, 118 insertions(+), 2 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 1141ca75..5a8439b4 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -155,6 +155,11 @@ def database_actions(datasette, actor, database, request): """Links for the database actions menu""" +@hookspec +def homepage_actions(datasette, actor, request): + """Links for the homepage actions menu""" + + @hookspec def skip_csrf(datasette, scope): """Mechanism for skipping CSRF checks for certain requests""" diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 02e6fb3d..b5a0edc4 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -44,7 +44,6 @@
{% endif %} - {{ top_database() }} {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/templates/index.html b/datasette/templates/index.html index 203abca8..d08cdb10 100644 --- a/datasette/templates/index.html +++ b/datasette/templates/index.html @@ -7,6 +7,37 @@ {% block content %}

{{ metadata.title or "Datasette" }}{% if private %} 🔒{% endif %}

+{% set links = homepage_actions %}{% if links %} +
+
+ +
+ + Homepage actions + + + + Homepage actions +
+ +
+
+ {% if links %} + + {% endif %} +
+
+
+{% endif %} + {{ top_homepage() }} {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/views/index.py b/datasette/views/index.py index 2cb18b1c..6546b7ae 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -1,6 +1,12 @@ import json -from datasette.utils import add_cors_headers, make_slot_function, CustomJSONEncoder +from datasette.plugins import pm +from datasette.utils import ( + add_cors_headers, + await_me_maybe, + make_slot_function, + CustomJSONEncoder, +) from datasette.utils.asgi import Response from datasette.version import __version__ @@ -131,6 +137,15 @@ class IndexView(BaseView): headers=headers, ) else: + homepage_actions = [] + for hook in pm.hook.homepage_actions( + datasette=self.ds, + actor=request.actor, + request=request, + ): + extra_links = await await_me_maybe(hook) + if extra_links: + homepage_actions.extend(extra_links) return await self.render( ["index.html"], request=request, @@ -144,5 +159,6 @@ class IndexView(BaseView): "top_homepage": make_slot_function( "top_homepage", self.ds, request ), + "homepage_actions": homepage_actions, }, ) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 91db80f8..4ac391ef 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1629,6 +1629,39 @@ This example adds a new database action for creating a table, if the user has th Example: `datasette-graphql `_, `datasette-edit-schema `_ +.. _plugin_hook_homepage_actions: + +homepage_actions(datasette, actor, request) +------------------------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``request`` - :ref:`internals_request` + The current HTTP request. + +This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the index page of the Datasette instance. + +This example adds a link an imagined tool for editing the homepage, only for signed in users: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def homepage_actions(datasette, actor): + if actor: + return [ + { + "href": datasette.urls.path("/-/customize-homepage"), + "label": "Customize homepage", + } + ] + .. _plugin_hook_skip_csrf: skip_csrf(datasette, scope) diff --git a/tests/fixtures.py b/tests/fixtures.py index c3c77fce..61d240da 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -42,6 +42,7 @@ EXPECTED_PLUGINS = [ "extra_js_urls", "extra_template_vars", "forbidden", + "homepage_actions", "menu_links", "permission_allowed", "prepare_connection", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 5f000537..94267f04 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -425,6 +425,18 @@ def database_actions(datasette, database, actor, request): ] +@hookimpl +def homepage_actions(datasette, actor, request): + if actor: + label = f"Custom homepage for: {actor['id']}" + return [ + { + "href": datasette.urls.path("/-/custom-homepage"), + "label": label, + } + ] + + @hookimpl def skip_csrf(scope): return scope["path"] == "/skip-csrf" diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 9f69e4fa..dcc5de20 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -995,6 +995,25 @@ async def test_hook_database_actions(ds_client): ] +@pytest.mark.asyncio +async def test_hook_homepage_actions(ds_client): + response = await ds_client.get("/") + # No button for anonymous users + assert "Homepage actions" not in response.text + # Signed in user gets an action + response2 = await ds_client.get( + "/", cookies={"ds_actor": ds_client.actor_cookie({"id": "troy"})} + ) + assert "Homepage actions" in response2.text + assert get_actions_links(response2.text) == [ + { + "label": "Custom homepage for: troy", + "href": "/-/custom-homepage", + "description": None, + }, + ] + + def test_hook_skip_csrf(app_client): cookie = app_client.actor_cookie({"id": "test"}) csrf_response = app_client.post( From 909c85cd2b64646edaaa8e3c19f5cd4e51a13fad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 14:25:07 -0700 Subject: [PATCH 015/334] view_actions plugin hook, closes #2297 --- datasette/hookspecs.py | 5 +++++ datasette/templates/table.html | 6 +++--- datasette/views/table.py | 30 ++++++++++++++++++------------ docs/plugin_hooks.rst | 26 +++++++++++++++++++++++++- tests/fixtures.py | 1 + tests/plugins/my_plugin.py | 12 ++++++++++++ tests/test_plugins.py | 26 +++++++++++++++++++++----- 7 files changed, 85 insertions(+), 21 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 5a8439b4..6ce1e85e 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -145,6 +145,11 @@ def table_actions(datasette, actor, database, table, request): """Links for the table actions menu""" +@hookspec +def view_actions(datasette, actor, database, view, request): + """Links for the view actions menu""" + + @hookspec def query_actions(datasette, actor, database, query_name, request, sql, params): """Links for the query and canned query actions menu""" diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 1d328366..505335d6 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -24,17 +24,17 @@

{{ metadata.get("title") or table }}{% if is_view %} (view){% endif %}{% if private %} 🔒{% endif %}

-{% set links = table_actions() %}{% if links %} +{% set links = actions() %}{% if links %}
- Table actions + {% if is_view %}View{% else %}Table{% endif %} actions - Table actions + {% if is_view %}View{% else %}Table{% endif %} actions
diff --git a/datasette/views/table.py b/datasette/views/table.py index 6d0d9885..ba03241d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1401,22 +1401,28 @@ async def table_view_data( "Primary keys for this table" return pks - async def extra_table_actions(): - async def table_actions(): + async def extra_actions(): + async def actions(): links = [] - for hook in pm.hook.table_actions( - datasette=datasette, - table=table_name, - database=database_name, - actor=request.actor, - request=request, - ): + kwargs = { + "datasette": datasette, + "database": database_name, + "actor": request.actor, + "request": request, + } + if is_view: + kwargs["view"] = table_name + method = pm.hook.view_actions + else: + kwargs["table"] = table_name + method = pm.hook.table_actions + for hook in method(**kwargs): extra_links = await await_me_maybe(hook) if extra_links: links.extend(extra_links) return links - return table_actions + return actions async def extra_is_view(): return is_view @@ -1606,7 +1612,7 @@ async def table_view_data( "database", "table", "database_color", - "table_actions", + "actions", "filters", "renderers", "custom_table_templates", @@ -1647,7 +1653,7 @@ async def table_view_data( extra_database, extra_table, extra_database_color, - extra_table_actions, + extra_actions, extra_filters, extra_renderers, extra_custom_table_templates, diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4ac391ef..d08ea57a 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1521,6 +1521,28 @@ This example adds a new table action if the signed in user is ``"root"``: Example: `datasette-graphql `_ +.. _plugin_hook_view_actions: + +view_actions(datasette, actor, database, view, request) +------------------------------------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``database`` - string + The name of the database. + +``view`` - string + The name of the SQL view. + +``request`` - :ref:`internals_request` or None + The current HTTP request. This can be ``None`` if the request object is not available. + +Like :ref:`plugin_hook_table_actions` but for SQL views. + .. _plugin_hook_query_actions: query_actions(datasette, actor, database, query_name, request, sql, params) @@ -1657,7 +1679,9 @@ This example adds a link an imagined tool for editing the homepage, only for sig if actor: return [ { - "href": datasette.urls.path("/-/customize-homepage"), + "href": datasette.urls.path( + "/-/customize-homepage" + ), "label": "Customize homepage", } ] diff --git a/tests/fixtures.py b/tests/fixtures.py index 61d240da..d668a8ea 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -56,6 +56,7 @@ EXPECTED_PLUGINS = [ "skip_csrf", "startup", "table_actions", + "view_actions", ], }, { diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 94267f04..509418d4 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -391,6 +391,18 @@ def table_actions(datasette, database, table, actor): ] +@hookimpl +def view_actions(datasette, database, view, actor): + if actor: + return [ + { + "href": datasette.urls.instance(), + "label": f"Database: {database}", + }, + {"href": datasette.urls.instance(), "label": f"View: {view}"}, + ] + + @hookimpl def query_actions(datasette, database, query_name, sql): # Don't explain an explain diff --git a/tests/test_plugins.py b/tests/test_plugins.py index dcc5de20..88af3999 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -923,18 +923,34 @@ async def test_hook_menu_links(ds_client): @pytest.mark.asyncio -@pytest.mark.parametrize("table_or_view", ["facetable", "simple_view"]) -async def test_hook_table_actions(ds_client, table_or_view): - response = await ds_client.get(f"/fixtures/{table_or_view}") +async def test_hook_table_actions(ds_client): + response = await ds_client.get("/fixtures/facetable") assert get_actions_links(response.text) == [] - response_2 = await ds_client.get(f"/fixtures/{table_or_view}?_bot=1&_hello=BOB") + response_2 = await ds_client.get("/fixtures/facetable?_bot=1&_hello=BOB") assert sorted( get_actions_links(response_2.text), key=lambda link: link["label"] ) == [ {"label": "Database: fixtures", "href": "/", "description": None}, {"label": "From async BOB", "href": "/", "description": None}, - {"label": f"Table: {table_or_view}", "href": "/", "description": None}, + {"label": "Table: facetable", "href": "/", "description": None}, + ] + + +@pytest.mark.asyncio +async def test_hook_view_actions(ds_client): + response = await ds_client.get("/fixtures/simple_view") + assert get_actions_links(response.text) == [] + + response_2 = await ds_client.get( + "/fixtures/simple_view", + cookies={"ds_actor": ds_client.actor_cookie({"id": "bob"})}, + ) + assert sorted( + get_actions_links(response_2.text), key=lambda link: link["label"] + ) == [ + {"label": "Database: fixtures", "href": "/", "description": None}, + {"label": "View: simple_view", "href": "/", "description": None}, ] From 06281a0b8e3d7cdb59230ace0abb026de3919d47 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 14:32:48 -0700 Subject: [PATCH 016/334] Test for labels on Table/View action buttons, refs #2297 --- tests/test_plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 88af3999..fdfd531e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -926,8 +926,8 @@ async def test_hook_menu_links(ds_client): async def test_hook_table_actions(ds_client): response = await ds_client.get("/fixtures/facetable") assert get_actions_links(response.text) == [] - response_2 = await ds_client.get("/fixtures/facetable?_bot=1&_hello=BOB") + assert ">Table actions<" in response_2.text assert sorted( get_actions_links(response_2.text), key=lambda link: link["label"] ) == [ @@ -941,11 +941,11 @@ async def test_hook_table_actions(ds_client): async def test_hook_view_actions(ds_client): response = await ds_client.get("/fixtures/simple_view") assert get_actions_links(response.text) == [] - response_2 = await ds_client.get( "/fixtures/simple_view", cookies={"ds_actor": ds_client.actor_cookie({"id": "bob"})}, ) + assert ">View actions<" in response_2.text assert sorted( get_actions_links(response_2.text), key=lambda link: link["label"] ) == [ From 7339cc51de96a55d6d0b9022ee0511c3a9640c42 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 15:44:10 -0700 Subject: [PATCH 017/334] Rearrange plugin hooks page with more sections, closes #2300 --- docs/plugin_hooks.rst | 534 ++++++++++++++++++++++-------------------- 1 file changed, 274 insertions(+), 260 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index d08ea57a..e43aa784 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -92,10 +92,17 @@ This function can return an awaitable function if it needs to run any async code Examples: `datasette-edit-templates `_ +.. _plugin_page_extras: + +Page extras +----------- + +These plugin hooks can be used to affect the way HTML pages for different Datasette interfaces are rendered. + .. _plugin_hook_extra_template_vars: extra_template_vars(template, database, table, columns, view_name, request, datasette) --------------------------------------------------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Extra template variables that should be made available in the rendered template context. @@ -184,7 +191,7 @@ Examples: `datasette-search-all ` @@ -238,7 +245,7 @@ Examples: `datasette-cluster-map ` @@ -288,7 +295,7 @@ Examples: `datasette-cluster-map `` block at the end of the ```` element on the page. @@ -1430,262 +1437,6 @@ This example logs an error to `Sentry `__ and then renders a Example: `datasette-sentry `_ -.. _plugin_hook_menu_links: - -menu_links(datasette, actor, request) -------------------------------------- - -``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. - -``actor`` - dictionary or None - The currently authenticated :ref:`actor `. - -``request`` - :ref:`internals_request` or None - The current HTTP request. This can be ``None`` if the request object is not available. - -This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon. - -The hook should return a list of ``{"href": "...", "label": "..."}`` menu items. These will be added to the menu. - -It can alternatively return an ``async def`` awaitable function which returns a list of menu items. - -This example adds a new menu item but only if the signed in user is ``"root"``: - -.. code-block:: python - - from datasette import hookimpl - - - @hookimpl - def menu_links(datasette, actor): - if actor and actor.get("id") == "root": - return [ - { - "href": datasette.urls.path( - "/-/edit-schema" - ), - "label": "Edit schema", - }, - ] - -Using :ref:`internals_datasette_urls` here ensures that links in the menu will take the :ref:`setting_base_url` setting into account. - -Examples: `datasette-search-all `_, `datasette-graphql `_ - -.. _plugin_hook_table_actions: - -table_actions(datasette, actor, database, table, request) ---------------------------------------------------------- - -``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. - -``actor`` - dictionary or None - The currently authenticated :ref:`actor `. - -``database`` - string - The name of the database. - -``table`` - string - The name of the table. - -``request`` - :ref:`internals_request` or None - The current HTTP request. This can be ``None`` if the request object is not available. - -This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items, with optional ``"description": "..."`` keys describing each action in more detail. - -It can alternatively return an ``async def`` awaitable function which returns a list of menu items. - -This example adds a new table action if the signed in user is ``"root"``: - -.. code-block:: python - - from datasette import hookimpl - - - @hookimpl - def table_actions(datasette, actor, database, table): - if actor and actor.get("id") == "root": - return [ - { - "href": datasette.urls.path( - "/-/edit-schema/{}/{}".format( - database, table - ) - ), - "label": "Edit schema for this table", - "description": "Add, remove, rename or alter columns for this table.", - } - ] - -Example: `datasette-graphql `_ - -.. _plugin_hook_view_actions: - -view_actions(datasette, actor, database, view, request) -------------------------------------------------------- - -``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. - -``actor`` - dictionary or None - The currently authenticated :ref:`actor `. - -``database`` - string - The name of the database. - -``view`` - string - The name of the SQL view. - -``request`` - :ref:`internals_request` or None - The current HTTP request. This can be ``None`` if the request object is not available. - -Like :ref:`plugin_hook_table_actions` but for SQL views. - -.. _plugin_hook_query_actions: - -query_actions(datasette, actor, database, query_name, request, sql, params) ---------------------------------------------------------------------------- - -``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. - -``actor`` - dictionary or None - The currently authenticated :ref:`actor `. - -``database`` - string - The name of the database. - -``query_name`` - string or None - The name of the canned query, or ``None`` if this is an arbitrary SQL query. - -``request`` - :ref:`internals_request` - The current HTTP request. - -``sql`` - string - The SQL query being executed - -``params`` - dictionary - The parameters passed to the SQL query, if any. - -This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the canned query and arbitrary SQL query pages. - -This example adds a new query action linking to a page for explaining a query: - -.. code-block:: python - - from datasette import hookimpl - import urllib - - - @hookimpl - def query_actions(datasette, database, query_name, sql): - # Don't explain an explain - if sql.lower().startswith("explain"): - return - return [ - { - "href": datasette.urls.database(database) - + "?" - + urllib.parse.urlencode( - { - "sql": "explain " + sql, - } - ), - "label": "Explain this query", - "description": "Get a summary of how SQLite executes the query", - }, - ] - -Example: `datasette-create-view `_ - -.. _plugin_hook_database_actions: - -database_actions(datasette, actor, database, request) ------------------------------------------------------ - -``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. - -``actor`` - dictionary or None - The currently authenticated :ref:`actor `. - -``database`` - string - The name of the database. - -``request`` - :ref:`internals_request` - The current HTTP request. - -This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. - -This example adds a new database action for creating a table, if the user has the ``edit-schema`` permission: - -.. code-block:: python - - from datasette import hookimpl - - - @hookimpl - def database_actions(datasette, actor, database): - async def inner(): - if not await datasette.permission_allowed( - actor, - "edit-schema", - resource=database, - default=False, - ): - return [] - return [ - { - "href": datasette.urls.path( - "/-/edit-schema/{}/-/create".format( - database - ) - ), - "label": "Create a table", - } - ] - - return inner - -Example: `datasette-graphql `_, `datasette-edit-schema `_ - -.. _plugin_hook_homepage_actions: - -homepage_actions(datasette, actor, request) -------------------------------------------- - -``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. - -``actor`` - dictionary or None - The currently authenticated :ref:`actor `. - -``request`` - :ref:`internals_request` - The current HTTP request. - -This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the index page of the Datasette instance. - -This example adds a link an imagined tool for editing the homepage, only for signed in users: - -.. code-block:: python - - from datasette import hookimpl - - - @hookimpl - def homepage_actions(datasette, actor): - if actor: - return [ - { - "href": datasette.urls.path( - "/-/customize-homepage" - ), - "label": "Customize homepage", - } - ] - .. _plugin_hook_skip_csrf: skip_csrf(datasette, scope) @@ -1756,6 +1507,269 @@ This hook is responsible for returning a dictionary corresponding to Datasette : Example: `datasette-remote-metadata plugin `__ +.. _plugin_hook_menu_links: + +menu_links(datasette, actor, request) +------------------------------------- + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``request`` - :ref:`internals_request` or None + The current HTTP request. This can be ``None`` if the request object is not available. + +This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon. + +The hook should return a list of ``{"href": "...", "label": "..."}`` menu items. These will be added to the menu. + +It can alternatively return an ``async def`` awaitable function which returns a list of menu items. + +This example adds a new menu item but only if the signed in user is ``"root"``: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def menu_links(datasette, actor): + if actor and actor.get("id") == "root": + return [ + { + "href": datasette.urls.path( + "/-/edit-schema" + ), + "label": "Edit schema", + }, + ] + +Using :ref:`internals_datasette_urls` here ensures that links in the menu will take the :ref:`setting_base_url` setting into account. + +Examples: `datasette-search-all `_, `datasette-graphql `_ + +.. _plugin_actions: + +Action hooks +------------ + +Action hooks can be used to add items to the action menus that appear at the top of different pages within Datasette. Unlike :ref:`menu_links() `, actions which are displayed on every page, actions should only be relevant to the page the user is currently viewing. + +.. _plugin_hook_table_actions: + +table_actions(datasette, actor, database, table, request) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``database`` - string + The name of the database. + +``table`` - string + The name of the table. + +``request`` - :ref:`internals_request` or None + The current HTTP request. This can be ``None`` if the request object is not available. + +This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items, with optional ``"description": "..."`` keys describing each action in more detail. + +It can alternatively return an ``async def`` awaitable function which returns a list of menu items. + +This example adds a new table action if the signed in user is ``"root"``: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def table_actions(datasette, actor, database, table): + if actor and actor.get("id") == "root": + return [ + { + "href": datasette.urls.path( + "/-/edit-schema/{}/{}".format( + database, table + ) + ), + "label": "Edit schema for this table", + "description": "Add, remove, rename or alter columns for this table.", + } + ] + +Example: `datasette-graphql `_ + +.. _plugin_hook_view_actions: + +view_actions(datasette, actor, database, view, request) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``database`` - string + The name of the database. + +``view`` - string + The name of the SQL view. + +``request`` - :ref:`internals_request` or None + The current HTTP request. This can be ``None`` if the request object is not available. + +Like :ref:`plugin_hook_table_actions` but for SQL views. + +.. _plugin_hook_query_actions: + +query_actions(datasette, actor, database, query_name, request, sql, params) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``database`` - string + The name of the database. + +``query_name`` - string or None + The name of the canned query, or ``None`` if this is an arbitrary SQL query. + +``request`` - :ref:`internals_request` + The current HTTP request. + +``sql`` - string + The SQL query being executed + +``params`` - dictionary + The parameters passed to the SQL query, if any. + +This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the canned query and arbitrary SQL query pages. + +This example adds a new query action linking to a page for explaining a query: + +.. code-block:: python + + from datasette import hookimpl + import urllib + + + @hookimpl + def query_actions(datasette, database, query_name, sql): + # Don't explain an explain + if sql.lower().startswith("explain"): + return + return [ + { + "href": datasette.urls.database(database) + + "?" + + urllib.parse.urlencode( + { + "sql": "explain " + sql, + } + ), + "label": "Explain this query", + "description": "Get a summary of how SQLite executes the query", + }, + ] + +Example: `datasette-create-view `_ + +.. _plugin_hook_database_actions: + +database_actions(datasette, actor, database, request) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``database`` - string + The name of the database. + +``request`` - :ref:`internals_request` + The current HTTP request. + +This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. + +This example adds a new database action for creating a table, if the user has the ``edit-schema`` permission: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def database_actions(datasette, actor, database): + async def inner(): + if not await datasette.permission_allowed( + actor, + "edit-schema", + resource=database, + default=False, + ): + return [] + return [ + { + "href": datasette.urls.path( + "/-/edit-schema/{}/-/create".format( + database + ) + ), + "label": "Create a table", + } + ] + + return inner + +Example: `datasette-graphql `_, `datasette-edit-schema `_ + +.. _plugin_hook_homepage_actions: + +homepage_actions(datasette, actor, request) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``request`` - :ref:`internals_request` + The current HTTP request. + +This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the index page of the Datasette instance. + +This example adds a link an imagined tool for editing the homepage, only for signed in users: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def homepage_actions(datasette, actor): + if actor: + return [ + { + "href": datasette.urls.path( + "/-/customize-homepage" + ), + "label": "Customize homepage", + } + ] + .. _plugin_hook_slots: Template slots From b8711988b903cafaeb0d8b07a7f98d75084f1c2b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 16:13:31 -0700 Subject: [PATCH 018/334] row_actions() plugin hook, closes #2299 --- datasette/hookspecs.py | 5 ++++ datasette/templates/row.html | 31 ++++++++++++++++++++ datasette/views/row.py | 17 +++++++++++ docs/plugin_hooks.rst | 57 +++++++++++++++++++++++++++++++----- tests/fixtures.py | 1 + tests/plugins/my_plugin.py | 12 ++++++++ tests/test_plugins.py | 18 ++++++++++++ 7 files changed, 134 insertions(+), 7 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 6ce1e85e..35468cc3 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -140,6 +140,11 @@ def menu_links(datasette, actor, request): """Links for the navigation menu""" +@hookspec +def row_actions(datasette, actor, request, database, table, row): + """Links for the row actions menu""" + + @hookspec def table_actions(datasette, actor, database, table, request): """Links for the table actions menu""" diff --git a/datasette/templates/row.html b/datasette/templates/row.html index 6d4b996e..c3e6bed7 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -22,6 +22,37 @@ {% block content %}

{{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}

+{% set links = row_actions %}{% if links %} +
+
+ +
+ + Row actions + + + + Row actions +
+ +
+
+ {% if links %} + + {% endif %} +
+
+
+{% endif %} + {{ top_row() }} {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/views/row.py b/datasette/views/row.py index 4d20e41a..49d390f6 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -3,10 +3,12 @@ from datasette.database import QueryInterrupted from datasette.events import UpdateRowEvent, DeleteRowEvent from .base import DataView, BaseView, _error from datasette.utils import ( + await_me_maybe, make_slot_function, to_css_class, escape_sqlite, ) +from datasette.plugins import pm import json import sqlite_utils from .table import display_columns_and_rows @@ -55,6 +57,20 @@ class RowView(DataView): ) for column in display_columns: column["sortable"] = False + + row_actions = [] + for hook in pm.hook.row_actions( + datasette=self.ds, + actor=request.actor, + request=request, + database=database, + table=table, + row=rows[0], + ): + extra_links = await await_me_maybe(hook) + if extra_links: + row_actions.extend(extra_links) + return { "private": private, "foreign_key_tables": await self.foreign_key_tables( @@ -68,6 +84,7 @@ class RowView(DataView): f"_table-row-{to_css_class(database)}-{to_css_class(table)}.html", "_table.html", ], + "row_actions": row_actions, "metadata": (self.ds.metadata("databases") or {}) .get(database, {}) .get("tables", {}) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index e43aa784..90ed8924 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1557,6 +1557,10 @@ Action hooks Action hooks can be used to add items to the action menus that appear at the top of different pages within Datasette. Unlike :ref:`menu_links() `, actions which are displayed on every page, actions should only be relevant to the page the user is currently viewing. +Each of these hooks should return return a list of ``{"href": "...", "label": "..."}`` menu items, with optional ``"description": "..."`` keys describing each action in more detail. + +They can alternatively return an ``async def`` awaitable function which, when called, returns a list of those menu items. + .. _plugin_hook_table_actions: table_actions(datasette, actor, database, table, request) @@ -1577,10 +1581,6 @@ table_actions(datasette, actor, database, table, request) ``request`` - :ref:`internals_request` or None The current HTTP request. This can be ``None`` if the request object is not available. -This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items, with optional ``"description": "..."`` keys describing each action in more detail. - -It can alternatively return an ``async def`` awaitable function which returns a list of menu items. - This example adds a new table action if the signed in user is ``"root"``: .. code-block:: python @@ -1653,7 +1653,7 @@ query_actions(datasette, actor, database, query_name, request, sql, params) ``params`` - dictionary The parameters passed to the SQL query, if any. -This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the canned query and arbitrary SQL query pages. +Populates a "Query actions" menu on the canned query and arbitrary SQL query pages. This example adds a new query action linking to a page for explaining a query: @@ -1684,6 +1684,49 @@ This example adds a new query action linking to a page for explaining a query: Example: `datasette-create-view `_ +.. _plugin_hook_row_actions: + +row_actions(datasette, actor, request, database, table, row) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor `. + +``request`` - :ref:`internals_request` or None + The current HTTP request. + +``database`` - string + The name of the database. + +``table`` - string + The name of the table. + +``row`` - ``sqlite.Row`` + The SQLite row object being dispayed on the page. + +Return links for the "Row actions" menu shown at the top of the row page. + +This example displays the row in JSON plus some additional debug information if the user is signed in: + +.. code-block:: python + + from datasette import hookimpl + + + @hookimpl + def row_actions(datasette, database, table, actor, row): + if actor: + return [ + { + "href": datasette.urls.instance(), + "label": f"Row details for {actor['id']}", + "description": json.dumps(dict(row), default=repr), + }, + ] + .. _plugin_hook_database_actions: database_actions(datasette, actor, database, request) @@ -1701,7 +1744,7 @@ database_actions(datasette, actor, database, request) ``request`` - :ref:`internals_request` The current HTTP request. -This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the database page. +Populates an actions menu on the database page. This example adds a new database action for creating a table, if the user has the ``edit-schema`` permission: @@ -1749,7 +1792,7 @@ homepage_actions(datasette, actor, request) ``request`` - :ref:`internals_request` The current HTTP request. -This hook is similar to :ref:`plugin_hook_table_actions` but populates an actions menu on the index page of the Datasette instance. +Populates an actions menu on the top-level index homepage of the Datasette instance. This example adds a link an imagined tool for editing the homepage, only for signed in users: diff --git a/tests/fixtures.py b/tests/fixtures.py index d668a8ea..af6b610b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -53,6 +53,7 @@ EXPECTED_PLUGINS = [ "register_permissions", "register_routes", "render_cell", + "row_actions", "skip_csrf", "startup", "table_actions", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 509418d4..9ef10181 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -423,6 +423,18 @@ def query_actions(datasette, database, query_name, sql): ] +@hookimpl +def row_actions(datasette, database, table, actor, row): + if actor: + return [ + { + "href": datasette.urls.instance(), + "label": f"Row details for {actor['id']}", + "description": json.dumps(dict(row), default=repr), + }, + ] + + @hookimpl def database_actions(datasette, database, actor, request): if actor: diff --git a/tests/test_plugins.py b/tests/test_plugins.py index fdfd531e..9c8d02ec 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1000,6 +1000,24 @@ async def test_hook_query_actions(ds_client, path, expected_url): ] +@pytest.mark.asyncio +async def test_hook_row_actions(ds_client): + response = await ds_client.get("/fixtures/facet_cities/1") + assert get_actions_links(response.text) == [] + + response_2 = await ds_client.get( + "/fixtures/facet_cities/1", + cookies={"ds_actor": ds_client.actor_cookie({"id": "sam"})}, + ) + assert get_actions_links(response_2.text) == [ + { + "label": "Row details for sam", + "href": "/", + "description": '{"id": 1, "name": "San Francisco"}', + } + ] + + @pytest.mark.asyncio async def test_hook_database_actions(ds_client): response = await ds_client.get("/fixtures") From 8d456aae45914d8ee135cf5cc139948e9092e633 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 16:17:53 -0700 Subject: [PATCH 019/334] Fix spelling of displayed, refs #2299 --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 90ed8924..111677d8 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1705,7 +1705,7 @@ row_actions(datasette, actor, request, database, table, row) The name of the table. ``row`` - ``sqlite.Row`` - The SQLite row object being dispayed on the page. + The SQLite row object being displayed on the page. Return links for the "Row actions" menu shown at the top of the row page. From 828ef9899f9fbe66de6a480705430cd8696f75de Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 16:25:25 -0700 Subject: [PATCH 020/334] Ran blacken-docs, refs #2299 --- docs/plugin_hooks.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 111677d8..5e4b5077 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1723,7 +1723,9 @@ This example displays the row in JSON plus some additional debug information if { "href": datasette.urls.instance(), "label": f"Row details for {actor['id']}", - "description": json.dumps(dict(row), default=repr), + "description": json.dumps( + dict(row), default=repr + ), }, ] From e088abdb463836b4165782d20cdfbd3cd2fef65e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 16:34:00 -0700 Subject: [PATCH 021/334] Refactored action menus to a shared include, closes #2301 --- datasette/templates/_action_menu.html | 28 +++++++++++++++++++++++ datasette/templates/database.html | 32 ++------------------------ datasette/templates/index.html | 32 ++------------------------ datasette/templates/query.html | 33 ++------------------------- datasette/templates/row.html | 32 ++------------------------ datasette/templates/table.html | 32 ++------------------------ 6 files changed, 38 insertions(+), 151 deletions(-) create mode 100644 datasette/templates/_action_menu.html diff --git a/datasette/templates/_action_menu.html b/datasette/templates/_action_menu.html new file mode 100644 index 00000000..7d1d4a55 --- /dev/null +++ b/datasette/templates/_action_menu.html @@ -0,0 +1,28 @@ +{% if action_links %} +
+
+ +
+ + {{ action_title }} + + + + {{ action_title }} +
+ +
+
+ +
+
+
+{% endif %} \ No newline at end of file diff --git a/datasette/templates/database.html b/datasette/templates/database.html index b5a0edc4..4a4a05cb 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -13,36 +13,8 @@

{{ metadata.title or database }}{% if private %} 🔒{% endif %}

-{% set links = database_actions() %}{% if links %} -
-
- -
- - Database actions - - - - Database actions -
- -
-
- {% if links %} - - {% endif %} -
-
-
-{% endif %} +{% set action_links, action_title = database_actions(), "Database actions" %} +{% include "_action_menu.html" %} {{ top_database() }} diff --git a/datasette/templates/index.html b/datasette/templates/index.html index d08cdb10..6e95126d 100644 --- a/datasette/templates/index.html +++ b/datasette/templates/index.html @@ -7,36 +7,8 @@ {% block content %}

{{ metadata.title or "Datasette" }}{% if private %} 🔒{% endif %}

-{% set links = homepage_actions %}{% if links %} -
-
- -
- - Homepage actions - - - - Homepage actions -
- -
-
- {% if links %} - - {% endif %} -
-
-
-{% endif %} +{% set action_links, action_title = homepage_actions, "Homepage actions" %} +{% include "_action_menu.html" %} {{ top_homepage() }} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 09a29118..f7c8d0a3 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -29,37 +29,8 @@ {% endif %}

{{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

-{% set links = query_actions() %}{% if links %} -
-
- -
- - Query actions - - - - Query actions -
- -
-
- {% if links %} - - {% endif %} -
-
-
-{% endif %} - +{% set action_links, action_title = query_actions(), "Query actions" %} +{% include "_action_menu.html" %} {% if canned_query %}{{ top_canned_query() }}{% else %}{{ top_query() }}{% endif %} diff --git a/datasette/templates/row.html b/datasette/templates/row.html index c3e6bed7..db43e71a 100644 --- a/datasette/templates/row.html +++ b/datasette/templates/row.html @@ -22,36 +22,8 @@ {% block content %}

{{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}

-{% set links = row_actions %}{% if links %} -
-
- -
- - Row actions - - - - Row actions -
- -
-
- {% if links %} - - {% endif %} -
-
-
-{% endif %} +{% set action_links, action_title = row_actions, "Row actions" %} +{% include "_action_menu.html" %} {{ top_row() }} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 505335d6..35e0b9c1 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -24,36 +24,8 @@

{{ metadata.get("title") or table }}{% if is_view %} (view){% endif %}{% if private %} 🔒{% endif %}

-{% set links = actions() %}{% if links %} -
-
- -
- - {% if is_view %}View{% else %}Table{% endif %} actions - - - - {% if is_view %}View{% else %}Table{% endif %} actions -
- -
-
- {% if links %} - - {% endif %} -
-
-
-{% endif %} +{% set action_links, action_title = actions(), "View actions" if is_view else "Table actions" %} +{% include "_action_menu.html" %} {{ top_table() }} From 9cc6f1908f844f683cc946e52671da60b15a1bba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 16:54:03 -0700 Subject: [PATCH 022/334] Gradient on header and footer, closes #2302 --- datasette/static/app.css | 10 ++++++++-- datasette/templates/base.html | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index e4a0ee10..632204fb 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -269,6 +269,7 @@ header, footer { padding: 0.6rem 1rem 0.5rem 1rem; background-color: #276890; + background: linear-gradient(180deg, rgba(96,144,173,1) 0%, rgba(39,104,144,1) 50%); color: rgba(255,255,244,0.9); overflow: hidden; box-sizing: border-box; @@ -346,12 +347,17 @@ details.nav-menu > summary::-webkit-details-marker { } details .nav-menu-inner { position: absolute; - top: 2rem; + top: 2.6rem; right: 10px; width: 180px; background-color: #276890; - padding: 1rem; z-index: 1000; + padding: 0; +} +.nav-menu-inner li, +form.nav-menu-logout { + padding: 0.3rem 0.5rem; + border-top: 1px solid #ffffff69; } .nav-menu-inner a { display: block; diff --git a/datasette/templates/base.html b/datasette/templates/base.html index ceead217..afb8dcd3 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -37,7 +37,7 @@ {% endif %} {% if show_logout %} -
+
{% endif %} From feddd61789983dc75f127956176cac493d877be8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 17:01:51 -0700 Subject: [PATCH 023/334] Fix tests I broke in #2302 --- datasette/templates/patterns.html | 2 +- tests/test_auth.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index 33db3d1a..cb0daf9a 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -26,7 +26,7 @@
  • Installed plugins
  • Version info
    • -
    +
    diff --git a/tests/test_auth.py b/tests/test_auth.py index f2359df7..74e8283a 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -110,7 +110,7 @@ async def test_logout_button_in_navigation(ds_client, path): anon_response = await ds_client.get(path) for fragment in ( "test", - '
    ', + '', ): assert fragment in response.text assert fragment not in anon_response.text @@ -121,7 +121,10 @@ async def test_logout_button_in_navigation(ds_client, path): async def test_no_logout_button_in_navigation_if_no_ds_actor_cookie(ds_client, path): response = await ds_client.get(path + "?_bot=1") assert "bot" in response.text - assert '' not in response.text + assert ( + '' + not in response.text + ) @pytest.mark.parametrize( From c92f326ed10b7eea8b9c4d3cd5caef9e7aceee1b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 19:10:53 -0700 Subject: [PATCH 024/334] Release 1.013a #2104, #2286, #2293, #2297, #2298, #2299, #2300, #2301, #2302 --- datasette/version.py | 2 +- docs/changelog.rst | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index b2cd10bf..38a52c01 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a12" +__version__ = "1.0a13" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 240df141..1d31e9e6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,21 @@ Changelog ========= +.. _v1_0_a13: + +1.0a13 (2024-03-12) +------------------- + +Each of the key concepts in Datasette now has an :ref:`actions menu `, which plugins can use to add additional functionality targeting that entity. + +- Plugin hook: :ref:`view_actions() ` for actions that can be applied to a SQL view. (:issue:`2297`) +- Plugin hook: :ref:`homepage_actions() ` for actions that apply to the instance homepage. (:issue:`2298`) +- Plugin hook: :ref:`row_actions() ` for actions that apply to the row page. (:issue:`2299`) +- :ref:`Plugin hooks ` documentation page is now organized with additional headings. (:issue:`2300`) +- Improved the display of action buttons on pages that also display metadata. (:issue:`2286`) +- The header and footer of the page now uses a subtle gradient effect, and options in the navigation menu are better visually defined. (:issue:`2302`) +- Table names that start with an underscore now default to hidden. (:issue:`2104`) + .. _v1_0_a12: 1.0a12 (2024-02-29) From 8b6f155b45b360a1c649efb24f985d926bb76991 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 Mar 2024 19:19:51 -0700 Subject: [PATCH 025/334] Added two things I left out of the 1.0a13 release notes Refs #2104, #2294 Closes #2303 --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1d31e9e6..ebed499f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,10 +14,12 @@ Each of the key concepts in Datasette now has an :ref:`actions menu ` for actions that can be applied to a SQL view. (:issue:`2297`) - Plugin hook: :ref:`homepage_actions() ` for actions that apply to the instance homepage. (:issue:`2298`) - Plugin hook: :ref:`row_actions() ` for actions that apply to the row page. (:issue:`2299`) +- Action menu items for all of the ``*_actions()`` plugin hooks can now return an optional ``"description"`` key, which will be displayed in the menu below the action label. (:issue:`2294`) - :ref:`Plugin hooks ` documentation page is now organized with additional headings. (:issue:`2300`) - Improved the display of action buttons on pages that also display metadata. (:issue:`2286`) - The header and footer of the page now uses a subtle gradient effect, and options in the navigation menu are better visually defined. (:issue:`2302`) - Table names that start with an underscore now default to hidden. (:issue:`2104`) +- ``pragma_table_list`` has been added to the allow-list of SQLite pragma functions supported by Datasette. ``select * from pragma_table_list()`` is no longer blocked. (`#2104 `__) .. _v1_0_a12: From 5af68377256a4aafebef7da98600f0bfd1546644 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Mar 2024 15:15:31 -0700 Subject: [PATCH 026/334] Fix httpx warning about app=self.app, refs #2307 --- datasette/app.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8591af6a..c0b14fb2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1934,36 +1934,52 @@ class DatasetteClient: return path async def get(self, path, **kwargs): - async with httpx.AsyncClient(app=self.app) as client: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app) + ) as client: return await client.get(self._fix(path), **kwargs) async def options(self, path, **kwargs): - async with httpx.AsyncClient(app=self.app) as client: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app) + ) as client: return await client.options(self._fix(path), **kwargs) async def head(self, path, **kwargs): - async with httpx.AsyncClient(app=self.app) as client: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app) + ) as client: return await client.head(self._fix(path), **kwargs) async def post(self, path, **kwargs): - async with httpx.AsyncClient(app=self.app) as client: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app) + ) as client: return await client.post(self._fix(path), **kwargs) async def put(self, path, **kwargs): - async with httpx.AsyncClient(app=self.app) as client: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app) + ) as client: return await client.put(self._fix(path), **kwargs) async def patch(self, path, **kwargs): - async with httpx.AsyncClient(app=self.app) as client: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app) + ) as client: return await client.patch(self._fix(path), **kwargs) async def delete(self, path, **kwargs): - async with httpx.AsyncClient(app=self.app) as client: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app) + ) as client: return await client.delete(self._fix(path), **kwargs) async def request(self, method, path, **kwargs): avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) - async with httpx.AsyncClient(app=self.app) as client: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.app) + ) as client: return await client.request( method, self._fix(path, avoid_path_rewrites), **kwargs ) From 54f5604cafb4a08ea9e94e12de69e5adae51c42b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Mar 2024 15:19:18 -0700 Subject: [PATCH 027/334] Fixed cookies= httpx warning, refs #2307 --- datasette/app.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index c0b14fb2..f1b057af 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1935,50 +1935,58 @@ class DatasetteClient: async def get(self, path, **kwargs): async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app) + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), ) as client: return await client.get(self._fix(path), **kwargs) async def options(self, path, **kwargs): async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app) + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), ) as client: return await client.options(self._fix(path), **kwargs) async def head(self, path, **kwargs): async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app) + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), ) as client: return await client.head(self._fix(path), **kwargs) async def post(self, path, **kwargs): async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app) + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), ) as client: return await client.post(self._fix(path), **kwargs) async def put(self, path, **kwargs): async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app) + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), ) as client: return await client.put(self._fix(path), **kwargs) async def patch(self, path, **kwargs): async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app) + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), ) as client: return await client.patch(self._fix(path), **kwargs) async def delete(self, path, **kwargs): async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app) + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), ) as client: return await client.delete(self._fix(path), **kwargs) async def request(self, method, path, **kwargs): avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app) + transport=httpx.ASGITransport(app=self.app), + cookies=kwargs.pop("cookies", None), ) as client: return await client.request( method, self._fix(path, avoid_path_rewrites), **kwargs From eb8545c172b4b12e89dcc08a976f037d881346c4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Mar 2024 15:29:03 -0700 Subject: [PATCH 028/334] Refactor duplicate code in DatasetteClient, closes #2307 --- datasette/app.py | 43 +++++++++++-------------------------------- 1 file changed, 11 insertions(+), 32 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f1b057af..aa75d772 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1933,54 +1933,33 @@ class DatasetteClient: path = f"http://localhost{path}" return path - async def get(self, path, **kwargs): + async def _request(self, method, path, **kwargs): async with httpx.AsyncClient( transport=httpx.ASGITransport(app=self.app), cookies=kwargs.pop("cookies", None), ) as client: - return await client.get(self._fix(path), **kwargs) + return await getattr(client, method)(self._fix(path), **kwargs) + + async def get(self, path, **kwargs): + return await self._request("get", path, **kwargs) async def options(self, path, **kwargs): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await client.options(self._fix(path), **kwargs) + return await self._request("options", path, **kwargs) async def head(self, path, **kwargs): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await client.head(self._fix(path), **kwargs) + return await self._request("head", path, **kwargs) async def post(self, path, **kwargs): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await client.post(self._fix(path), **kwargs) + return await self._request("post", path, **kwargs) async def put(self, path, **kwargs): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await client.put(self._fix(path), **kwargs) + return await self._request("put", path, **kwargs) async def patch(self, path, **kwargs): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await client.patch(self._fix(path), **kwargs) + return await self._request("patch", path, **kwargs) async def delete(self, path, **kwargs): - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=self.app), - cookies=kwargs.pop("cookies", None), - ) as client: - return await client.delete(self._fix(path), **kwargs) + return await self._request("delete", path, **kwargs) async def request(self, method, path, **kwargs): avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) From 261fc8d875c43d5421a6ee1096992fd636c2c278 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 15 Mar 2024 15:32:12 -0700 Subject: [PATCH 029/334] Fix datetime.utcnow deprecation warning --- datasette/app.py | 2 +- datasette/default_magic_parameters.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index aa75d772..23d21600 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -930,7 +930,7 @@ class Datasette: used_default = True self._permission_checks.append( { - "when": datetime.datetime.utcnow().isoformat(), + "when": datetime.datetime.now(datetime.timezone.utc).isoformat(), "actor": actor, "action": action, "resource": resource, diff --git a/datasette/default_magic_parameters.py b/datasette/default_magic_parameters.py index 19382207..91c1c5aa 100644 --- a/datasette/default_magic_parameters.py +++ b/datasette/default_magic_parameters.py @@ -24,9 +24,12 @@ def now(key, request): if key == "epoch": return int(time.time()) elif key == "date_utc": - return datetime.datetime.utcnow().date().isoformat() + return datetime.datetime.now(datetime.timezone.utc).date().isoformat() elif key == "datetime_utc": - return datetime.datetime.utcnow().strftime(r"%Y-%m-%dT%H:%M:%S") + "Z" + return ( + datetime.datetime.now(datetime.timezone.utc).strftime(r"%Y-%m-%dT%H:%M:%S") + + "Z" + ) else: raise KeyError From 67e66f36c1a0e9534d3bc3ea7f5469e886f48e4d Mon Sep 17 00:00:00 2001 From: Agustin Bacigalup Date: Sun, 17 Mar 2024 16:18:40 -0300 Subject: [PATCH 030/334] Add ETag header for static responses (#2306) * add etag to static responses * fix RuntimeError related to static headers * Remove unnecessary import --------- Co-authored-by: Simon Willison --- datasette/utils/__init__.py | 22 ++++++++++++++++++++++ datasette/utils/asgi.py | 19 +++++++++++++++++-- tests/test_html.py | 4 ++++ tests/test_utils.py | 12 ++++++++++++ 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 9c0bbfa3..e1108911 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1,5 +1,6 @@ import asyncio from contextlib import contextmanager +import aiofiles import click from collections import OrderedDict, namedtuple, Counter import copy @@ -1418,3 +1419,24 @@ def md5_not_usedforsecurity(s): except TypeError: # For Python 3.8 which does not support usedforsecurity=False return hashlib.md5(s.encode("utf8")).hexdigest() + + +_etag_cache = {} + + +async def calculate_etag(filepath, chunk_size=4096): + if filepath in _etag_cache: + return _etag_cache[filepath] + + hasher = hashlib.md5() + async with aiofiles.open(filepath, "rb") as f: + while True: + chunk = await f.read(chunk_size) + if not chunk: + break + hasher.update(chunk) + + etag = f'"{hasher.hexdigest()}"' + _etag_cache[filepath] = etag + + return etag diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index b2c6f3ab..2fad1d42 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -1,5 +1,6 @@ +import hashlib import json -from datasette.utils import MultiParams +from datasette.utils import MultiParams, calculate_etag from mimetypes import guess_type from urllib.parse import parse_qs, urlunparse, parse_qsl from pathlib import Path @@ -285,6 +286,7 @@ async def asgi_send_file( headers = headers or {} if filename: headers["content-disposition"] = f'attachment; filename="{filename}"' + first = True headers["content-length"] = str((await aiofiles.os.stat(str(filepath))).st_size) async with aiofiles.open(str(filepath), mode="rb") as fp: @@ -307,9 +309,14 @@ async def asgi_send_file( def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None): root_path = Path(root_path) + static_headers = {} + + if headers: + static_headers = headers.copy() async def inner_static(request, send): path = request.scope["url_route"]["kwargs"]["path"] + headers = static_headers.copy() try: full_path = (root_path / path).resolve().absolute() except FileNotFoundError: @@ -325,7 +332,15 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None): await asgi_send_html(send, "404: Path not inside root path", 404) return try: - await asgi_send_file(send, full_path, chunk_size=chunk_size) + # Calculate ETag for filepath + etag = await calculate_etag(full_path, chunk_size=chunk_size) + headers["ETag"] = etag + if_none_match = request.headers.get("if-none-match") + if if_none_match and if_none_match == etag: + return await asgi_send(send, "", 304) + await asgi_send_file( + send, full_path, chunk_size=chunk_size, headers=headers + ) except FileNotFoundError: await asgi_send_html(send, "404: File not found", 404) return diff --git a/tests/test_html.py b/tests/test_html.py index 8229b166..42b290c8 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -78,6 +78,10 @@ async def test_static(ds_client): response = await ds_client.get("/-/static/app.css") assert response.status_code == 200 assert "text/css" == response.headers["content-type"] + assert "etag" in response.headers + etag = response.headers.get("etag") + response = await ds_client.get("/-/static/app.css", headers={"if-none-match": etag}) + assert response.status_code == 304 def test_static_mounts(): diff --git a/tests/test_utils.py b/tests/test_utils.py index 51577615..254b1300 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -706,3 +706,15 @@ def test_truncate_url(url, length, expected): def test_pairs_to_nested_config(pairs, expected): actual = utils.pairs_to_nested_config(pairs) assert actual == expected + + +@pytest.mark.asyncio +async def test_calculate_etag(tmp_path): + path = tmp_path / "test.txt" + path.write_text("hello") + etag = '"5d41402abc4b2a76b9719d911017c592"' + assert etag == await utils.calculate_etag(path) + assert utils._etag_cache[path] == etag + utils._etag_cache[path] = "hash" + assert "hash" == await utils.calculate_etag(path) + utils._etag_cache.clear() From da686627677b6819264bd787ba7d520edb069d12 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 Mar 2024 14:40:47 -0700 Subject: [PATCH 031/334] datasette-enrichments is example of row_actions Refs: - https://github.com/simonw/datasette/issues/2299 - https://github.com/datasette/datasette-enrichments/issues/41 --- docs/plugin_hooks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 5e4b5077..ada2fdbd 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1729,6 +1729,8 @@ This example displays the row in JSON plus some additional debug information if }, ] +Example: `datasette-enrichments `_ + .. _plugin_hook_database_actions: database_actions(datasette, actor, database, request) From 1edb24f12428bf130c9eb4c7ddc04b3d07c208cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 19 Mar 2024 09:15:39 -0700 Subject: [PATCH 032/334] Docs for 100 max rows in an insert, closes #2310 --- docs/json_api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/json_api.rst b/docs/json_api.rst index 4b39a048..5a28f042 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -568,6 +568,8 @@ To insert multiple rows at a time, use the same API method but send a list of di If successful, this will return a ``201`` status code and a ``{"ok": true}`` response body. +The maximum number rows that can be submitted at once defaults to 100, but this can be changed using the :ref:`setting_max_insert_rows` setting. + To return the newly inserted rows, add the ``"return": true`` key to the request body: .. code-block:: json From 19b6a3733664403d944925a6887b38ff313a5570 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 21 Mar 2024 10:15:57 -0700 Subject: [PATCH 033/334] z-index: 10000 on dropdown menu, closes #2311 --- datasette/static/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 632204fb..562d6adb 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -375,6 +375,7 @@ form.nav-menu-logout { position: absolute; top: calc(100% + 10px); left: 0; + z-index: 10000; } .page-action-menu .icon-text { display: inline-flex; From d32176c5b8283fbcc5c8a1f8e6d39339d73013c4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Apr 2024 16:50:09 -0700 Subject: [PATCH 034/334] Typo fix triggera -> triggers --- docs/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index e9f8b391..5cdf49b4 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1222,7 +1222,7 @@ Plugins can access this database by calling ``internal_db = datasette.get_intern Plugin authors are asked to practice good etiquette when using the internal database, as all plugins use the same database to store data. For example: -1. Use a unique prefix when creating tables, indices, and triggera in the internal database. If your plugin is called ``datasette-xyz``, then prefix names with ``datasette_xyz_*``. +1. Use a unique prefix when creating tables, indices, and triggers in the internal database. If your plugin is called ``datasette-xyz``, then prefix names with ``datasette_xyz_*``. 2. Avoid long-running write statements that may stall or block other plugins that are trying to write at the same time. 3. Use temporary tables or shared in-memory attached databases when possible. 4. Avoid implementing features that could expose private data stored in the internal database by other plugins. From 63714cb2b7b15dfa8e10bf017efda03b9b25558d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Apr 2024 17:04:55 -0700 Subject: [PATCH 035/334] Fixed some typos spotted by Gemini Pro 1.5, closes #2318 --- docs/authentication.rst | 2 +- docs/internals.rst | 2 +- docs/plugin_hooks.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index a8dc5637..5452b9c3 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -312,7 +312,7 @@ If you want to provide access to any actor with a value for a specific key, use } .. [[[end]]] -You can specify that only unauthenticated actors (from anynomous HTTP requests) should be allowed access using the special ``"unauthenticated": true`` key in an allow block (`allow demo `__, `deny demo `__): +You can specify that only unauthenticated actors (from anonymous HTTP requests) should be allowed access using the special ``"unauthenticated": true`` key in an allow block (`allow demo `__, `deny demo `__): .. [[[cog from metadata_doc import config_example diff --git a/docs/internals.rst b/docs/internals.rst index 5cdf49b4..b57a289a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1234,7 +1234,7 @@ The datasette.utils module The ``datasette.utils`` module contains various utility functions used by Datasette. As a general rule you should consider anything in this module to be unstable - functions and classes here could change without warning or be removed entirely between Datasette releases, without being mentioned in the release notes. -The exception to this rule is anythang that is documented here. If you find a need for an undocumented utility function in your own work, consider `opening an issue `__ requesting that the function you are using be upgraded to documented and supported status. +The exception to this rule is anything that is documented here. If you find a need for an undocumented utility function in your own work, consider `opening an issue `__ requesting that the function you are using be upgraded to documented and supported status. .. _internals_utils_parse_metadata: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index ada2fdbd..78489d1e 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -494,7 +494,7 @@ This will register ``render_demo`` to be called when paths with the extension `` ``render_demo`` is a Python function. It can be a regular function or an ``async def render_demo()`` awaitable function, depending on if it needs to make any asynchronous calls. -``can_render_demo`` is a Python function (or ``async def`` function) which accepts the same arguments as ``render_demo`` but just returns ``True`` or ``False``. It lets Datasette know if the current SQL query can be represented by the plugin - and hence influnce if a link to this output format is displayed in the user interface. If you omit the ``"can_render"`` key from the dictionary every query will be treated as being supported by the plugin. +``can_render_demo`` is a Python function (or ``async def`` function) which accepts the same arguments as ``render_demo`` but just returns ``True`` or ``False``. It lets Datasette know if the current SQL query can be represented by the plugin - and hence influence if a link to this output format is displayed in the user interface. If you omit the ``"can_render"`` key from the dictionary every query will be treated as being supported by the plugin. When a request is received, the ``"render"`` callback function is called with zero or more of the following arguments. Datasette will inspect your callback function and pass arguments that match its function signature. From 2a08ffed5c9fe0c0ac3d227a6e02bd7a83fc27e2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Apr 2024 18:47:01 -0700 Subject: [PATCH 036/334] Async example for track_event hook Closes #2319 --- docs/plugin_hooks.rst | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 78489d1e..87460e26 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1993,6 +1993,49 @@ This example plugin logs details of all events to standard error: ) print(msg, file=sys.stderr, flush=True) +The function can also return an async function which will be awaited. This is useful for writing to a database. + +This example logs events to a `datasette_events` table in a database called `events`. It uses the `startup()` hook to create that table if it does not exist. + +.. code-block:: python + + from datasette import hookimpl + import json + + + @hookimpl + def startup(datasette): + async def inner(): + db = datasette.get_database("events") + await db.execute_write( + """ + create table if not exists datasette_events ( + id integer primary key, + event_type text, + created text, + properties text + ) + """ + ) + + return inner + + + @hookimpl + def track_event(datasette, event): + async def inner(): + db = datasette.get_database("events") + properties = event.properties() + await db.execute_write( + """ + insert into datasette_events (event_type, created, properties) + values (?, strftime('%Y-%m-%d %H:%M:%S', 'now'),?) + """, + (event.name, json.dumps(properties)), + ) + + return inner + Example: `datasette-events-db `_ .. _plugin_hook_register_events: From 7d6d471dc5559e174507c8e85a38a00ea8009123 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Apr 2024 18:53:07 -0700 Subject: [PATCH 037/334] Include actor in track_event async example, refs #2319 --- docs/plugin_hooks.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 87460e26..972f3856 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -2001,8 +2001,7 @@ This example logs events to a `datasette_events` table in a database called `eve from datasette import hookimpl import json - - + @hookimpl def startup(datasette): async def inner(): @@ -2013,14 +2012,15 @@ This example logs events to a `datasette_events` table in a database called `eve id integer primary key, event_type text, created text, + actor text, properties text ) """ ) - + return inner - - + + @hookimpl def track_event(datasette, event): async def inner(): @@ -2028,12 +2028,12 @@ This example logs events to a `datasette_events` table in a database called `eve properties = event.properties() await db.execute_write( """ - insert into datasette_events (event_type, created, properties) - values (?, strftime('%Y-%m-%d %H:%M:%S', 'now'),?) + insert into datasette_events (event_type, created, actor, properties) + values (?, strftime('%Y-%m-%d %H:%M:%S', 'now'), ?, ?) """, - (event.name, json.dumps(properties)), + (event.name, json.dumps(event.actor), json.dumps(properties)), ) - + return inner Example: `datasette-events-db `_ From 8f9509f00cceea8dc87403c28b2056db7b246ed4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Apr 2024 16:01:37 -0700 Subject: [PATCH 038/334] datasette, not self.ds, in internals documentation --- docs/internals.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index b57a289a..79585659 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -386,7 +386,7 @@ This is useful when you need to check multiple permissions at once. For example, .. code-block:: python - await self.ds.ensure_permissions( + await datasette.ensure_permissions( request.actor, [ ("view-table", (database, table)), @@ -420,7 +420,7 @@ This example checks if the user can access a specific table, and sets ``private` .. code-block:: python - visible, private = await self.ds.check_visibility( + visible, private = await datasette.check_visibility( request.actor, action="view-table", resource=(database, table), @@ -430,7 +430,7 @@ The following example runs three checks in a row, similar to :ref:`datasette_ens .. code-block:: python - visible, private = await self.ds.check_visibility( + visible, private = await datasette.check_visibility( request.actor, permissions=[ ("view-table", (database, table)), From e1bfab3fca22e4f8d06dcf419e945f13fc20aef1 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Tue, 11 Jun 2024 09:33:23 -0700 Subject: [PATCH 039/334] Move Metadata to `--internal` database Refs: - https://github.com/simonw/datasette/pull/2343 - https://github.com/simonw/datasette/issues/2341 --- datasette/app.py | 202 +++++++++++++++++++++----------- datasette/default_menu_links.py | 4 - datasette/facets.py | 13 +- datasette/hookspecs.py | 5 - datasette/renderer.py | 1 - datasette/utils/internal_db.py | 37 ++++++ datasette/views/base.py | 6 +- datasette/views/database.py | 7 +- datasette/views/index.py | 10 +- datasette/views/row.py | 5 +- datasette/views/table.py | 38 ++++-- docs/codespell-ignore-words.txt | 3 +- docs/plugin_hooks.rst | 7 +- tests/test_api.py | 62 ++++++---- tests/test_cli.py | 4 +- tests/test_config_dir.py | 17 --- tests/test_facets.py | 9 +- tests/test_html.py | 14 +-- tests/test_permissions.py | 1 - tests/test_plugins.py | 41 +------ tests/test_routes.py | 2 - tests/test_table_html.py | 4 +- 22 files changed, 282 insertions(+), 210 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 23d21600..8020c5da 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -443,6 +443,37 @@ class Datasette: self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self) + async def apply_metadata_json(self): + # Apply any metadata entries from metadata.json to the internal tables + # step 1: top-level metadata + for key in self._metadata_local or {}: + if key == "databases": + continue + await self.set_instance_metadata(key, self._metadata_local[key]) + + # step 2: database-level metadata + for dbname, db in self._metadata_local.get("databases", {}).items(): + for key, value in db.items(): + if key == "tables": + continue + await self.set_database_metadata(dbname, key, value) + + # step 3: table-level metadata + for tablename, table in db.get("tables", {}).items(): + for key, value in table.items(): + if key == "columns": + continue + await self.set_resource_metadata(dbname, tablename, key, value) + + # step 4: column-level metadata (only descriptions in metadata.json) + for columnname, column_description in table.get("columns", {}).items(): + await self.set_column_metadata( + dbname, tablename, columnname, "description", column_description + ) + + # TODO(alex) is metadata.json was loaded in, and --internal is not memory, then log + # a warning to user that they should delete their metadata.json file + def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env if request: @@ -476,6 +507,7 @@ class Datasette: internal_db = self.get_internal_database() if not self.internal_db_created: await init_internal_db(internal_db) + await self.apply_metadata_json() self.internal_db_created = True current_schema_versions = { row["database_name"]: row["schema_version"] @@ -646,57 +678,113 @@ class Datasette: orig[key] = upd_value return orig - def metadata(self, key=None, database=None, table=None, fallback=True): - """ - Looks up metadata, cascading backwards from specified level. - Returns None if metadata value is not found. - """ - assert not ( - database is None and table is not None - ), "Cannot call metadata() with table= specified but not database=" - metadata = {} + async def get_instance_metadata(self): + rows = await self.get_internal_database().execute( + """ + SELECT + key, + value + FROM datasette_metadata_instance_entries + """ + ) + return dict(rows) - for hook_dbs in pm.hook.get_metadata( - datasette=self, key=key, database=database, table=table - ): - metadata = self._metadata_recursive_update(metadata, hook_dbs) + async def get_database_metadata(self, database_name: str): + rows = await self.get_internal_database().execute( + """ + SELECT + key, + value + FROM datasette_metadata_database_entries + WHERE database_name = ? + """, + [database_name], + ) + return dict(rows) - # security precaution!! don't allow anything in the local config - # to be overwritten. this is a temporary measure, not sure if this - # is a good idea long term or maybe if it should just be a concern - # of the plugin's implemtnation - metadata = self._metadata_recursive_update(metadata, self._metadata_local) + async def get_resource_metadata(self, database_name: str, resource_name: str): + rows = await self.get_internal_database().execute( + """ + SELECT + key, + value + FROM datasette_metadata_resource_entries + WHERE database_name = ? + AND resource_name = ? + """, + [database_name, resource_name], + ) + return dict(rows) - databases = metadata.get("databases") or {} + async def get_column_metadata( + self, database_name: str, resource_name: str, column_name: str + ): + rows = await self.get_internal_database().execute( + """ + SELECT + key, + value + FROM datasette_metadata_column_entries + WHERE database_name = ? + AND resource_name = ? + AND column_name = ? + """, + [database_name, resource_name, column_name], + ) + return dict(rows) - search_list = [] - if database is not None: - search_list.append(databases.get(database) or {}) - if table is not None: - table_metadata = ((databases.get(database) or {}).get("tables") or {}).get( - table - ) or {} - search_list.insert(0, table_metadata) + async def set_instance_metadata(self, key: str, value: str): + # TODO upsert only supported on SQLite 3.24.0 (2018-06-04) + await self.get_internal_database().execute_write( + """ + INSERT INTO datasette_metadata_instance_entries(key, value) + VALUES(?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value; + """, + [key, value], + ) - search_list.append(metadata) - if not fallback: - # No fallback allowed, so just use the first one in the list - search_list = search_list[:1] - if key is not None: - for item in search_list: - if key in item: - return item[key] - return None - else: - # Return the merged list - m = {} - for item in search_list: - m.update(item) - return m + async def set_database_metadata(self, database_name: str, key: str, value: str): + # TODO upsert only supported on SQLite 3.24.0 (2018-06-04) + await self.get_internal_database().execute_write( + """ + INSERT INTO datasette_metadata_database_entries(database_name, key, value) + VALUES(?, ?, ?) + ON CONFLICT(database_name, key) DO UPDATE SET value = excluded.value; + """, + [database_name, key, value], + ) - @property - def _metadata(self): - return self.metadata() + async def set_resource_metadata( + self, database_name: str, resource_name: str, key: str, value: str + ): + # TODO upsert only supported on SQLite 3.24.0 (2018-06-04) + await self.get_internal_database().execute_write( + """ + INSERT INTO datasette_metadata_resource_entries(database_name, resource_name, key, value) + VALUES(?, ?, ?, ?) + ON CONFLICT(database_name, resource_name, key) DO UPDATE SET value = excluded.value; + """, + [database_name, resource_name, key, value], + ) + + async def set_column_metadata( + self, + database_name: str, + resource_name: str, + column_name: str, + key: str, + value: str, + ): + # TODO upsert only supported on SQLite 3.24.0 (2018-06-04) + await self.get_internal_database().execute_write( + """ + INSERT INTO datasette_metadata_column_entries(database_name, resource_name, column_name, key, value) + VALUES(?, ?, ?, ?, ?) + ON CONFLICT(database_name, resource_name, column_name, key) DO UPDATE SET value = excluded.value; + """, + [database_name, resource_name, column_name, key, value], + ) def get_internal_database(self): return self._internal_database @@ -774,20 +862,6 @@ class Datasette: if query: return query - def update_with_inherited_metadata(self, metadata): - # Fills in source/license with defaults, if available - metadata.update( - { - "source": metadata.get("source") or self.metadata("source"), - "source_url": metadata.get("source_url") or self.metadata("source_url"), - "license": metadata.get("license") or self.metadata("license"), - "license_url": metadata.get("license_url") - or self.metadata("license_url"), - "about": metadata.get("about") or self.metadata("about"), - "about_url": metadata.get("about_url") or self.metadata("about_url"), - } - ) - def _prepare_connection(self, conn, database): conn.row_factory = sqlite3.Row conn.text_factory = lambda x: str(x, "utf-8", "replace") @@ -1079,11 +1153,6 @@ class Datasette: url = "https://" + url[len("http://") :] return url - def _register_custom_units(self): - """Register any custom units defined in the metadata.json with Pint""" - for unit in self.metadata("custom_units") or []: - ureg.define(unit) - def _connected_databases(self): return [ { @@ -1436,10 +1505,6 @@ class Datasette: ), r"/:memory:(?P.*)$", ) - add_route( - JsonDataView.as_view(self, "metadata.json", lambda: self.metadata()), - r"/-/metadata(\.(?Pjson))?$", - ) add_route( JsonDataView.as_view(self, "versions.json", self._versions), r"/-/versions(\.(?Pjson))?$", @@ -1585,7 +1650,6 @@ class Datasette: def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() - self._register_custom_units() async def setup_db(): # First time server starts up, calculate table counts for immutable databases diff --git a/datasette/default_menu_links.py b/datasette/default_menu_links.py index 56f481ef..22e6e46a 100644 --- a/datasette/default_menu_links.py +++ b/datasette/default_menu_links.py @@ -17,10 +17,6 @@ def menu_links(datasette, actor): "href": datasette.urls.path("/-/versions"), "label": "Version info", }, - { - "href": datasette.urls.path("/-/metadata"), - "label": "Metadata", - }, { "href": datasette.urls.path("/-/settings"), "label": "Settings", diff --git a/datasette/facets.py b/datasette/facets.py index f1cfc68f..ccd85461 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -103,10 +103,15 @@ class Facet: max_returned_rows = self.ds.setting("max_returned_rows") table_facet_size = None if self.table: - tables_metadata = self.ds.metadata("tables", database=self.database) or {} - table_metadata = tables_metadata.get(self.table) or {} - if table_metadata: - table_facet_size = table_metadata.get("facet_size") + config_facet_size = ( + self.ds.config.get("databases", {}) + .get(self.database, {}) + .get("tables", {}) + .get(self.table, {}) + .get("facet_size") + ) + if config_facet_size: + table_facet_size = config_facet_size custom_facet_size = self.request.args.get("_facet_size") if custom_facet_size: if custom_facet_size == "max": diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 35468cc3..bcc2e229 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -10,11 +10,6 @@ def startup(datasette): """Fires directly after Datasette first starts running""" -@hookspec -def get_metadata(datasette, key, database, table): - """Return metadata to be merged into Datasette's metadata dictionary""" - - @hookspec def asgi_wrapper(datasette): """Returns an ASGI middleware callable to wrap our ASGI application with""" diff --git a/datasette/renderer.py b/datasette/renderer.py index a446e69d..483c81e9 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -56,7 +56,6 @@ def json_renderer(request, args, data, error, truncated=None): if truncated is not None: data["truncated"] = truncated - if shape == "arrayfirst": if not data["rows"]: data = [] diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index dbfcceb4..6a5e08cb 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -63,6 +63,43 @@ async def init_internal_db(db): """ ).strip() await db.execute_write_script(create_tables_sql) + await initialize_metadata_tables(db) + + +async def initialize_metadata_tables(db): + await db.execute_write_script( + """ + CREATE TABLE IF NOT EXISTS datasette_metadata_instance_entries( + key text, + value text, + unique(key) + ); + + CREATE TABLE IF NOT EXISTS datasette_metadata_database_entries( + database_name text, + key text, + value text, + unique(database_name, key) + ); + + CREATE TABLE IF NOT EXISTS datasette_metadata_resource_entries( + database_name text, + resource_name text, + key text, + value text, + unique(database_name, resource_name, key) + ); + + CREATE TABLE IF NOT EXISTS datasette_metadata_column_entries( + database_name text, + resource_name text, + column_name text, + key text, + value text, + unique(database_name, resource_name, column_name, key) + ); + """ + ) async def populate_schema_tables(internal_db, db): diff --git a/datasette/views/base.py b/datasette/views/base.py index 9d7a854c..2e78b0a5 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -274,10 +274,6 @@ class DataView(BaseView): end = time.perf_counter() data["query_ms"] = (end - start) * 1000 - for key in ("source", "source_url", "license", "license_url"): - value = self.ds.metadata(key) - if value: - data[key] = value # Special case for .jsono extension - redirect to _shape=objects if _format == "jsono": @@ -385,7 +381,7 @@ class DataView(BaseView): }, } if "metadata" not in context: - context["metadata"] = self.ds.metadata() + context["metadata"] = await self.ds.get_instance_metadata() r = await self.render(templates, request=request, context=context) if status_code is not None: r.status = status_code diff --git a/datasette/views/database.py b/datasette/views/database.py index 851ae21f..2698a0eb 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -63,8 +63,7 @@ class DatabaseView(View): if format_ not in ("html", "json"): raise NotFound("Invalid format: {}".format(format_)) - metadata = (datasette.metadata("databases") or {}).get(database, {}) - datasette.update_with_inherited_metadata(metadata) + metadata = await datasette.get_database_metadata(database) sql_views = [] for view_name in await db.view_names(): @@ -131,6 +130,7 @@ class DatabaseView(View): "table_columns": ( await _table_columns(datasette, database) if allow_execute_sql else {} ), + "metadata": await datasette.get_database_metadata(database), } if format_ == "json": @@ -625,8 +625,7 @@ class QueryView(View): ) } ) - metadata = (datasette.metadata("databases") or {}).get(database, {}) - datasette.update_with_inherited_metadata(metadata) + metadata = await datasette.get_database_metadata(database) renderers = {} for key, (_, can_render) in datasette.renderers.items(): diff --git a/datasette/views/index.py b/datasette/views/index.py index 6546b7ae..a3178f53 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -132,7 +132,13 @@ class IndexView(BaseView): if self.ds.cors: add_cors_headers(headers) return Response( - json.dumps({db["name"]: db for db in databases}, cls=CustomJSONEncoder), + json.dumps( + { + "databases": {db["name"]: db for db in databases}, + "metadata": await self.ds.get_instance_metadata(), + }, + cls=CustomJSONEncoder, + ), content_type="application/json; charset=utf-8", headers=headers, ) @@ -151,7 +157,7 @@ class IndexView(BaseView): request=request, context={ "databases": databases, - "metadata": self.ds.metadata(), + "metadata": await self.ds.get_instance_metadata(), "datasette_version": __version__, "private": not await self.ds.permission_allowed( None, "view-instance" diff --git a/datasette/views/row.py b/datasette/views/row.py index 49d390f6..6180446f 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -85,10 +85,6 @@ class RowView(DataView): "_table.html", ], "row_actions": row_actions, - "metadata": (self.ds.metadata("databases") or {}) - .get(database, {}) - .get("tables", {}) - .get(table, {}), "top_row": make_slot_function( "top_row", self.ds, @@ -97,6 +93,7 @@ class RowView(DataView): table=resolved.table, row=rows[0], ), + "metadata": {}, } data = { diff --git a/datasette/views/table.py b/datasette/views/table.py index ba03241d..e3bfb260 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -147,7 +147,21 @@ async def display_columns_and_rows( """Returns columns, rows for specified table - including fancy foreign key treatment""" sortable_columns = sortable_columns or set() db = datasette.databases[database_name] - column_descriptions = datasette.metadata("columns", database_name, table_name) or {} + column_descriptions = dict( + await datasette.get_internal_database().execute( + """ + SELECT + column_name, + value + FROM datasette_metadata_column_entries + WHERE database_name = ? + AND resource_name = ? + AND key = 'description' + """, + [database_name, table_name], + ) + ) + column_details = { col.name: col for col in await db.table_column_details(table_name) } @@ -1478,14 +1492,22 @@ async def table_view_data( async def extra_metadata(): "Metadata about the table and database" - metadata = ( - (datasette.metadata("databases") or {}) - .get(database_name, {}) - .get("tables", {}) - .get(table_name, {}) + tablemetadata = await datasette.get_resource_metadata(database_name, table_name) + + rows = await datasette.get_internal_database().execute( + """ + SELECT + column_name, + value + FROM datasette_metadata_column_entries + WHERE database_name = ? + AND resource_name = ? + AND key = 'description' + """, + [database_name, table_name], ) - datasette.update_with_inherited_metadata(metadata) - return metadata + tablemetadata["columns"] = dict(rows) + return tablemetadata async def extra_database(): return database_name diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt index 1959863f..072f32b4 100644 --- a/docs/codespell-ignore-words.txt +++ b/docs/codespell-ignore-words.txt @@ -2,4 +2,5 @@ alls fo ro te -ths \ No newline at end of file +ths +notin \ No newline at end of file diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 972f3856..f4ef5d64 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -2002,6 +2002,7 @@ This example logs events to a `datasette_events` table in a database called `eve from datasette import hookimpl import json + @hookimpl def startup(datasette): async def inner(): @@ -2031,7 +2032,11 @@ This example logs events to a `datasette_events` table in a database called `eve insert into datasette_events (event_type, created, actor, properties) values (?, strftime('%Y-%m-%d %H:%M:%S', 'now'), ?, ?) """, - (event.name, json.dumps(event.actor), json.dumps(properties)), + ( + event.name, + json.dumps(event.actor), + json.dumps(properties), + ), ) return inner diff --git a/tests/test_api.py b/tests/test_api.py index 4ad55d72..bd734677 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -29,8 +29,19 @@ async def test_homepage(ds_client): assert response.status_code == 200 assert "application/json; charset=utf-8" == response.headers["content-type"] data = response.json() - assert data.keys() == {"fixtures": 0}.keys() - d = data["fixtures"] + assert sorted(list(data.get("metadata").keys())) == [ + "about", + "about_url", + "description_html", + "license", + "license_url", + "source", + "source_url", + "title", + ] + databases = data.get("databases") + assert databases.keys() == {"fixtures": 0}.keys() + d = databases["fixtures"] assert d["name"] == "fixtures" assert isinstance(d["tables_count"], int) assert isinstance(len(d["tables_and_views_truncated"]), int) @@ -45,7 +56,8 @@ async def test_homepage_sort_by_relationships(ds_client): response = await ds_client.get("/.json?_sort=relationships") assert response.status_code == 200 tables = [ - t["name"] for t in response.json()["fixtures"]["tables_and_views_truncated"] + t["name"] + for t in response.json()["databases"]["fixtures"]["tables_and_views_truncated"] ] assert tables == [ "simple_primary_key", @@ -590,21 +602,24 @@ def test_no_files_uses_memory_database(app_client_no_files): response = app_client_no_files.get("/.json") assert response.status == 200 assert { - "_memory": { - "name": "_memory", - "hash": None, - "color": "a6c7b9", - "path": "/_memory", - "tables_and_views_truncated": [], - "tables_and_views_more": False, - "tables_count": 0, - "table_rows_sum": 0, - "show_table_row_counts": False, - "hidden_table_rows_sum": 0, - "hidden_tables_count": 0, - "views_count": 0, - "private": False, - } + "databases": { + "_memory": { + "name": "_memory", + "hash": None, + "color": "a6c7b9", + "path": "/_memory", + "tables_and_views_truncated": [], + "tables_and_views_more": False, + "tables_count": 0, + "table_rows_sum": 0, + "show_table_row_counts": False, + "hidden_table_rows_sum": 0, + "hidden_tables_count": 0, + "views_count": 0, + "private": False, + }, + }, + "metadata": {}, } == response.json # Try that SQL query response = app_client_no_files.get( @@ -768,12 +783,6 @@ def test_databases_json(app_client_two_attached_databases_one_immutable): assert False == fixtures_database["is_memory"] -@pytest.mark.asyncio -async def test_metadata_json(ds_client): - response = await ds_client.get("/-/metadata.json") - assert response.json() == ds_client.ds.metadata() - - @pytest.mark.asyncio async def test_threads_json(ds_client): response = await ds_client.get("/-/threads.json") @@ -1039,8 +1048,8 @@ async def test_tilde_encoded_database_names(db_name): ds = Datasette() ds.add_memory_database(db_name) response = await ds.client.get("/.json") - assert db_name in response.json().keys() - path = response.json()[db_name]["path"] + assert db_name in response.json()["databases"].keys() + path = response.json()["databases"][db_name]["path"] # And the JSON for that database response2 = await ds.client.get(path + ".json") assert response2.status_code == 200 @@ -1083,6 +1092,7 @@ async def test_config_json(config, expected): @pytest.mark.asyncio +@pytest.mark.skip(reason="rm?") @pytest.mark.parametrize( "metadata,expected_config,expected_metadata", ( diff --git a/tests/test_cli.py b/tests/test_cli.py index bda17eed..486852cd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -159,8 +159,8 @@ def test_metadata_yaml(): internal=None, ) client = _TestClient(ds) - response = client.get("/-/metadata.json") - assert {"title": "Hello from YAML"} == response.json + response = client.get("/.json") + assert {"title": "Hello from YAML"} == response.json["metadata"] @mock.patch("datasette.cli.run_module") diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index 66114a27..46a6d341 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -99,12 +99,6 @@ def config_dir_client(config_dir): yield _TestClient(ds) -def test_metadata(config_dir_client): - response = config_dir_client.get("/-/metadata.json") - assert 200 == response.status - assert METADATA == response.json - - def test_settings(config_dir_client): response = config_dir_client.get("/-/settings.json") assert 200 == response.status @@ -149,17 +143,6 @@ def test_databases(config_dir_client): assert db["is_mutable"] == (expected_name != "immutable") -@pytest.mark.parametrize("filename", ("metadata.yml", "metadata.yaml")) -def test_metadata_yaml(tmp_path_factory, filename): - config_dir = tmp_path_factory.mktemp("yaml-config-dir") - (config_dir / filename).write_text("title: Title from metadata", "utf-8") - ds = Datasette([], config_dir=config_dir) - client = _TestClient(ds) - response = client.get("/-/metadata.json") - assert 200 == response.status - assert {"title": "Title from metadata"} == response.json - - def test_store_config_dir(config_dir_client): ds = config_dir_client.ds diff --git a/tests/test_facets.py b/tests/test_facets.py index 76344108..023efcf0 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -584,9 +584,9 @@ async def test_facet_size(): data5 = response5.json() assert len(data5["facet_results"]["results"]["city"]["results"]) == 20 # Now try messing with facet_size in the table metadata - orig_metadata = ds._metadata_local + orig_config = ds.config try: - ds._metadata_local = { + ds.config = { "databases": { "test_facet_size": {"tables": {"neighbourhoods": {"facet_size": 6}}} } @@ -597,7 +597,7 @@ async def test_facet_size(): data6 = response6.json() assert len(data6["facet_results"]["results"]["city"]["results"]) == 6 # Setting it to max bumps it up to 50 again - ds._metadata_local["databases"]["test_facet_size"]["tables"]["neighbourhoods"][ + ds.config["databases"]["test_facet_size"]["tables"]["neighbourhoods"][ "facet_size" ] = "max" data7 = ( @@ -605,7 +605,7 @@ async def test_facet_size(): ).json() assert len(data7["facet_results"]["results"]["city"]["results"]) == 20 finally: - ds._metadata_local = orig_metadata + ds.config = orig_config def test_other_types_of_facet_in_metadata(): @@ -655,7 +655,6 @@ async def test_facet_against_in_memory_database(): to_insert = [{"name": "one", "name2": "1"} for _ in range(800)] + [ {"name": "two", "name2": "2"} for _ in range(300) ] - print(to_insert) await db.execute_write_many( "insert into t (name, name2) values (:name, :name2)", to_insert ) diff --git a/tests/test_html.py b/tests/test_html.py index 42b290c8..de732d2c 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -446,7 +446,7 @@ async def test_database_metadata(ds_client): soup.find("div", {"class": "metadata-description"}) ) # The source/license should be inherited - assert_footer_links(soup) + # assert_footer_links(soup) TODO(alex) ensure @pytest.mark.asyncio @@ -459,7 +459,7 @@ async def test_database_metadata_with_custom_sql(ds_client): # Description should be custom assert "Custom SQL query returning" in soup.find("h3").text # The source/license should be inherited - assert_footer_links(soup) + # assert_footer_links(soup)TODO(alex) ensure def test_database_download_for_immutable(): @@ -752,14 +752,6 @@ async def test_blob_download_invalid_messages(ds_client, path, expected_message) assert expected_message in response.text -@pytest.mark.asyncio -async def test_metadata_json_html(ds_client): - response = await ds_client.get("/-/metadata") - assert response.status_code == 200 - pre = Soup(response.content, "html.parser").find("pre") - assert ds_client.ds.metadata() == json.loads(pre.text) - - @pytest.mark.asyncio @pytest.mark.parametrize( "path", @@ -931,7 +923,7 @@ def test_edit_sql_link_not_shown_if_user_lacks_permission(permission_allowed): [ (None, None, None), ("test", None, ["/-/permissions"]), - ("root", ["/-/permissions", "/-/allow-debug", "/-/metadata"], None), + ("root", ["/-/permissions", "/-/allow-debug"], None), ], ) async def test_navigation_menu_links( diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 72795319..fe91ec3d 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -453,7 +453,6 @@ def view_instance_client(): "/", "/fixtures", "/fixtures/facetable", - "/-/metadata", "/-/versions", "/-/plugins", "/-/settings", diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 9c8d02ec..cc2e16c8 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -331,14 +331,14 @@ def test_hook_extra_template_vars(restore_working_directory): with make_app_client( template_dir=str(pathlib.Path(__file__).parent / "test_templates") ) as client: - response = client.get("/-/metadata") + response = client.get("/-/versions") assert response.status_code == 200 extra_template_vars = json.loads( Soup(response.text, "html.parser").select("pre.extra_template_vars")[0].text ) assert { "template": "show_json.html", - "scope_path": "/-/metadata", + "scope_path": "/-/versions", "columns": None, } == extra_template_vars extra_template_vars_from_awaitable = json.loads( @@ -349,7 +349,7 @@ def test_hook_extra_template_vars(restore_working_directory): assert { "template": "show_json.html", "awaitable": True, - "scope_path": "/-/metadata", + "scope_path": "/-/versions", } == extra_template_vars_from_awaitable @@ -357,7 +357,7 @@ def test_plugins_async_template_function(restore_working_directory): with make_app_client( template_dir=str(pathlib.Path(__file__).parent / "test_templates") ) as client: - response = client.get("/-/metadata") + response = client.get("/-/versions") assert response.status_code == 200 extra_from_awaitable_function = ( Soup(response.text, "html.parser") @@ -422,7 +422,7 @@ def view_names_client(tmp_path_factory): ("/fixtures", "database"), ("/fixtures/units", "table"), ("/fixtures/units/1", "row"), - ("/-/metadata", "json_data"), + ("/-/versions", "json_data"), ("/fixtures?sql=select+1", "database"), ), ) @@ -1073,36 +1073,6 @@ def test_hook_skip_csrf(app_client): assert second_missing_csrf_response.status_code == 403 -@pytest.mark.asyncio -async def test_hook_get_metadata(ds_client): - try: - orig_metadata = ds_client.ds._metadata_local - ds_client.ds._metadata_local = { - "title": "Testing get_metadata hook!", - "databases": {"from-local": {"title": "Hello from local metadata"}}, - } - og_pm_hook_get_metadata = pm.hook.get_metadata - - def get_metadata_mock(*args, **kwargs): - return [ - { - "databases": { - "from-hook": {"title": "Hello from the plugin hook"}, - "from-local": {"title": "This will be overwritten!"}, - } - } - ] - - pm.hook.get_metadata = get_metadata_mock - meta = ds_client.ds.metadata() - assert "Testing get_metadata hook!" == meta["title"] - assert "Hello from local metadata" == meta["databases"]["from-local"]["title"] - assert "Hello from the plugin hook" == meta["databases"]["from-hook"]["title"] - pm.hook.get_metadata = og_pm_hook_get_metadata - finally: - ds_client.ds._metadata_local = orig_metadata - - def _extract_commands(output): lines = output.split("Commands:\n", 1)[1].split("\n") return {line.split()[0].replace("*", "") for line in lines if line.strip()} @@ -1550,6 +1520,7 @@ async def test_hook_register_events(): assert any(k.__name__ == "OneEvent" for k in datasette.event_classes) +@pytest.mark.skip(reason="TODO") @pytest.mark.parametrize( "metadata,config,expected_metadata,expected_config", ( diff --git a/tests/test_routes.py b/tests/test_routes.py index 85945dec..b2d31759 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -43,8 +43,6 @@ def routes(): "RowView", {"format": "json", "database": "foo", "pks": "1", "table": "humbug"}, ), - ("/-/metadata.json", "JsonDataView", {"format": "json"}), - ("/-/metadata", "JsonDataView", {"format": None}), ), ) def test_routes(routes, path, expected_name, expected_matches): diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 2a658663..fcf914a6 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -792,8 +792,6 @@ async def test_table_metadata(ds_client): assert "Simple primary key" == inner_html( soup.find("div", {"class": "metadata-description"}) ) - # The source/license should be inherited - assert_footer_links(soup) @pytest.mark.asyncio @@ -1101,8 +1099,8 @@ async def test_column_metadata(ds_client): soup = Soup(response.text, "html.parser") dl = soup.find("dl") assert [(dt.text, dt.nextSibling.text) for dt in dl.findAll("dt")] == [ - ("name", "The name of the attraction"), ("address", "The street address for the attraction"), + ("name", "The name of the attraction"), ] assert ( soup.select("th[data-column=name]")[0]["data-column-description"] From c698d008e08396ca0d288febbb9f656d96db83a9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Jun 2024 10:03:57 -0700 Subject: [PATCH 040/334] Only test first wheel, fixes surprise bug https://github.com/simonw/datasette/issues/2351#issuecomment-2161211173 --- test-in-pyodide-with-shot-scraper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh index 0c140818..acd93465 100755 --- a/test-in-pyodide-with-shot-scraper.sh +++ b/test-in-pyodide-with-shot-scraper.sh @@ -6,7 +6,7 @@ set -e python3 -m build # Find name of wheel, strip off the dist/ -wheel=$(basename $(ls dist/*.whl)) +wheel=$(basename $(ls dist/*.whl) | head -n 1) # Create a blank index page echo ' From 9a3c3bfcc7d30d65514421bd315f1b8b5b735b86 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Jun 2024 10:11:34 -0700 Subject: [PATCH 041/334] Fix for pyodide test failure, refs #2351 --- test-in-pyodide-with-shot-scraper.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh index acd93465..5845a0c3 100755 --- a/test-in-pyodide-with-shot-scraper.sh +++ b/test-in-pyodide-with-shot-scraper.sh @@ -18,6 +18,13 @@ cd dist python3 -m http.server 8529 & cd .. +# Register the kill_server function to be called on script exit +kill_server() { + pkill -f 'http.server 8529' +} +trap kill_server EXIT + + shot-scraper javascript http://localhost:8529/ " async () => { let pyodide = await loadPyodide(); @@ -26,6 +33,8 @@ async () => { import micropip await micropip.install('h11==0.12.0') await micropip.install('httpx==0.23') + # To avoid 'from typing_extensions import deprecated' error: + await micropip.install('typing-extensions>=4.12.2') await micropip.install('http://localhost:8529/$wheel') import ssl import setuptools @@ -38,7 +47,4 @@ async () => { } return 'Test passed!'; } -" - -# Shut down the server -pkill -f 'http.server 8529' +" \ No newline at end of file From 7437d40e5dd4d614bb769e16c0c1b96c6c19647f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Jun 2024 10:17:02 -0700 Subject: [PATCH 042/334] , closes #2348 --- datasette/templates/base.html | 2 +- datasette/templates/patterns.html | 2 +- tests/test_html.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/datasette/templates/base.html b/datasette/templates/base.html index afb8dcd3..51f602b6 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -1,5 +1,5 @@ {% import "_crumbs.html" as crumbs with context %} - + {% block title %}{% endblock %} diff --git a/datasette/templates/patterns.html b/datasette/templates/patterns.html index cb0daf9a..24fe5531 100644 --- a/datasette/templates/patterns.html +++ b/datasette/templates/patterns.html @@ -1,5 +1,5 @@ - + Datasette: Pattern Portfolio diff --git a/tests/test_html.py b/tests/test_html.py index de732d2c..a83f0e47 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -21,6 +21,8 @@ def test_homepage(app_client_two_attached_databases): response = app_client_two_attached_databases.get("/") assert response.status_code == 200 assert "text/html; charset=utf-8" == response.headers["content-type"] + # Should have a html lang="en" attribute + assert '' in response.text soup = Soup(response.content, "html.parser") assert "Datasette Fixtures" == soup.find("h1").text assert ( From 2b6bfddafc1f3ad95bbb334e34a026964db4a813 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Jun 2024 14:04:55 -0700 Subject: [PATCH 043/334] Workaround for #2353 --- datasette/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index e1108911..80754202 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1146,7 +1146,7 @@ async def derive_named_parameters(db: "Database", sql: str) -> List[str]: try: results = await db.execute(explain, {p: None for p in possible_params}) return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"] - except sqlite3.DatabaseError: + except (sqlite3.DatabaseError, AttributeError): return possible_params From 780deaa275849a39a283ec7b57a0351aac18a047 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Jun 2024 16:12:05 -0700 Subject: [PATCH 044/334] Reminder about how to deploy a release branch --- docs/contributing.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index b678e637..45330a83 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -284,6 +284,10 @@ You can generate the list of issue references for a specific release by copying To create the tag for the release, create `a new release `__ on GitHub matching the new version number. You can convert the release notes to Markdown by copying and pasting the rendered HTML into this `Paste to Markdown tool `__. +Don't forget to create the release from the correct branch - usually ``main``, but sometimes ``0.64.x`` or similar for a bugfix release. + +While the release is running you can confirm that the correct commits made it into the release using the https://github.com/simonw/datasette/compare/0.64.6...0.64.7 URL. + Finally, post a news item about the release on `datasette.io `__ by editing the `news.yaml `__ file in that site's repository. .. _contributing_alpha_beta: From b39b01a89066b1322bb9f731529e636d707e19f6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Jun 2024 16:21:07 -0700 Subject: [PATCH 045/334] Copy across release notes from 0.64.7 Refs #2353 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ebed499f..90bf75e1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v0_64_7: + +0.64.7 (2023-06-12) +------------------- + +- Fixed a bug where canned queries with named parameters threw an error when run against SQLite 3.46.0. (:issue:`2353`) + .. _v1_0_a13: 1.0a13 (2024-03-12) From d118d5c5bbb675f35baa5c376ee297b510dccfc0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Jun 2024 16:51:07 -0700 Subject: [PATCH 046/334] named_parameters(sql) sync function, refs #2354 Also refs #2353 and #2352 --- datasette/utils/__init__.py | 33 ++++++++++++++++++++++++--------- datasette/views/database.py | 6 ++---- docs/internals.rst | 10 +++++----- tests/test_utils.py | 8 ++++++-- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 80754202..b3b51c5f 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1131,23 +1131,38 @@ class StartupError(Exception): pass -_re_named_parameter = re.compile(":([a-zA-Z0-9_]+)") +_single_line_comment_re = re.compile(r"--.*") +_multi_line_comment_re = re.compile(r"/\*.*?\*/", re.DOTALL) +_single_quote_re = re.compile(r"'(?:''|[^'])*'") +_double_quote_re = re.compile(r'"(?:\"\"|[^"])*"') +_named_param_re = re.compile(r":(\w+)") @documented -async def derive_named_parameters(db: "Database", sql: str) -> List[str]: +def named_parameters(sql: str) -> List[str]: """ Given a SQL statement, return a list of named parameters that are used in the statement e.g. for ``select * from foo where id=:id`` this would return ``["id"]`` """ - explain = "explain {}".format(sql.strip().rstrip(";")) - possible_params = _re_named_parameter.findall(sql) - try: - results = await db.execute(explain, {p: None for p in possible_params}) - return [row["p4"].lstrip(":") for row in results if row["opcode"] == "Variable"] - except (sqlite3.DatabaseError, AttributeError): - return possible_params + # Remove single-line comments + sql = _single_line_comment_re.sub("", sql) + # Remove multi-line comments + sql = _multi_line_comment_re.sub("", sql) + # Remove single-quoted strings + sql = _single_quote_re.sub("", sql) + # Remove double-quoted strings + sql = _double_quote_re.sub("", sql) + # Extract parameters from what is left + return _named_param_re.findall(sql) + + +async def derive_named_parameters(db: "Database", sql: str) -> List[str]: + """ + This undocumented but stable method exists for backwards compatibility + with plugins that were using it before it switched to named_parameters() + """ + return named_parameters(sql) def add_cors_headers(headers): diff --git a/datasette/views/database.py b/datasette/views/database.py index 2698a0eb..1d76c5e0 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -17,7 +17,7 @@ from datasette.utils import ( add_cors_headers, await_me_maybe, call_with_supported_arguments, - derive_named_parameters, + named_parameters as derive_named_parameters, format_bytes, make_slot_function, tilde_decode, @@ -484,9 +484,7 @@ class QueryView(View): if canned_query and canned_query.get("params"): named_parameters = canned_query["params"] if not named_parameters: - named_parameters = await derive_named_parameters( - datasette.get_database(database), sql - ) + named_parameters = derive_named_parameters(sql) named_parameter_values = { named_parameter: params.get(named_parameter) or "" for named_parameter in named_parameters diff --git a/docs/internals.rst b/docs/internals.rst index 79585659..38e66a57 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1256,14 +1256,14 @@ Utility function for calling ``await`` on a return value if it is awaitable, oth .. autofunction:: datasette.utils.await_me_maybe -.. _internals_utils_derive_named_parameters: +.. _internals_utils_named_parameters: -derive_named_parameters(db, sql) --------------------------------- +named_parameters(sql) +--------------------- -Derive the list of named parameters referenced in a SQL query, using an ``explain`` query executed against the provided database. +Derive the list of ``:named`` parameters referenced in a SQL query. -.. autofunction:: datasette.utils.derive_named_parameters +.. autofunction:: datasette.utils.named_parameters .. _internals_tilde_encoding: diff --git a/tests/test_utils.py b/tests/test_utils.py index 254b1300..88a4532a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -612,10 +612,14 @@ def test_parse_metadata(content, expected): ("select this is invalid :one, :two, :three", ["one", "two", "three"]), ), ) -async def test_derive_named_parameters(sql, expected): +@pytest.mark.parametrize("use_async_version", (False, True)) +async def test_named_parameters(sql, expected, use_async_version): ds = Datasette([], memory=True) db = ds.get_database("_memory") - params = await utils.derive_named_parameters(db, sql) + if use_async_version: + params = await utils.derive_named_parameters(db, sql) + else: + params = utils.named_parameters(sql) assert params == expected From 64a125b860eda6f821ce5a67f48b13fd173d8671 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Jun 2024 16:56:46 -0700 Subject: [PATCH 047/334] Removed unnecessary comments, refs #2354 --- datasette/utils/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index b3b51c5f..c87de5f0 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1145,13 +1145,9 @@ def named_parameters(sql: str) -> List[str]: e.g. for ``select * from foo where id=:id`` this would return ``["id"]`` """ - # Remove single-line comments sql = _single_line_comment_re.sub("", sql) - # Remove multi-line comments sql = _multi_line_comment_re.sub("", sql) - # Remove single-quoted strings sql = _single_quote_re.sub("", sql) - # Remove double-quoted strings sql = _double_quote_re.sub("", sql) # Extract parameters from what is left return _named_param_re.findall(sql) From 8f86d2af6a5b90bd0bf12812b4f37c0fbfc1bceb Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 13 Jun 2024 10:09:45 -0700 Subject: [PATCH 048/334] Test against multiple SQLite versions (#2352) * Use sqlite-versions action for testing multiple versions --- .github/workflows/test-sqlite-support.yml | 53 +++++++++++++++++++++++ tests/test_csv.py | 1 + 2 files changed, 54 insertions(+) create mode 100644 .github/workflows/test-sqlite-support.yml diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml new file mode 100644 index 00000000..7882e05d --- /dev/null +++ b/.github/workflows/test-sqlite-support.yml @@ -0,0 +1,53 @@ +name: Test SQLite versions + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + test: + runs-on: ${{ matrix.platform }} + continue-on-error: true + strategy: + matrix: + platform: [ubuntu-latest] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12"] + sqlite-version: [ + #"3", # latest version + "3.46", + #"3.45", + #"3.27", + #"3.26", + "3.25", + #"3.25.3", # 2018-09-25, window functions breaks test_upsert for some reason on 3.10, skip for now + #"3.24", # 2018-06-04, added UPSERT support + #"3.23.1" # 2018-04-10, before UPSERT + ] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + cache: pip + cache-dependency-path: setup.py + - name: Set up SQLite ${{ matrix.sqlite-version }} + uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 + with: + version: ${{ matrix.sqlite-version }} + cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" + - run: python3 -c "import sqlite3; print(sqlite3.sqlite_version)" + - run: echo $LD_LIBRARY_PATH + - name: Build extension for --load-extension test + run: |- + (cd tests && gcc ext.c -fPIC -shared -o ext.so) + - name: Install dependencies + run: | + pip install -e '.[test]' + pip freeze + - name: Run tests + run: | + pytest -n auto -m "not serial" + pytest -m "serial" diff --git a/tests/test_csv.py b/tests/test_csv.py index 9f772f89..d4c072c0 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -189,6 +189,7 @@ async def test_csv_with_non_ascii_characters(ds_client): assert response.text == "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n" +@pytest.mark.skip(reason="flakey") def test_max_csv_mb(app_client_csv_max_mb_one): # This query deliberately generates a really long string # should be 100*100*100*2 = roughly 2MB From 45c27603d2b1f3536eccee7bdf20b9626a128bb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Jun 2024 10:15:38 -0700 Subject: [PATCH 049/334] xfail two flaky tests, #2355, #2356 --- tests/test_api_write.py | 1 + tests/test_csv.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 6a7ddeb6..2b57a99a 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1359,6 +1359,7 @@ async def test_create_table_permissions( @pytest.mark.asyncio +@pytest.mark.xfail(reason="Flaky, see https://github.com/simonw/datasette/issues/2356") @pytest.mark.parametrize( "input,expected_rows_after", ( diff --git a/tests/test_csv.py b/tests/test_csv.py index d4c072c0..6dbe693b 100644 --- a/tests/test_csv.py +++ b/tests/test_csv.py @@ -189,7 +189,7 @@ async def test_csv_with_non_ascii_characters(ds_client): assert response.text == "text,number\r\n𝐜𝐢𝐭𝐢𝐞𝐬,1\r\nbob,2\r\n" -@pytest.mark.skip(reason="flakey") +@pytest.mark.xfail(reason="Flaky, see https://github.com/simonw/datasette/issues/2355") def test_max_csv_mb(app_client_csv_max_mb_one): # This query deliberately generates a really long string # should be 100*100*100*2 = roughly 2MB From 93534fd3d0c34aac9ff5f8677f1a8d8bddf886f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Jun 2024 10:19:26 -0700 Subject: [PATCH 050/334] Show response.text on test_upsert failure, refs #2356 --- tests/test_api_write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 2b57a99a..7dbe12b5 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -537,7 +537,7 @@ async def test_upsert(ds_write, initial, input, expected_rows, should_return): json=input, headers=_headers(token), ) - assert response.status_code == 200 + assert response.status_code == 200, response.text assert response.json()["ok"] is True # Analytics event From 62686114eee429d6f5e6badc4f11076b35c987df Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 21 Jun 2024 16:02:15 -0700 Subject: [PATCH 051/334] Do not show database name in Database Not Found error, refs #2359 --- datasette/app.py | 4 +--- datasette/utils/asgi.py | 4 ++-- tests/test_api_write.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8020c5da..f732f95c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1617,9 +1617,7 @@ class Datasette: try: return self.get_database(route=database_route) except KeyError: - raise DatabaseNotFound( - "Database not found: {}".format(database_route), database_route - ) + raise DatabaseNotFound(database_route) async def resolve_table(self, request): db = await self.resolve_database(request) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 2fad1d42..b3f00bb3 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -23,9 +23,9 @@ class NotFound(Base400): class DatabaseNotFound(NotFound): - def __init__(self, message, database_name): - super().__init__(message) + def __init__(self, database_name): self.database_name = database_name + super().__init__("Database not found") class TableNotFound(NotFound): diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 7dbe12b5..2cd87858 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -133,7 +133,7 @@ async def test_insert_rows(ds_write, return_rows): {}, None, 404, - ["Database not found: data2"], + ["Database not found"], ), ( "/data/docs2/-/insert", From 7316dd4ac629d3b0e93a92b07cd6e565e2a66a07 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 21 Jun 2024 16:09:20 -0700 Subject: [PATCH 052/334] Fix for TableNotFound, refs #2359 --- datasette/app.py | 4 +--- datasette/utils/asgi.py | 4 ++-- tests/test_api_write.py | 10 +++++----- tests/test_table_api.py | 2 +- tests/test_table_html.py | 2 +- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f732f95c..001cf4c3 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1628,9 +1628,7 @@ class Datasette: if not table_exists: is_view = await db.view_exists(table_name) if not (table_exists or is_view): - raise TableNotFound( - "Table not found: {}".format(table_name), db.name, table_name - ) + raise TableNotFound(db.name, table_name) return ResolvedTable(db, table_name, is_view) async def resolve_row(self, request): diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index b3f00bb3..6bdba714 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -29,8 +29,8 @@ class DatabaseNotFound(NotFound): class TableNotFound(NotFound): - def __init__(self, message, database_name, table): - super().__init__(message) + def __init__(self, database_name, table): + super().__init__("Table not found") self.database_name = database_name self.table = table diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 2cd87858..94eb902b 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -140,7 +140,7 @@ async def test_insert_rows(ds_write, return_rows): {}, None, 404, - ["Table not found: docs2"], + ["Table not found"], ), ( "/data/docs/-/insert", @@ -274,7 +274,7 @@ async def test_insert_rows(ds_write, return_rows): {"rows": [{"title": "Test"}]}, None, 404, - ["Table not found: badtable"], + ["Table not found"], ), # missing primary key ( @@ -598,7 +598,7 @@ async def test_delete_row_errors(ds_write, scenario): assert ( response.json()["errors"] == ["Permission denied"] if scenario == "no_token" - else ["Table not found: bad_table"] + else ["Table not found"] ) assert len((await ds_write.client.get("/data/docs.json?_shape=array")).json()) == 1 @@ -703,7 +703,7 @@ async def test_update_row_check_permission(ds_write, scenario): assert ( response.json()["errors"] == ["Permission denied"] if scenario == "no_token" - else ["Table not found: bad_table"] + else ["Table not found"] ) @@ -830,7 +830,7 @@ async def test_drop_table(ds_write, scenario): assert response.json()["ok"] is False expected_error = "Permission denied" if scenario == "bad_table": - expected_error = "Table not found: bad_table" + expected_error = "Table not found" elif scenario == "immutable": expected_error = "Database is immutable" assert response.json()["errors"] == [expected_error] diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 58930950..8360157d 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -36,7 +36,7 @@ async def test_table_json(ds_client): async def test_table_not_exists_json(ds_client): assert (await ds_client.get("/fixtures/blah.json")).json() == { "ok": False, - "error": "Table not found: blah", + "error": "Table not found", "status": 404, "title": None, } diff --git a/tests/test_table_html.py b/tests/test_table_html.py index fcf914a6..1a9ed540 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -535,7 +535,7 @@ async def test_csv_json_export_links_include_labels_if_foreign_keys(ds_client): @pytest.mark.asyncio async def test_table_not_exists(ds_client): - assert "Table not found: blah" in (await ds_client.get("/fixtures/blah")).text + assert "Table not found" in (await ds_client.get("/fixtures/blah")).text @pytest.mark.asyncio From 263788906aad8d53cb29369983f72a3b861bb4da Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 21 Jun 2024 16:10:16 -0700 Subject: [PATCH 053/334] Fix for RowNotFound, refs #2359 --- datasette/app.py | 4 +--- datasette/utils/asgi.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 001cf4c3..1c5d6a37 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1638,9 +1638,7 @@ class Datasette: results = await db.execute(sql, params, truncate=True) row = results.first() if row is None: - raise RowNotFound( - "Row not found: {}".format(pk_values), db.name, table_name, pk_values - ) + raise RowNotFound(db.name, table_name, pk_values) return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) def app(self): diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 6bdba714..1699847e 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -36,8 +36,8 @@ class TableNotFound(NotFound): class RowNotFound(NotFound): - def __init__(self, message, database_name, table, pk_values): - super().__init__(message) + def __init__(self, database_name, table, pk_values): + super().__init__("Row not found") self.database_name = database_name self.table_name = table self.pk_values = pk_values From c2e8e5085b5ccc49121feaf11b57eb7d5af53bd9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 21 Jun 2024 16:36:58 -0700 Subject: [PATCH 054/334] Release notes for 0.64.8 on main --- docs/changelog.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 90bf75e1..dab17bc6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,14 @@ Changelog ========= +.. _v0_64_8: + +0.64.8 (2023-06-21) +------------------- + +- Security improvement: 404 pages used to reflect content from the URL path, which could be used to display misleading information to Datasette users. 404 errors no longer display additional information from the URL. (:issue:`2359`) +- Backported a better fix for correctly extracting named parameters from canned query SQL against SQLite 3.46.0. (:issue:`2353`) + .. _v0_64_7: 0.64.7 (2023-06-12) From 56adfff8d2112c5890888af4da65d842ba7f6f86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 09:45:15 -0700 Subject: [PATCH 055/334] Bump the python-packages group across 1 directory with 4 updates (#2362) Bumps the python-packages group with 4 updates in the / directory: [sphinx](https://github.com/sphinx-doc/sphinx), [furo](https://github.com/pradyunsg/furo), [blacken-docs](https://github.com/adamchainz/blacken-docs) and [black](https://github.com/psf/black). Updates `sphinx` from 7.2.6 to 7.3.7 - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.6...v7.3.7) Updates `furo` from 2024.1.29 to 2024.5.6 - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2024.01.29...2024.05.06) Updates `blacken-docs` from 1.16.0 to 1.18.0 - [Changelog](https://github.com/adamchainz/blacken-docs/blob/main/CHANGELOG.rst) - [Commits](https://github.com/adamchainz/blacken-docs/compare/1.16.0...1.18.0) Updates `black` from 24.2.0 to 24.4.2 - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/24.2.0...24.4.2) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: blacken-docs dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-packages ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index e51d8247..fa61729f 100644 --- a/setup.py +++ b/setup.py @@ -70,8 +70,8 @@ setup( """, extras_require={ "docs": [ - "Sphinx==7.2.6", - "furo==2024.1.29", + "Sphinx==7.3.7", + "furo==2024.5.6", "sphinx-autobuild", "codespell>=2.2.5", "blacken-docs", @@ -84,8 +84,8 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==24.2.0", - "blacken-docs==1.16.0", + "black==24.4.2", + "blacken-docs==1.18.0", "pytest-timeout>=1.4.2", "trustme>=0.7", "cogapp>=3.3.0", From a23c2aee006bea5f50c13e687dac77c66df1888b Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 15 Jul 2024 10:33:51 -0700 Subject: [PATCH 056/334] Introduce new `/$DB/-/query` endpoint, soft replaces `/$DB?sql=...` (#2363) * Introduce new default /$DB/-/query endpoint * Fix a lot of tests * Update pyodide test to use query endpoint * Link to /fixtures/-/query in a few places * Documentation for QueryView --------- Co-authored-by: Simon Willison --- datasette/app.py | 6 ++- datasette/templates/database.html | 4 +- datasette/views/database.py | 8 ++++ docs/pages.rst | 15 +++++++ test-in-pyodide-with-shot-scraper.sh | 4 +- tests/plugins/my_plugin.py | 1 + tests/test_api.py | 38 ++++++++++++------ tests/test_api_write.py | 12 ++++-- tests/test_canned_queries.py | 2 +- tests/test_cli.py | 8 ++-- tests/test_cli_serve_get.py | 2 +- tests/test_crossdb.py | 6 ++- tests/test_csv.py | 10 ++--- tests/test_html.py | 59 ++++++++++++++++------------ tests/test_load_extensions.py | 12 +++--- tests/test_messages.py | 4 +- tests/test_permissions.py | 8 ++-- tests/test_plugins.py | 22 ++++++----- tests/test_routes.py | 2 +- tests/test_table_api.py | 4 +- tests/test_table_html.py | 4 +- 21 files changed, 148 insertions(+), 83 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 1c5d6a37..5b8f910c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -37,7 +37,7 @@ from jinja2.exceptions import TemplateNotFound from .events import Event from .views import Context from .views.base import ureg -from .views.database import database_download, DatabaseView, TableCreateView +from .views.database import database_download, DatabaseView, TableCreateView, QueryView from .views.index import IndexView from .views.special import ( JsonDataView, @@ -1578,6 +1578,10 @@ class Datasette: r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") + add_route( + wrap_view(QueryView, self), + r"/(?P[^\/\.]+)/-/query(\.(?P\w+))?$", + ) add_route( wrap_view(table_view, self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)(\.(?P\w+))?$", diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 4a4a05cb..f921bc2d 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -21,7 +21,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if allow_execute_sql %} - +

    Custom SQL query

    @@ -36,7 +36,7 @@

    The following databases are attached to this connection, and can be used for cross-database joins:

      {% for db_name in attached_databases %} -
    • {{ db_name }} - tables
      • +
    • {{ db_name }} - tables
      • {% endfor %}
    diff --git a/datasette/views/database.py b/datasette/views/database.py index 1d76c5e0..9ab061a1 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -58,6 +58,11 @@ class DatabaseView(View): sql = (request.args.get("sql") or "").strip() if sql: + redirect_url = "/" + request.url_vars.get("database") + "/-/query" + if request.url_vars.get("format"): + redirect_url += "." + request.url_vars.get("format") + redirect_url += "?" + request.query_string + return Response.redirect(redirect_url) return await QueryView()(request, datasette) if format_ not in ("html", "json"): @@ -433,6 +438,8 @@ class QueryView(View): async def get(self, request, datasette): from datasette.app import TableNotFound + await datasette.refresh_schemas() + db = await datasette.resolve_database(request) database = db.name @@ -686,6 +693,7 @@ class QueryView(View): if allow_execute_sql and is_validated_sql and ":_" not in sql: edit_sql_url = ( datasette.urls.database(database) + + "/-/query" + "?" + urlencode( { diff --git a/docs/pages.rst b/docs/pages.rst index 1c3e2c1e..239c9f80 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -55,6 +55,21 @@ The following tables are hidden by default: - Tables relating to the inner workings of the SpatiaLite SQLite extension. - ``sqlite_stat`` tables used to store statistics used by the query optimizer. +.. _QueryView: + +Queries +======= + +The ``/database-name/-/query`` page can be used to execute an arbitrary SQL query against that database, if the :ref:`permissions_execute_sql` permission is enabled. This query is passed as the ``?sql=`` query string parameter. + +This means you can link directly to a query by constructing the following URL: + +``/database-name/-/query?sql=SELECT+*+FROM+table_name`` + +Each configured :ref:`canned query ` has its own page, at ``/database-name/query-name``. Viewing this page will execute the query and display the results. + +In both cases adding a ``.json`` extension to the URL will return the results as JSON. + .. _TableView: Table diff --git a/test-in-pyodide-with-shot-scraper.sh b/test-in-pyodide-with-shot-scraper.sh index 5845a0c3..4d1c4968 100755 --- a/test-in-pyodide-with-shot-scraper.sh +++ b/test-in-pyodide-with-shot-scraper.sh @@ -40,11 +40,11 @@ async () => { import setuptools from datasette.app import Datasette ds = Datasette(memory=True, settings={'num_sql_threads': 0}) - (await ds.client.get('/_memory.json?sql=select+55+as+itworks&_shape=array')).text + (await ds.client.get('/_memory/-/query.json?sql=select+55+as+itworks&_shape=array')).text \`); if (JSON.parse(output)[0].itworks != 55) { throw 'Got ' + output + ', expected itworks: 55'; } return 'Test passed!'; } -" \ No newline at end of file +" diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 9ef10181..4ca4f989 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -411,6 +411,7 @@ def query_actions(datasette, database, query_name, sql): return [ { "href": datasette.urls.database(database) + + "/-/query" + "?" + urllib.parse.urlencode( { diff --git a/tests/test_api.py b/tests/test_api.py index bd734677..431ab5ce 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -623,7 +623,7 @@ def test_no_files_uses_memory_database(app_client_no_files): } == response.json # Try that SQL query response = app_client_no_files.get( - "/_memory.json?sql=select+sqlite_version()&_shape=array" + "/_memory/-/query.json?sql=select+sqlite_version()&_shape=array" ) assert 1 == len(response.json) assert ["sqlite_version()"] == list(response.json[0].keys()) @@ -653,7 +653,7 @@ def test_database_page_for_database_with_dot_in_name(app_client_with_dot): @pytest.mark.asyncio async def test_custom_sql(ds_client): response = await ds_client.get( - "/fixtures.json?sql=select+content+from+simple_primary_key" + "/fixtures/-/query.json?sql=select+content+from+simple_primary_key", ) data = response.json() assert data == { @@ -670,7 +670,9 @@ async def test_custom_sql(ds_client): def test_sql_time_limit(app_client_shorter_time_limit): - response = app_client_shorter_time_limit.get("/fixtures.json?sql=select+sleep(0.5)") + response = app_client_shorter_time_limit.get( + "/fixtures/-/query.json?sql=select+sleep(0.5)", + ) assert 400 == response.status assert response.json == { "ok": False, @@ -691,16 +693,22 @@ def test_sql_time_limit(app_client_shorter_time_limit): @pytest.mark.asyncio async def test_custom_sql_time_limit(ds_client): - response = await ds_client.get("/fixtures.json?sql=select+sleep(0.01)") + response = await ds_client.get( + "/fixtures/-/query.json?sql=select+sleep(0.01)", + ) assert response.status_code == 200 - response = await ds_client.get("/fixtures.json?sql=select+sleep(0.01)&_timelimit=5") + response = await ds_client.get( + "/fixtures/-/query.json?sql=select+sleep(0.01)&_timelimit=5", + ) assert response.status_code == 400 assert response.json()["title"] == "SQL Interrupted" @pytest.mark.asyncio async def test_invalid_custom_sql(ds_client): - response = await ds_client.get("/fixtures.json?sql=.schema") + response = await ds_client.get( + "/fixtures/-/query.json?sql=.schema", + ) assert response.status_code == 400 assert response.json()["ok"] is False assert "Statement must be a SELECT" == response.json()["error"] @@ -883,9 +891,13 @@ async def test_json_columns(ds_client, extra_args, expected): select 1 as intval, "s" as strval, 0.5 as floatval, '{"foo": "bar"}' as jsonval """ - path = "/fixtures.json?" + urllib.parse.urlencode({"sql": sql, "_shape": "array"}) + path = "/fixtures/-/query.json?" + urllib.parse.urlencode( + {"sql": sql, "_shape": "array"} + ) path += extra_args - response = await ds_client.get(path) + response = await ds_client.get( + path, + ) assert response.json() == expected @@ -917,7 +929,7 @@ def test_config_force_https_urls(): ("/fixtures.json", 200), ("/fixtures/no_primary_key.json", 200), # A 400 invalid SQL query should still have the header: - ("/fixtures.json?sql=select+blah", 400), + ("/fixtures/-/query.json?sql=select+blah", 400), # Write APIs ("/fixtures/-/create", 405), ("/fixtures/facetable/-/insert", 405), @@ -930,7 +942,9 @@ def test_cors( path, status_code, ): - response = app_client_with_cors.get(path) + response = app_client_with_cors.get( + path, + ) assert response.status == status_code assert response.headers["Access-Control-Allow-Origin"] == "*" assert ( @@ -946,7 +960,9 @@ def test_cors( # should not have those headers - I'm using that fixture because # regular app_client doesn't have immutable fixtures.db which means # the test for /fixtures.db returns a 403 error - response = app_client_two_attached_databases_one_immutable.get(path) + response = app_client_two_attached_databases_one_immutable.get( + path, + ) assert response.status == status_code assert "Access-Control-Allow-Origin" not in response.headers assert "Access-Control-Allow-Headers" not in response.headers diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 94eb902b..b442113b 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -637,7 +637,9 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): # Should be a single row assert ( await ds_write.client.get( - "/data.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(table) + "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format( + table + ) ) ).json() == [1] # Now delete the row @@ -645,7 +647,9 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): # Special case for that rowid table delete_path = ( await ds_write.client.get( - "/data.json?_shape=arrayfirst&sql=select+rowid+from+{}".format(table) + "/data/-/query.json?_shape=arrayfirst&sql=select+rowid+from+{}".format( + table + ) ) ).json()[0] @@ -663,7 +667,9 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path): assert event.pks == str(delete_path).split(",") assert ( await ds_write.client.get( - "/data.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(table) + "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format( + table + ) ) ).json() == [0] diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index 69ed5ff9..d1ed06d1 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -412,7 +412,7 @@ def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_js def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client): response = magic_parameters_client.get( - "/data.json?sql=select+:_header_host&_shape=array" + "/data/-/query.json?sql=select+:_header_host&_shape=array" ) assert 400 == response.status assert response.json["error"].startswith("You did not supply a value for binding") diff --git a/tests/test_cli.py b/tests/test_cli.py index 486852cd..d53e0a5f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -250,7 +250,7 @@ def test_plugin_s_overwrite(): "--plugins-dir", plugins_dir, "--get", - "/_memory.json?sql=select+prepare_connection_args()", + "/_memory/-/query.json?sql=select+prepare_connection_args()", ], ) assert result.exit_code == 0, result.output @@ -265,7 +265,7 @@ def test_plugin_s_overwrite(): "--plugins-dir", plugins_dir, "--get", - "/_memory.json?sql=select+prepare_connection_args()", + "/_memory/-/query.json?sql=select+prepare_connection_args()", "-s", "plugins.name-of-plugin", "OVERRIDE", @@ -295,7 +295,7 @@ def test_setting_default_allow_sql(default_allow_sql): "default_allow_sql", "on" if default_allow_sql else "off", "--get", - "/_memory.json?sql=select+21&_shape=objects", + "/_memory/-/query.json?sql=select+21&_shape=objects", ], ) if default_allow_sql: @@ -309,7 +309,7 @@ def test_setting_default_allow_sql(default_allow_sql): def test_sql_errors_logged_to_stderr(): runner = CliRunner(mix_stderr=False) - result = runner.invoke(cli, ["--get", "/_memory.json?sql=select+blah"]) + result = runner.invoke(cli, ["--get", "/_memory/-/query.json?sql=select+blah"]) assert result.exit_code == 1 assert "sql = 'select blah', params = {}: no such column: blah\n" in result.stderr diff --git a/tests/test_cli_serve_get.py b/tests/test_cli_serve_get.py index 1088d906..513669a1 100644 --- a/tests/test_cli_serve_get.py +++ b/tests/test_cli_serve_get.py @@ -31,7 +31,7 @@ def test_serve_with_get(tmp_path_factory): "--plugins-dir", str(plugins_dir), "--get", - "/_memory.json?sql=select+sqlite_version()", + "/_memory/-/query.json?sql=select+sqlite_version()", ], ) assert result.exit_code == 0, result.output diff --git a/tests/test_crossdb.py b/tests/test_crossdb.py index 01c51130..58b81f70 100644 --- a/tests/test_crossdb.py +++ b/tests/test_crossdb.py @@ -25,7 +25,8 @@ def test_crossdb_join(app_client_two_attached_databases_crossdb_enabled): fixtures.searchable """ response = app_client.get( - "/_memory.json?" + urllib.parse.urlencode({"sql": sql, "_shape": "array"}) + "/_memory/-/query.json?" + + urllib.parse.urlencode({"sql": sql, "_shape": "array"}) ) assert response.status == 200 assert response.json == [ @@ -67,9 +68,10 @@ def test_crossdb_attached_database_list_display( ): app_client = app_client_two_attached_databases_crossdb_enabled response = app_client.get("/_memory") + response2 = app_client.get("/") for fragment in ( "databases are attached to this connection", "
  • fixtures - ", - "
  • extra database - ", + '
  • extra database - json' in response.text - assert 'CSV' in response.text + assert 'json' in response.text + assert ( + 'CSV' + in response.text + ) @pytest.mark.asyncio async def test_query_parameter_form_fields(ds_client): - response = await ds_client.get("/fixtures?sql=select+:name") + response = await ds_client.get("/fixtures/-/query?sql=select+:name") assert response.status_code == 200 assert ( ' ' in response.text ) - response2 = await ds_client.get("/fixtures?sql=select+:name&name=hello") + response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") assert response2.status_code == 200 assert ( ' ' @@ -453,7 +458,9 @@ async def test_database_metadata(ds_client): @pytest.mark.asyncio async def test_database_metadata_with_custom_sql(ds_client): - response = await ds_client.get("/fixtures?sql=select+*+from+simple_primary_key") + response = await ds_client.get( + "/fixtures/-/query?sql=select+*+from+simple_primary_key" + ) assert response.status_code == 200 soup = Soup(response.text, "html.parser") # Page title should be the default @@ -591,7 +598,7 @@ async def test_canned_query_with_custom_metadata(ds_client): @pytest.mark.asyncio async def test_urlify_custom_queries(ds_client): - path = "/fixtures?" + urllib.parse.urlencode( + path = "/fixtures/-/query?" + urllib.parse.urlencode( {"sql": "select ('https://twitter.com/' || 'simonw') as user_url;"} ) response = await ds_client.get(path) @@ -609,7 +616,7 @@ async def test_urlify_custom_queries(ds_client): @pytest.mark.asyncio async def test_show_hide_sql_query(ds_client): - path = "/fixtures?" + urllib.parse.urlencode( + path = "/fixtures/-/query?" + urllib.parse.urlencode( {"sql": "select ('https://twitter.com/' || 'simonw') as user_url;"} ) response = await ds_client.get(path) @@ -696,15 +703,15 @@ def test_canned_query_show_hide_metadata_option( @pytest.mark.asyncio async def test_binary_data_display_in_query(ds_client): - response = await ds_client.get("/fixtures?sql=select+*+from+binary_data") + response = await ds_client.get("/fixtures/-/query?sql=select+*+from+binary_data") assert response.status_code == 200 table = Soup(response.content, "html.parser").find("table") expected_tds = [ [ - '
    • ' + '' ], [ - '' + '' ], [''], ] @@ -719,7 +726,7 @@ async def test_binary_data_display_in_query(ds_client): [ ("/fixtures/binary_data/1.blob?_blob_column=data", "binary_data-1-data.blob"), ( - "/fixtures.blob?sql=select+*+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d", + "/fixtures/-/query.blob?sql=select+*+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d", "data-f30889.blob", ), ], @@ -758,7 +765,7 @@ async def test_blob_download_invalid_messages(ds_client, path, expected_message) @pytest.mark.parametrize( "path", [ - "/fixtures?sql=select+*+from+[123_starts_with_digits]", + "/fixtures/-/query?sql=select+*+from+[123_starts_with_digits]", "/fixtures/123_starts_with_digits", ], ) @@ -771,7 +778,7 @@ async def test_zero_results(ds_client, path): @pytest.mark.asyncio async def test_query_error(ds_client): - response = await ds_client.get("/fixtures?sql=select+*+from+notatable") + response = await ds_client.get("/fixtures/-/query?sql=select+*+from+notatable") html = response.text assert '

    no such table: notatable

    ' in html assert ' diff --git a/datasette/templates/api_explorer.html b/datasette/templates/api_explorer.html index 109fb1e9..dc393c20 100644 --- a/datasette/templates/api_explorer.html +++ b/datasette/templates/api_explorer.html @@ -19,7 +19,7 @@

    GET - +
    @@ -29,7 +29,7 @@
    POST - +
    diff --git a/datasette/templates/create_token.html b/datasette/templates/create_token.html index 2be98d38..409fb8a9 100644 --- a/datasette/templates/create_token.html +++ b/datasette/templates/create_token.html @@ -39,7 +39,7 @@ {% endfor %} {% endif %} - +

    diff --git a/datasette/templates/logout.html b/datasette/templates/logout.html index 4c4a7d11..c8fc642a 100644 --- a/datasette/templates/logout.html +++ b/datasette/templates/logout.html @@ -8,7 +8,7 @@

    You are logged in as {{ display_actor(actor) }}

    - +
    diff --git a/datasette/templates/messages_debug.html b/datasette/templates/messages_debug.html index e0ab9a40..2940cd69 100644 --- a/datasette/templates/messages_debug.html +++ b/datasette/templates/messages_debug.html @@ -8,7 +8,7 @@

    Set a message:

    - +
    diff --git a/datasette/templates/permissions_debug.html b/datasette/templates/permissions_debug.html index 5a5c1aa6..83891181 100644 --- a/datasette/templates/permissions_debug.html +++ b/datasette/templates/permissions_debug.html @@ -47,7 +47,7 @@ textarea {

    This tool lets you simulate an actor and a permission check for that actor.

    - +

    diff --git a/datasette/templates/query.html b/datasette/templates/query.html index f7c8d0a3..a6e9a3aa 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -36,7 +36,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

    diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 7246ff5d..c9e0e87b 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -48,7 +48,7 @@ {% endif %} - + {% if supports_search %}
    {% endif %} @@ -152,7 +152,7 @@ object {% endif %}

    - +

    CSV options: diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 534d8b33..8cc40f0f 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -83,6 +83,15 @@ database column they are representing, for example:

    <Binary:\xa07\xa0bytes><Binary:\xa07\xa0bytes><Binary:\xa07\xa0bytes><Binary:\xa07\xa0bytes>\xa0
    +.. _customization_css: + +Writing custom CSS +~~~~~~~~~~~~~~~~~~ + +Custom templates need to take Datasette's default CSS into account. The pattern portfolio at ``/-/patterns`` (`example here `__) is a useful reference for understanding the available CSS classes. + +The ``core`` class is particularly useful - you can apply this directly to a ```` or ``