From bee25f58cbeafb5aba3648a7e4f516632bc7d81d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 11:47:13 -0700 Subject: [PATCH 001/156] set-column-types permission, refs #2671 --- datasette/default_actions.py | 6 ++++++ docs/authentication.rst | 16 +++++++++++++++- tests/test_auth.py | 19 +++++++++++++++++++ tests/test_internals_datasette.py | 9 ++++++++- tests/test_permissions.py | 16 ++++++++++++++++ 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 87d98fac..216d0046 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -85,6 +85,12 @@ def register_actions(): description="Alter tables", resource_class=TableResource, ), + Action( + name="set-column-types", + abbr="sct", + description="Set column types", + resource_class=TableResource, + ), Action( name="drop-table", abbr="dt", diff --git a/docs/authentication.rst b/docs/authentication.rst index 1b949f9a..90fd26ce 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -33,7 +33,7 @@ The one exception is the "root" account, which you can sign into while using Dat The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including: * All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.) -* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``drop-table``) +* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-types``, ``drop-table``) * Debug permissions (``permissions-debug``, ``debug-menu``) * Any custom permissions defined by plugins @@ -886,6 +886,8 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` } .. [[[end]]] +Other table-scoped write permissions, including ``set-column-types``, can be configured in the same place. + And for ``insert-row`` against the ``reports`` table in that ``docs`` database: .. [[[cog @@ -1343,6 +1345,18 @@ alter-table Actor is allowed to alter a database table. +``resource`` - ``datasette.resources.TableResource(database, table)`` + ``database`` is the name of the database (string) + + ``table`` is the name of the table (string) + +.. _actions_set_column_types: + +set-column-types +---------------- + +Actor is allowed to set assigned column types for columns in a table. + ``resource`` - ``datasette.resources.TableResource(database, table)`` ``database`` is the name of the database (string) diff --git a/tests/test_auth.py b/tests/test_auth.py index 1e1cd622..cb77c7a2 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -191,6 +191,7 @@ def test_auth_create_token( "all:view-query", "database:fixtures:drop-table", "resource:fixtures:foreign_key_references:insert-row", + "resource:fixtures:facetable:set-column-types", } ) # Now try actually creating one @@ -427,6 +428,15 @@ async def test_root_with_root_enabled_gets_all_permissions(ds_client): is True ) + assert ( + await ds_client.ds.allowed( + action="set-column-types", + resource=TableResource("fixtures", "facetable"), + actor=root_actor, + ) + is True + ) + assert ( await ds_client.ds.allowed( action="drop-table", @@ -491,3 +501,12 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client): ) is not True ), "Root without root_enabled should not automatically get drop-table" + + assert ( + await ds_client.ds.allowed( + action="set-column-types", + resource=TableResource("fixtures", "facetable"), + actor=root_actor, + ) + is not True + ), "Root without root_enabled should not automatically get set-column-types" diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index b378a158..008fa7cd 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -164,7 +164,14 @@ def test_datasette_error_if_string_not_list(tmpdir): @pytest.mark.asyncio async def test_get_action(ds_client): ds = ds_client.ds - for name_or_abbr in ("vi", "view-instance", "vt", "view-table"): + for name_or_abbr in ( + "vi", + "view-instance", + "vt", + "view-table", + "sct", + "set-column-types", + ): action = ds.get_action(name_or_abbr) if "-" in name_or_abbr: assert action.name == name_or_abbr diff --git a/tests/test_permissions.py b/tests/test_permissions.py index be8969d7..4db89a0e 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -831,6 +831,22 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "t1"), expected_result=True, ), + # set-column-types on specific table + PermConfigTestCase( + config={ + "databases": { + "perms_ds_one": { + "tables": { + "t1": {"permissions": {"set-column-types": {"id": "user"}}} + } + } + } + }, + actor={"id": "user"}, + action="set-column-types", + resource=("perms_ds_one", "t1"), + expected_result=True, + ), # insert-row on database PermConfigTestCase( config={ From 3f5dd2b876c2bdefa9fc86f03ec7616f915da2cc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 12:15:42 -0700 Subject: [PATCH 002/156] /db/table/-/set-column-type JSON API, refs #2671 --- datasette/app.py | 5 + datasette/views/table.py | 116 +++++++++++++++++++ docs/json_api.rst | 64 +++++++++++ tests/test_column_types.py | 220 +++++++++++++++++++++++++++++++++++++ 4 files changed, 405 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 6e3e6815..4c98e521 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -68,6 +68,7 @@ from .views.special import ( from .views.table import ( TableInsertView, TableUpsertView, + TableSetColumnTypeView, TableDropView, table_view, ) @@ -2240,6 +2241,10 @@ class Datasette: TableUpsertView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/upsert$", ) + add_route( + TableSetColumnTypeView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-column-type$", + ) add_route( TableDropView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/drop$", diff --git a/datasette/views/table.py b/datasette/views/table.py index cf5b5b64..a6b13918 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -666,6 +666,122 @@ class TableUpsertView(TableInsertView): return await super().post(request, upsert=True) +class TableSetColumnTypeView(BaseView): + name = "table-set-column-type" + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + try: + resolved = await self.ds.resolve_table(request) + except NotFound as e: + return _error([e.args[0]], 404) + + database_name = resolved.db.name + table_name = resolved.table + + if not await self.ds.allowed( + action="set-column-types", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return _error(["Permission denied"], 403) + + content_type = request.headers.get("content-type") or "" + if not content_type.startswith("application/json"): + return _error(["Invalid content-type, must be application/json"], 400) + + try: + data = json.loads(await request.post_body()) + except json.JSONDecodeError as e: + return _error(["Invalid JSON: {}".format(e)], 400) + + if not isinstance(data, dict): + return _error(["JSON must be a dictionary"], 400) + + invalid_keys = set(data.keys()) - {"column", "column_type"} + if invalid_keys: + return _error( + ['Invalid parameter: "{}"'.format('", "'.join(sorted(invalid_keys)))], + 400, + ) + + if "column" not in data: + return _error(['"column" is required'], 400) + column = data["column"] + if not isinstance(column, str): + return _error(['"column" must be a string'], 400) + + if "column_type" not in data: + return _error(['"column_type" is required'], 400) + + column_details = await self.ds._get_resource_column_details( + database_name, table_name + ) + if column not in column_details: + return _error(["Column not found: {}".format(column)], 400) + + column_type_data = data["column_type"] + if column_type_data is None: + await self.ds.remove_column_type(database_name, table_name, column) + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "column": column, + "column_type": None, + }, + status=200, + ) + + if not isinstance(column_type_data, dict): + return _error(['"column_type" must be an object or null'], 400) + + invalid_column_type_keys = set(column_type_data.keys()) - {"type", "config"} + if invalid_column_type_keys: + return _error( + [ + 'Invalid column_type parameter: "{}"'.format( + '", "'.join(sorted(invalid_column_type_keys)) + ) + ], + 400, + ) + + if "type" not in column_type_data: + return _error(['"column_type.type" is required'], 400) + column_type = column_type_data["type"] + if not isinstance(column_type, str): + return _error(['"column_type.type" must be a string'], 400) + + config = column_type_data.get("config") + if config is not None and not isinstance(config, dict): + return _error(['"column_type.config" must be a dictionary'], 400) + + if column_type not in self.ds._column_types: + return _error(["Unknown column type: {}".format(column_type)], 400) + + try: + await self.ds.set_column_type( + database_name, table_name, column, column_type, config + ) + except ValueError as e: + return _error([str(e)], 400) + + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "column": column, + "column_type": {"type": column_type, "config": config}, + }, + status=200, + ) + + class TableDropView(BaseView): name = "table-drop" diff --git a/docs/json_api.rst b/docs/json_api.rst index 891aa9e0..7a48a26e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -941,6 +941,70 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission. +.. _TableSetColumnTypeView: + +Setting a column type +~~~~~~~~~~~~~~~~~~~~~ + +To set a column type for a table column, make a ``POST`` to ``//
/-/set-column-type``. This requires the :ref:`actions_set_column_types` permission. + +:: + + POST //
/-/set-column-type + Content-Type: application/json + Authorization: Bearer dstok_ + +.. code-block:: json + + { + "column": "title", + "column_type": { + "type": "email" + } + } + +This will return a ``200`` response like this: + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "posts", + "column": "title", + "column_type": { + "type": "email", + "config": null + } + } + +To provide column type configuration, include a ``config`` object: + +.. code-block:: json + + { + "column": "title", + "column_type": { + "type": "url", + "config": { + "max_length": 200 + } + } + } + +To clear an existing column type assignment, set ``column_type`` to ``null``: + +.. code-block:: json + + { + "column": "title", + "column_type": null + } + +This API stores the assignment in Datasette's internal database, so it can be used with immutable databases as well as mutable ones. + +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. + .. _TableDropView: Dropping tables diff --git a/tests/test_column_types.py b/tests/test_column_types.py index 538217b0..4fd30812 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -186,6 +186,226 @@ async def test_set_column_type_with_config(ds_ct): assert ct.config == {"max_length": 200} +@pytest.mark.asyncio +async def test_set_column_type_api(ds_ct): + await ds_ct.invoke_startup() + token = write_token(ds_ct, permissions=["sct"]) + response = await ds_ct.client.post( + "/data/posts/-/set-column-type", + json={"column": "title", "column_type": {"type": "email"}}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json() == { + "ok": True, + "database": "data", + "table": "posts", + "column": "title", + "column_type": {"type": "email", "config": None}, + } + ct = await ds_ct.get_column_type("data", "posts", "title") + assert ct.name == "email" + assert ct.config is None + + +@pytest.mark.asyncio +async def test_set_column_type_api_with_config(ds_ct): + await ds_ct.invoke_startup() + token = write_token(ds_ct, permissions=["sct"]) + response = await ds_ct.client.post( + "/data/posts/-/set-column-type", + json={ + "column": "title", + "column_type": {"type": "url", "config": {"max_length": 200}}, + }, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json()["column_type"] == { + "type": "url", + "config": {"max_length": 200}, + } + ct = await ds_ct.get_column_type("data", "posts", "title") + assert ct.name == "url" + assert ct.config == {"max_length": 200} + + +@pytest.mark.asyncio +async def test_clear_column_type_api(ds_ct): + await ds_ct.invoke_startup() + await ds_ct.set_column_type("data", "posts", "title", "email") + token = write_token(ds_ct, permissions=["sct"]) + response = await ds_ct.client.post( + "/data/posts/-/set-column-type", + json={"column": "title", "column_type": None}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json() == { + "ok": True, + "database": "data", + "table": "posts", + "column": "title", + "column_type": None, + } + ct = await ds_ct.get_column_type("data", "posts", "title") + assert ct is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body,special_case,expected_status,expected_errors", + ( + ( + {"column": "title", "column_type": {"type": "email"}}, + "no_permission", + 403, + ["Permission denied"], + ), + ( + None, + "invalid_json", + 400, + [ + "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" + ], + ), + ( + {"column": "title", "column_type": {"type": "email"}}, + "invalid_content_type", + 400, + ["Invalid content-type, must be application/json"], + ), + ( + [], + None, + 400, + ["JSON must be a dictionary"], + ), + ( + {"column_type": {"type": "email"}}, + None, + 400, + ['"column" is required'], + ), + ( + {"column": 1, "column_type": {"type": "email"}}, + None, + 400, + ['"column" must be a string'], + ), + ( + {"column": "not_a_column", "column_type": {"type": "email"}}, + None, + 400, + ["Column not found: not_a_column"], + ), + ( + {"column": "title", "column_type": "email"}, + None, + 400, + ['"column_type" must be an object or null'], + ), + ( + {"column": "title", "column_type": {}}, + None, + 400, + ['"column_type.type" is required'], + ), + ( + {"column": "title", "column_type": {"type": 1}}, + None, + 400, + ['"column_type.type" must be a string'], + ), + ( + {"column": "title", "column_type": {"type": "url", "config": []}}, + None, + 400, + ['"column_type.config" must be a dictionary'], + ), + ( + {"column": "title", "column_type": {"type": "markdown"}}, + None, + 400, + ["Unknown column type: markdown"], + ), + ( + {"column": "id", "column_type": {"type": "json"}}, + None, + 400, + [ + "Column type 'json' is only applicable to SQLite types TEXT but data.posts.id has SQLite type INTEGER" + ], + ), + ( + { + "column": "title", + "column_type": {"type": "email"}, + "extra": True, + }, + None, + 400, + ['Invalid parameter: "extra"'], + ), + ), +) +async def test_set_column_type_api_errors( + ds_ct, body, special_case, expected_status, expected_errors +): + await ds_ct.invoke_startup() + token = write_token( + ds_ct, + permissions=(["sct"] if special_case != "no_permission" else ["vi"]), + ) + kwargs = { + "headers": { + "Authorization": f"Bearer {token}", + "Content-Type": ( + "text/plain" + if special_case == "invalid_content_type" + else "application/json" + ), + } + } + if special_case == "invalid_json": + kwargs["content"] = "{bad json" + else: + kwargs["json"] = body + response = await ds_ct.client.post("/data/posts/-/set-column-type", **kwargs) + assert response.status_code == expected_status + assert response.json() == {"ok": False, "errors": expected_errors} + + +@pytest.mark.asyncio +async def test_set_column_type_api_works_for_immutable_database(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "immutable.db") + db = sqlite3.connect(str(db_path)) + db.execute("vacuum") + db.execute("create table posts (id integer primary key, title text)") + db.commit() + ds = Datasette([], immutables=[db_path]) + ds.root_enabled = True + try: + await ds.invoke_startup() + token = write_token(ds, permissions=["sct"]) + response = await ds.client.post( + "/immutable/posts/-/set-column-type", + json={"column": "title", "column_type": {"type": "email"}}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json()["column_type"] == {"type": "email", "config": None} + ct = await ds.get_column_type("immutable", "posts", "title") + assert ct.name == "email" + finally: + db.close() + for database in ds.databases.values(): + if not database.is_memory: + database.close() + + @pytest.mark.asyncio async def test_set_column_type_rejects_incompatible_sqlite_type(ds_ct): await ds_ct.invoke_startup() From 341a488db0a93ebb16e11d1a0d82f6bc58fef204 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 12:18:48 -0700 Subject: [PATCH 003/156] Rename set-column-types action to et-column-type Refs https://github.com/simonw/datasette/pull/2674#issuecomment-4085015792 --- datasette/default_actions.py | 4 ++-- datasette/views/table.py | 2 +- docs/authentication.rst | 14 ++++++-------- docs/json_api.rst | 2 +- tests/test_auth.py | 8 ++++---- tests/test_internals_datasette.py | 2 +- tests/test_permissions.py | 6 +++--- 7 files changed, 18 insertions(+), 20 deletions(-) diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 216d0046..149a4e5f 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -86,9 +86,9 @@ def register_actions(): resource_class=TableResource, ), Action( - name="set-column-types", + name="set-column-type", abbr="sct", - description="Set column types", + description="Set column type", resource_class=TableResource, ), Action( diff --git a/datasette/views/table.py b/datasette/views/table.py index a6b13918..e7a226af 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -682,7 +682,7 @@ class TableSetColumnTypeView(BaseView): table_name = resolved.table if not await self.ds.allowed( - action="set-column-types", + action="set-column-type", resource=TableResource(database=database_name, table=table_name), actor=request.actor, ): diff --git a/docs/authentication.rst b/docs/authentication.rst index 90fd26ce..951a65ec 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -33,7 +33,7 @@ The one exception is the "root" account, which you can sign into while using Dat The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including: * All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.) -* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-types``, ``drop-table``) +* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-type``, ``drop-table``) * Debug permissions (``permissions-debug``, ``debug-menu``) * Any custom permissions defined by plugins @@ -886,7 +886,7 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` } .. [[[end]]] -Other table-scoped write permissions, including ``set-column-types``, can be configured in the same place. +Other table-scoped write permissions, including ``set-column-type``, can be configured in the same place. And for ``insert-row`` against the ``reports`` table in that ``docs`` database: @@ -1212,9 +1212,7 @@ To include an expiry pass ``expire_after=`` to ``datasette.set_actor_cookie()`` .. code-block:: python response = Response.redirect("/") - datasette.set_actor_cookie( - response, {"id": "cleopaws"}, expire_after=60 * 60 * 24 - ) + datasette.set_actor_cookie(response, {"id": "cleopaws"}, expire_after=60 * 60 * 24) The resulting cookie will encode data that looks something like this: @@ -1350,10 +1348,10 @@ Actor is allowed to alter a database table. ``table`` is the name of the table (string) -.. _actions_set_column_types: +.. _actions_set_column_type: -set-column-types ----------------- +set-column-type +--------------- Actor is allowed to set assigned column types for columns in a table. diff --git a/docs/json_api.rst b/docs/json_api.rst index 7a48a26e..48c70af6 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -946,7 +946,7 @@ Pass ``"alter": true`` to automatically add any missing columns to the existing Setting a column type ~~~~~~~~~~~~~~~~~~~~~ -To set a column type for a table column, make a ``POST`` to ``//
/-/set-column-type``. This requires the :ref:`actions_set_column_types` permission. +To set a column type for a table column, make a ``POST`` to ``//
/-/set-column-type``. This requires the :ref:`actions_set_column_type` permission. :: diff --git a/tests/test_auth.py b/tests/test_auth.py index cb77c7a2..5868a21c 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -191,7 +191,7 @@ def test_auth_create_token( "all:view-query", "database:fixtures:drop-table", "resource:fixtures:foreign_key_references:insert-row", - "resource:fixtures:facetable:set-column-types", + "resource:fixtures:facetable:set-column-type", } ) # Now try actually creating one @@ -430,7 +430,7 @@ async def test_root_with_root_enabled_gets_all_permissions(ds_client): assert ( await ds_client.ds.allowed( - action="set-column-types", + action="set-column-type", resource=TableResource("fixtures", "facetable"), actor=root_actor, ) @@ -504,9 +504,9 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client): assert ( await ds_client.ds.allowed( - action="set-column-types", + action="set-column-type", resource=TableResource("fixtures", "facetable"), actor=root_actor, ) is not True - ), "Root without root_enabled should not automatically get set-column-types" + ), "Root without root_enabled should not automatically get set-column-type" diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 008fa7cd..ec0180a7 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -170,7 +170,7 @@ async def test_get_action(ds_client): "vt", "view-table", "sct", - "set-column-types", + "set-column-type", ): action = ds.get_action(name_or_abbr) if "-" in name_or_abbr: diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 4db89a0e..f9303759 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -831,19 +831,19 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "t1"), expected_result=True, ), - # set-column-types on specific table + # set-column-type on specific table PermConfigTestCase( config={ "databases": { "perms_ds_one": { "tables": { - "t1": {"permissions": {"set-column-types": {"id": "user"}}} + "t1": {"permissions": {"set-column-type": {"id": "user"}}} } } } }, actor={"id": "user"}, - action="set-column-types", + action="set-column-type", resource=("perms_ds_one", "t1"), expected_result=True, ), From 2704bc9c61ca79f9aa7a73fd49565ec166e7573d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 12:30:26 -0700 Subject: [PATCH 004/156] blacken-docs --- docs/authentication.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 951a65ec..a796d11e 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1212,7 +1212,9 @@ To include an expiry pass ``expire_after=`` to ``datasette.set_actor_cookie()`` .. code-block:: python response = Response.redirect("/") - datasette.set_actor_cookie(response, {"id": "cleopaws"}, expire_after=60 * 60 * 24) + datasette.set_actor_cookie( + response, {"id": "cleopaws"}, expire_after=60 * 60 * 24 + ) The resulting cookie will encode data that looks something like this: From fa1d8f0fa58cf2efdb552c76607806d5aa72098b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 11:47:13 -0700 Subject: [PATCH 005/156] set-column-types permission, refs #2671 --- datasette/default_actions.py | 6 ++++++ docs/authentication.rst | 16 +++++++++++++++- tests/test_auth.py | 19 +++++++++++++++++++ tests/test_internals_datasette.py | 9 ++++++++- tests/test_permissions.py | 16 ++++++++++++++++ 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 87d98fac..216d0046 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -85,6 +85,12 @@ def register_actions(): description="Alter tables", resource_class=TableResource, ), + Action( + name="set-column-types", + abbr="sct", + description="Set column types", + resource_class=TableResource, + ), Action( name="drop-table", abbr="dt", diff --git a/docs/authentication.rst b/docs/authentication.rst index 1b949f9a..90fd26ce 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -33,7 +33,7 @@ The one exception is the "root" account, which you can sign into while using Dat The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including: * All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.) -* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``drop-table``) +* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-types``, ``drop-table``) * Debug permissions (``permissions-debug``, ``debug-menu``) * Any custom permissions defined by plugins @@ -886,6 +886,8 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` } .. [[[end]]] +Other table-scoped write permissions, including ``set-column-types``, can be configured in the same place. + And for ``insert-row`` against the ``reports`` table in that ``docs`` database: .. [[[cog @@ -1343,6 +1345,18 @@ alter-table Actor is allowed to alter a database table. +``resource`` - ``datasette.resources.TableResource(database, table)`` + ``database`` is the name of the database (string) + + ``table`` is the name of the table (string) + +.. _actions_set_column_types: + +set-column-types +---------------- + +Actor is allowed to set assigned column types for columns in a table. + ``resource`` - ``datasette.resources.TableResource(database, table)`` ``database`` is the name of the database (string) diff --git a/tests/test_auth.py b/tests/test_auth.py index 1e1cd622..cb77c7a2 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -191,6 +191,7 @@ def test_auth_create_token( "all:view-query", "database:fixtures:drop-table", "resource:fixtures:foreign_key_references:insert-row", + "resource:fixtures:facetable:set-column-types", } ) # Now try actually creating one @@ -427,6 +428,15 @@ async def test_root_with_root_enabled_gets_all_permissions(ds_client): is True ) + assert ( + await ds_client.ds.allowed( + action="set-column-types", + resource=TableResource("fixtures", "facetable"), + actor=root_actor, + ) + is True + ) + assert ( await ds_client.ds.allowed( action="drop-table", @@ -491,3 +501,12 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client): ) is not True ), "Root without root_enabled should not automatically get drop-table" + + assert ( + await ds_client.ds.allowed( + action="set-column-types", + resource=TableResource("fixtures", "facetable"), + actor=root_actor, + ) + is not True + ), "Root without root_enabled should not automatically get set-column-types" diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index b378a158..008fa7cd 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -164,7 +164,14 @@ def test_datasette_error_if_string_not_list(tmpdir): @pytest.mark.asyncio async def test_get_action(ds_client): ds = ds_client.ds - for name_or_abbr in ("vi", "view-instance", "vt", "view-table"): + for name_or_abbr in ( + "vi", + "view-instance", + "vt", + "view-table", + "sct", + "set-column-types", + ): action = ds.get_action(name_or_abbr) if "-" in name_or_abbr: assert action.name == name_or_abbr diff --git a/tests/test_permissions.py b/tests/test_permissions.py index be8969d7..4db89a0e 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -831,6 +831,22 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "t1"), expected_result=True, ), + # set-column-types on specific table + PermConfigTestCase( + config={ + "databases": { + "perms_ds_one": { + "tables": { + "t1": {"permissions": {"set-column-types": {"id": "user"}}} + } + } + } + }, + actor={"id": "user"}, + action="set-column-types", + resource=("perms_ds_one", "t1"), + expected_result=True, + ), # insert-row on database PermConfigTestCase( config={ From d440c209849ab441ff1f86f33687fab3beb0f672 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 12:15:42 -0700 Subject: [PATCH 006/156] /db/table/-/set-column-type JSON API, refs #2671 --- datasette/app.py | 5 + datasette/views/table.py | 116 +++++++++++++++++++ docs/json_api.rst | 64 +++++++++++ tests/test_column_types.py | 220 +++++++++++++++++++++++++++++++++++++ 4 files changed, 405 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 6e3e6815..4c98e521 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -68,6 +68,7 @@ from .views.special import ( from .views.table import ( TableInsertView, TableUpsertView, + TableSetColumnTypeView, TableDropView, table_view, ) @@ -2240,6 +2241,10 @@ class Datasette: TableUpsertView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/upsert$", ) + add_route( + TableSetColumnTypeView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/set-column-type$", + ) add_route( TableDropView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/drop$", diff --git a/datasette/views/table.py b/datasette/views/table.py index cf5b5b64..a6b13918 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -666,6 +666,122 @@ class TableUpsertView(TableInsertView): return await super().post(request, upsert=True) +class TableSetColumnTypeView(BaseView): + name = "table-set-column-type" + + def __init__(self, datasette): + self.ds = datasette + + async def post(self, request): + try: + resolved = await self.ds.resolve_table(request) + except NotFound as e: + return _error([e.args[0]], 404) + + database_name = resolved.db.name + table_name = resolved.table + + if not await self.ds.allowed( + action="set-column-types", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return _error(["Permission denied"], 403) + + content_type = request.headers.get("content-type") or "" + if not content_type.startswith("application/json"): + return _error(["Invalid content-type, must be application/json"], 400) + + try: + data = json.loads(await request.post_body()) + except json.JSONDecodeError as e: + return _error(["Invalid JSON: {}".format(e)], 400) + + if not isinstance(data, dict): + return _error(["JSON must be a dictionary"], 400) + + invalid_keys = set(data.keys()) - {"column", "column_type"} + if invalid_keys: + return _error( + ['Invalid parameter: "{}"'.format('", "'.join(sorted(invalid_keys)))], + 400, + ) + + if "column" not in data: + return _error(['"column" is required'], 400) + column = data["column"] + if not isinstance(column, str): + return _error(['"column" must be a string'], 400) + + if "column_type" not in data: + return _error(['"column_type" is required'], 400) + + column_details = await self.ds._get_resource_column_details( + database_name, table_name + ) + if column not in column_details: + return _error(["Column not found: {}".format(column)], 400) + + column_type_data = data["column_type"] + if column_type_data is None: + await self.ds.remove_column_type(database_name, table_name, column) + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "column": column, + "column_type": None, + }, + status=200, + ) + + if not isinstance(column_type_data, dict): + return _error(['"column_type" must be an object or null'], 400) + + invalid_column_type_keys = set(column_type_data.keys()) - {"type", "config"} + if invalid_column_type_keys: + return _error( + [ + 'Invalid column_type parameter: "{}"'.format( + '", "'.join(sorted(invalid_column_type_keys)) + ) + ], + 400, + ) + + if "type" not in column_type_data: + return _error(['"column_type.type" is required'], 400) + column_type = column_type_data["type"] + if not isinstance(column_type, str): + return _error(['"column_type.type" must be a string'], 400) + + config = column_type_data.get("config") + if config is not None and not isinstance(config, dict): + return _error(['"column_type.config" must be a dictionary'], 400) + + if column_type not in self.ds._column_types: + return _error(["Unknown column type: {}".format(column_type)], 400) + + try: + await self.ds.set_column_type( + database_name, table_name, column, column_type, config + ) + except ValueError as e: + return _error([str(e)], 400) + + return Response.json( + { + "ok": True, + "database": database_name, + "table": table_name, + "column": column, + "column_type": {"type": column_type, "config": config}, + }, + status=200, + ) + + class TableDropView(BaseView): name = "table-drop" diff --git a/docs/json_api.rst b/docs/json_api.rst index 891aa9e0..7a48a26e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -941,6 +941,70 @@ To use the ``"replace": true`` option you will also need the :ref:`actions_updat Pass ``"alter": true`` to automatically add any missing columns to the existing table that are present in the rows you are submitting. This requires the :ref:`actions_alter_table` permission. +.. _TableSetColumnTypeView: + +Setting a column type +~~~~~~~~~~~~~~~~~~~~~ + +To set a column type for a table column, make a ``POST`` to ``//
/-/set-column-type``. This requires the :ref:`actions_set_column_types` permission. + +:: + + POST //
/-/set-column-type + Content-Type: application/json + Authorization: Bearer dstok_ + +.. code-block:: json + + { + "column": "title", + "column_type": { + "type": "email" + } + } + +This will return a ``200`` response like this: + +.. code-block:: json + + { + "ok": true, + "database": "data", + "table": "posts", + "column": "title", + "column_type": { + "type": "email", + "config": null + } + } + +To provide column type configuration, include a ``config`` object: + +.. code-block:: json + + { + "column": "title", + "column_type": { + "type": "url", + "config": { + "max_length": 200 + } + } + } + +To clear an existing column type assignment, set ``column_type`` to ``null``: + +.. code-block:: json + + { + "column": "title", + "column_type": null + } + +This API stores the assignment in Datasette's internal database, so it can be used with immutable databases as well as mutable ones. + +Any errors will return ``{"errors": ["... descriptive message ..."], "ok": false}``, and a ``400`` status code for a bad input or a ``403`` status code for an authentication or permission error. + .. _TableDropView: Dropping tables diff --git a/tests/test_column_types.py b/tests/test_column_types.py index 538217b0..4fd30812 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -186,6 +186,226 @@ async def test_set_column_type_with_config(ds_ct): assert ct.config == {"max_length": 200} +@pytest.mark.asyncio +async def test_set_column_type_api(ds_ct): + await ds_ct.invoke_startup() + token = write_token(ds_ct, permissions=["sct"]) + response = await ds_ct.client.post( + "/data/posts/-/set-column-type", + json={"column": "title", "column_type": {"type": "email"}}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json() == { + "ok": True, + "database": "data", + "table": "posts", + "column": "title", + "column_type": {"type": "email", "config": None}, + } + ct = await ds_ct.get_column_type("data", "posts", "title") + assert ct.name == "email" + assert ct.config is None + + +@pytest.mark.asyncio +async def test_set_column_type_api_with_config(ds_ct): + await ds_ct.invoke_startup() + token = write_token(ds_ct, permissions=["sct"]) + response = await ds_ct.client.post( + "/data/posts/-/set-column-type", + json={ + "column": "title", + "column_type": {"type": "url", "config": {"max_length": 200}}, + }, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json()["column_type"] == { + "type": "url", + "config": {"max_length": 200}, + } + ct = await ds_ct.get_column_type("data", "posts", "title") + assert ct.name == "url" + assert ct.config == {"max_length": 200} + + +@pytest.mark.asyncio +async def test_clear_column_type_api(ds_ct): + await ds_ct.invoke_startup() + await ds_ct.set_column_type("data", "posts", "title", "email") + token = write_token(ds_ct, permissions=["sct"]) + response = await ds_ct.client.post( + "/data/posts/-/set-column-type", + json={"column": "title", "column_type": None}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json() == { + "ok": True, + "database": "data", + "table": "posts", + "column": "title", + "column_type": None, + } + ct = await ds_ct.get_column_type("data", "posts", "title") + assert ct is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body,special_case,expected_status,expected_errors", + ( + ( + {"column": "title", "column_type": {"type": "email"}}, + "no_permission", + 403, + ["Permission denied"], + ), + ( + None, + "invalid_json", + 400, + [ + "Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)" + ], + ), + ( + {"column": "title", "column_type": {"type": "email"}}, + "invalid_content_type", + 400, + ["Invalid content-type, must be application/json"], + ), + ( + [], + None, + 400, + ["JSON must be a dictionary"], + ), + ( + {"column_type": {"type": "email"}}, + None, + 400, + ['"column" is required'], + ), + ( + {"column": 1, "column_type": {"type": "email"}}, + None, + 400, + ['"column" must be a string'], + ), + ( + {"column": "not_a_column", "column_type": {"type": "email"}}, + None, + 400, + ["Column not found: not_a_column"], + ), + ( + {"column": "title", "column_type": "email"}, + None, + 400, + ['"column_type" must be an object or null'], + ), + ( + {"column": "title", "column_type": {}}, + None, + 400, + ['"column_type.type" is required'], + ), + ( + {"column": "title", "column_type": {"type": 1}}, + None, + 400, + ['"column_type.type" must be a string'], + ), + ( + {"column": "title", "column_type": {"type": "url", "config": []}}, + None, + 400, + ['"column_type.config" must be a dictionary'], + ), + ( + {"column": "title", "column_type": {"type": "markdown"}}, + None, + 400, + ["Unknown column type: markdown"], + ), + ( + {"column": "id", "column_type": {"type": "json"}}, + None, + 400, + [ + "Column type 'json' is only applicable to SQLite types TEXT but data.posts.id has SQLite type INTEGER" + ], + ), + ( + { + "column": "title", + "column_type": {"type": "email"}, + "extra": True, + }, + None, + 400, + ['Invalid parameter: "extra"'], + ), + ), +) +async def test_set_column_type_api_errors( + ds_ct, body, special_case, expected_status, expected_errors +): + await ds_ct.invoke_startup() + token = write_token( + ds_ct, + permissions=(["sct"] if special_case != "no_permission" else ["vi"]), + ) + kwargs = { + "headers": { + "Authorization": f"Bearer {token}", + "Content-Type": ( + "text/plain" + if special_case == "invalid_content_type" + else "application/json" + ), + } + } + if special_case == "invalid_json": + kwargs["content"] = "{bad json" + else: + kwargs["json"] = body + response = await ds_ct.client.post("/data/posts/-/set-column-type", **kwargs) + assert response.status_code == expected_status + assert response.json() == {"ok": False, "errors": expected_errors} + + +@pytest.mark.asyncio +async def test_set_column_type_api_works_for_immutable_database(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "immutable.db") + db = sqlite3.connect(str(db_path)) + db.execute("vacuum") + db.execute("create table posts (id integer primary key, title text)") + db.commit() + ds = Datasette([], immutables=[db_path]) + ds.root_enabled = True + try: + await ds.invoke_startup() + token = write_token(ds, permissions=["sct"]) + response = await ds.client.post( + "/immutable/posts/-/set-column-type", + json={"column": "title", "column_type": {"type": "email"}}, + headers=_headers(token), + ) + assert response.status_code == 200 + assert response.json()["column_type"] == {"type": "email", "config": None} + ct = await ds.get_column_type("immutable", "posts", "title") + assert ct.name == "email" + finally: + db.close() + for database in ds.databases.values(): + if not database.is_memory: + database.close() + + @pytest.mark.asyncio async def test_set_column_type_rejects_incompatible_sqlite_type(ds_ct): await ds_ct.invoke_startup() From 2b06da29a16779ce54b2a57e79a9a2adb47ffcce Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 12:18:48 -0700 Subject: [PATCH 007/156] Rename set-column-types action to et-column-type Refs https://github.com/simonw/datasette/pull/2674#issuecomment-4085015792 --- datasette/default_actions.py | 4 ++-- datasette/views/table.py | 2 +- docs/authentication.rst | 14 ++++++-------- docs/json_api.rst | 2 +- tests/test_auth.py | 8 ++++---- tests/test_internals_datasette.py | 2 +- tests/test_permissions.py | 6 +++--- 7 files changed, 18 insertions(+), 20 deletions(-) diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 216d0046..149a4e5f 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -86,9 +86,9 @@ def register_actions(): resource_class=TableResource, ), Action( - name="set-column-types", + name="set-column-type", abbr="sct", - description="Set column types", + description="Set column type", resource_class=TableResource, ), Action( diff --git a/datasette/views/table.py b/datasette/views/table.py index a6b13918..e7a226af 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -682,7 +682,7 @@ class TableSetColumnTypeView(BaseView): table_name = resolved.table if not await self.ds.allowed( - action="set-column-types", + action="set-column-type", resource=TableResource(database=database_name, table=table_name), actor=request.actor, ): diff --git a/docs/authentication.rst b/docs/authentication.rst index 90fd26ce..951a65ec 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -33,7 +33,7 @@ The one exception is the "root" account, which you can sign into while using Dat The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including: * All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.) -* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-types``, ``drop-table``) +* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-type``, ``drop-table``) * Debug permissions (``permissions-debug``, ``debug-menu``) * Any custom permissions defined by plugins @@ -886,7 +886,7 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` } .. [[[end]]] -Other table-scoped write permissions, including ``set-column-types``, can be configured in the same place. +Other table-scoped write permissions, including ``set-column-type``, can be configured in the same place. And for ``insert-row`` against the ``reports`` table in that ``docs`` database: @@ -1212,9 +1212,7 @@ To include an expiry pass ``expire_after=`` to ``datasette.set_actor_cookie()`` .. code-block:: python response = Response.redirect("/") - datasette.set_actor_cookie( - response, {"id": "cleopaws"}, expire_after=60 * 60 * 24 - ) + datasette.set_actor_cookie(response, {"id": "cleopaws"}, expire_after=60 * 60 * 24) The resulting cookie will encode data that looks something like this: @@ -1350,10 +1348,10 @@ Actor is allowed to alter a database table. ``table`` is the name of the table (string) -.. _actions_set_column_types: +.. _actions_set_column_type: -set-column-types ----------------- +set-column-type +--------------- Actor is allowed to set assigned column types for columns in a table. diff --git a/docs/json_api.rst b/docs/json_api.rst index 7a48a26e..48c70af6 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -946,7 +946,7 @@ Pass ``"alter": true`` to automatically add any missing columns to the existing Setting a column type ~~~~~~~~~~~~~~~~~~~~~ -To set a column type for a table column, make a ``POST`` to ``//
/-/set-column-type``. This requires the :ref:`actions_set_column_types` permission. +To set a column type for a table column, make a ``POST`` to ``//
/-/set-column-type``. This requires the :ref:`actions_set_column_type` permission. :: diff --git a/tests/test_auth.py b/tests/test_auth.py index cb77c7a2..5868a21c 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -191,7 +191,7 @@ def test_auth_create_token( "all:view-query", "database:fixtures:drop-table", "resource:fixtures:foreign_key_references:insert-row", - "resource:fixtures:facetable:set-column-types", + "resource:fixtures:facetable:set-column-type", } ) # Now try actually creating one @@ -430,7 +430,7 @@ async def test_root_with_root_enabled_gets_all_permissions(ds_client): assert ( await ds_client.ds.allowed( - action="set-column-types", + action="set-column-type", resource=TableResource("fixtures", "facetable"), actor=root_actor, ) @@ -504,9 +504,9 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client): assert ( await ds_client.ds.allowed( - action="set-column-types", + action="set-column-type", resource=TableResource("fixtures", "facetable"), actor=root_actor, ) is not True - ), "Root without root_enabled should not automatically get set-column-types" + ), "Root without root_enabled should not automatically get set-column-type" diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 008fa7cd..ec0180a7 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -170,7 +170,7 @@ async def test_get_action(ds_client): "vt", "view-table", "sct", - "set-column-types", + "set-column-type", ): action = ds.get_action(name_or_abbr) if "-" in name_or_abbr: diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 4db89a0e..f9303759 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -831,19 +831,19 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "t1"), expected_result=True, ), - # set-column-types on specific table + # set-column-type on specific table PermConfigTestCase( config={ "databases": { "perms_ds_one": { "tables": { - "t1": {"permissions": {"set-column-types": {"id": "user"}}} + "t1": {"permissions": {"set-column-type": {"id": "user"}}} } } } }, actor={"id": "user"}, - action="set-column-types", + action="set-column-type", resource=("perms_ds_one", "t1"), expected_result=True, ), From 611b8ad4631d5d390d5b2d22cda7d7d79f0531cf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 12:30:26 -0700 Subject: [PATCH 008/156] blacken-docs --- docs/authentication.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 951a65ec..a796d11e 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1212,7 +1212,9 @@ To include an expiry pass ``expire_after=`` to ``datasette.set_actor_cookie()`` .. code-block:: python response = Response.redirect("/") - datasette.set_actor_cookie(response, {"id": "cleopaws"}, expire_after=60 * 60 * 24) + datasette.set_actor_cookie( + response, {"id": "cleopaws"}, expire_after=60 * 60 * 24 + ) The resulting cookie will encode data that looks something like this: From e2a4f9abb65eb064affcf41728a5c75b82abf473 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 13:00:27 -0700 Subject: [PATCH 009/156] Fixed some broken docs/ references --- docs/authentication.rst | 2 +- docs/changelog.rst | 2 +- docs/internals.rst | 4 ++-- docs/plugin_hooks.rst | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index a796d11e..7fa3a241 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1355,7 +1355,7 @@ Actor is allowed to alter a database table. set-column-type --------------- -Actor is allowed to set assigned column types for columns in a table. +Actor is allowed to set assigned :ref:`column types ` for columns in a table. ``resource`` - ``datasette.resources.TableResource(database, table)`` ``database`` is the name of the database (string) diff --git a/docs/changelog.rst b/docs/changelog.rst index 71d63e33..3d7d8405 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -934,7 +934,7 @@ Other small fixes 0.59 (2021-10-14) ----------------- -- Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`table_configuration_columns`. (:issue:`942`) +- Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`metadata_column_descriptions`. (:issue:`942`) - New :ref:`register_commands() ` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`) - Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`) - Upgraded dependency `httpx 0.20 `__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defaults to ``False`` where it previously defaulted to ``True``. (:issue:`1488`) diff --git a/docs/internals.rst b/docs/internals.rst index 2442e687..704643cc 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -922,7 +922,7 @@ await .get_column_type(database, resource, column) ``column`` - string The name of the column. -Returns a :ref:`ColumnType ` subclass instance with ``.config`` populated for the specified column, or ``None`` if no column type is assigned. +Returns a ``ColumnType`` subclass instance with ``.config`` populated for the specified column, or ``None`` if no column type is assigned. .. code-block:: python @@ -943,7 +943,7 @@ await .get_column_types(database, resource) ``resource`` - string The name of the table or view. -Returns a dictionary mapping column names to :ref:`ColumnType ` subclass instances (with ``.config`` populated) for all columns that have assigned types on the given resource. +Returns a dictionary mapping column names to ``ColumnType`` subclass instances (with ``.config`` populated) for all columns that have assigned types on the given resource. .. code-block:: python diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 53a47334..e375707d 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -503,10 +503,10 @@ Lets you customize the display of values within table cells in the HTML table vi ``request`` - :ref:`internals_request` The current request object -``column_type`` - :ref:`ColumnType ` subclass instance or None - The :ref:`ColumnType ` subclass instance assigned to this column (with ``.config`` populated), or ``None`` if no column type is assigned. You can access ``column_type.name``, ``column_type.config``, etc. +``column_type`` - :ref:`ColumnType ` subclass instance or None + The :ref:`ColumnType ` subclass instance assigned to this column (with ``.config`` populated), or ``None`` if no column type is assigned. You can access ``column_type.name``, ``column_type.config``, etc. -If a column has a :ref:`column type ` assigned and that column type's ``render_cell`` method returns a non-``None`` value, it will take priority over this plugin hook. +If a column has a :ref:`column type ` assigned and that column type's ``render_cell`` method returns a non-``None`` value, it will take priority over this plugin hook. If your hook returns ``None``, it will be ignored. Use this to indicate that your hook is not able to custom render this particular value. @@ -999,7 +999,7 @@ The permission system then uses this query along with rules from plugins to dete register_column_types(datasette) -------------------------------- -Return a list of :ref:`ColumnType ` **subclasses** (not instances) to register custom column types. Column types define how values in specific columns are rendered, validated, and transformed. +Return a list of :ref:`ColumnType ` **subclasses** (not instances) to register custom column types. Column types define how values in specific columns are rendered, validated, and transformed. .. code-block:: python @@ -1069,7 +1069,7 @@ And the following methods, all optional: Per-column configuration is available via ``self.config`` in all methods. When a column type is looked up for a specific column (via :ref:`get_column_type ` or :ref:`get_column_types `), the returned instance has ``config`` set to the parsed JSON config dict for that column assignment, or ``None`` if no config was provided. -Column types are assigned to columns via the ``column_types`` key in :ref:`table configuration `: +Column types are assigned to columns via the :ref:`column_types ` table configuration option: .. code-block:: yaml From cb5cc0cc22786d6facb81f6385457272e21fbdde Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 13:00:27 -0700 Subject: [PATCH 010/156] Fixed some broken docs/ references, refs #2671 --- docs/authentication.rst | 2 +- docs/changelog.rst | 2 +- docs/internals.rst | 4 ++-- docs/plugin_hooks.rst | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index a796d11e..7fa3a241 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1355,7 +1355,7 @@ Actor is allowed to alter a database table. set-column-type --------------- -Actor is allowed to set assigned column types for columns in a table. +Actor is allowed to set assigned :ref:`column types ` for columns in a table. ``resource`` - ``datasette.resources.TableResource(database, table)`` ``database`` is the name of the database (string) diff --git a/docs/changelog.rst b/docs/changelog.rst index 71d63e33..3d7d8405 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -934,7 +934,7 @@ Other small fixes 0.59 (2021-10-14) ----------------- -- Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`table_configuration_columns`. (:issue:`942`) +- Columns can now have associated metadata descriptions in ``metadata.json``, see :ref:`metadata_column_descriptions`. (:issue:`942`) - New :ref:`register_commands() ` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`) - Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`) - Upgraded dependency `httpx 0.20 `__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defaults to ``False`` where it previously defaulted to ``True``. (:issue:`1488`) diff --git a/docs/internals.rst b/docs/internals.rst index 2442e687..704643cc 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -922,7 +922,7 @@ await .get_column_type(database, resource, column) ``column`` - string The name of the column. -Returns a :ref:`ColumnType ` subclass instance with ``.config`` populated for the specified column, or ``None`` if no column type is assigned. +Returns a ``ColumnType`` subclass instance with ``.config`` populated for the specified column, or ``None`` if no column type is assigned. .. code-block:: python @@ -943,7 +943,7 @@ await .get_column_types(database, resource) ``resource`` - string The name of the table or view. -Returns a dictionary mapping column names to :ref:`ColumnType ` subclass instances (with ``.config`` populated) for all columns that have assigned types on the given resource. +Returns a dictionary mapping column names to ``ColumnType`` subclass instances (with ``.config`` populated) for all columns that have assigned types on the given resource. .. code-block:: python diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 53a47334..e375707d 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -503,10 +503,10 @@ Lets you customize the display of values within table cells in the HTML table vi ``request`` - :ref:`internals_request` The current request object -``column_type`` - :ref:`ColumnType ` subclass instance or None - The :ref:`ColumnType ` subclass instance assigned to this column (with ``.config`` populated), or ``None`` if no column type is assigned. You can access ``column_type.name``, ``column_type.config``, etc. +``column_type`` - :ref:`ColumnType ` subclass instance or None + The :ref:`ColumnType ` subclass instance assigned to this column (with ``.config`` populated), or ``None`` if no column type is assigned. You can access ``column_type.name``, ``column_type.config``, etc. -If a column has a :ref:`column type ` assigned and that column type's ``render_cell`` method returns a non-``None`` value, it will take priority over this plugin hook. +If a column has a :ref:`column type ` assigned and that column type's ``render_cell`` method returns a non-``None`` value, it will take priority over this plugin hook. If your hook returns ``None``, it will be ignored. Use this to indicate that your hook is not able to custom render this particular value. @@ -999,7 +999,7 @@ The permission system then uses this query along with rules from plugins to dete register_column_types(datasette) -------------------------------- -Return a list of :ref:`ColumnType ` **subclasses** (not instances) to register custom column types. Column types define how values in specific columns are rendered, validated, and transformed. +Return a list of :ref:`ColumnType ` **subclasses** (not instances) to register custom column types. Column types define how values in specific columns are rendered, validated, and transformed. .. code-block:: python @@ -1069,7 +1069,7 @@ And the following methods, all optional: Per-column configuration is available via ``self.config`` in all methods. When a column type is looked up for a specific column (via :ref:`get_column_type ` or :ref:`get_column_types `), the returned instance has ``config`` set to the parsed JSON config dict for that column assignment, or ``None`` if no config was provided. -Column types are assigned to columns via the ``column_types`` key in :ref:`table configuration `: +Column types are assigned to columns via the :ref:`column_types ` table configuration option: .. code-block:: yaml From cb293572c4b70ef064f32673a282b113ab3fd651 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 14:03:42 -0700 Subject: [PATCH 011/156] UI for setting custom column types, refs #2671 --- datasette/static/app.css | 189 ++++++++++++++++++++++ datasette/static/table.js | 278 +++++++++++++++++++++++++++++++++ datasette/templates/table.html | 5 + datasette/views/table.py | 43 +++++ tests/test_column_types.py | 102 ++++++++++++ 5 files changed, 617 insertions(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 0a6efd4c..26717c43 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -986,6 +986,180 @@ dialog.mobile-column-actions-dialog::backdrop { color: var(--ink); } +dialog.set-column-type-dialog { + --ink: #0f0f0f; + --paper: #f5f3ef; + --muted: #6b6b6b; + --rule: #e2dfd8; + --accent: #1a56db; + --card: #ffffff; + border: none; + border-radius: var(--modal-border-radius, 0.75rem); + padding: 0; + margin: auto; + width: min(520px, calc(100vw - 32px)); + max-width: 95vw; + max-height: min(720px, calc(100vh - 32px)); + box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04)); + animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out; + overflow: hidden; + font-family: system-ui, -apple-system, sans-serif; + background: var(--card); +} + +dialog.set-column-type-dialog[open] { + display: flex; + flex-direction: column; +} + +dialog.set-column-type-dialog::backdrop { + background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5)); + backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px)); + animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out; +} + +.set-column-type-dialog .modal-header { + padding: 20px 24px 12px; + border-bottom: 1px solid var(--rule); + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + flex-shrink: 0; +} + +.set-column-type-dialog .modal-title { + font-size: 1rem; + font-weight: 600; + color: var(--ink); +} + +.set-column-type-dialog .modal-meta { + font-family: ui-monospace, monospace; + font-size: 0.7rem; + color: var(--muted); + background: var(--paper); + padding: 3px 9px; + border-radius: 20px; +} + +.set-column-type-status, +.set-column-type-empty, +.set-column-type-error { + margin: 0; + padding: 12px 24px 0; +} + +.set-column-type-status, +.set-column-type-empty { + color: var(--muted); + font-size: 0.9rem; +} + +.set-column-type-error { + color: #b91c1c; + font-size: 0.9rem; +} + +.set-column-type-options { + padding: 16px 24px 24px; + overflow-y: auto; + display: grid; + gap: 12px; +} + +.set-column-type-option { + display: grid; + grid-template-columns: auto 1fr; + gap: 12px; + align-items: start; + padding: 14px 16px; + border: 1px solid var(--rule); + border-radius: 8px; + background: #fcfbf9; + cursor: pointer; +} + +.set-column-type-option:focus-within { + border-color: var(--accent); + box-shadow: 0 0 0 3px rgba(26, 86, 219, 0.12); +} + +.set-column-type-option input { + margin-top: 3px; +} + +.set-column-type-option-content { + display: grid; + gap: 4px; +} + +.set-column-type-option-name { + font-family: ui-monospace, monospace; + font-size: 0.95rem; + color: var(--ink); +} + +.set-column-type-option-description { + color: var(--muted); + font-size: 0.9rem; +} + +.set-column-type-dialog .modal-footer { + padding: 14px 20px; + border-top: 1px solid var(--rule); + display: flex; + align-items: center; + gap: 10px; + flex-shrink: 0; + background: var(--paper); +} + +.set-column-type-dialog .footer-info { + flex: 1; + font-family: ui-monospace, monospace; + font-size: 0.68rem; + color: var(--muted); +} + +.set-column-type-dialog .btn { + border: none; + border-radius: 5px; + padding: 9px 20px; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + touch-action: manipulation; + font-family: inherit; + transition: background 0.12s; +} + +.set-column-type-dialog .btn-ghost { + background: transparent; + color: var(--muted); + border: 1px solid var(--rule); +} + +.set-column-type-dialog .btn-ghost:hover { + background: var(--rule); + color: var(--ink); +} + +.set-column-type-dialog .btn-primary { + background: var(--accent); + color: #fff; +} + +.set-column-type-dialog .btn-primary:hover { + background: #1949b8; +} + +.set-column-type-dialog .btn:disabled { + opacity: 0.65; + cursor: wait; +} + @media (max-width: 640px) { dialog.mobile-column-actions-dialog { width: 95vw; @@ -1018,6 +1192,21 @@ dialog.mobile-column-actions-dialog::backdrop { padding-left: 18px; padding-right: 18px; } + + dialog.set-column-type-dialog { + width: 95vw; + max-height: 85vh; + border-radius: 0.5rem; + } + + .set-column-type-dialog .modal-header, + .set-column-type-status, + .set-column-type-empty, + .set-column-type-error, + .set-column-type-options { + padding-left: 18px; + padding-right: 18px; + } } @media only screen and (max-width: 576px) { diff --git a/datasette/static/table.js b/datasette/static/table.js index 1e243703..e9115453 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -10,6 +10,9 @@ var DROPDOWN_ICON_SVG = ` `; +var SET_COLUMN_TYPE_DIALOG_ID = "set-column-type-dialog"; +var setColumnTypeDialogState = null; + function getParams() { return new URLSearchParams(location.search); } @@ -99,6 +102,259 @@ function getColumnTypeText(th) { return `Type: ${columnType.toUpperCase()}${notNull}`; } +function getSetColumnTypeData() { + return window._setColumnTypeData || null; +} + +function getSetColumnTypeConfig(column) { + var data = getSetColumnTypeData(); + if (!data || !data.columns) { + return null; + } + return data.columns[column] || null; +} + +function canSetColumnType() { + return !!(getSetColumnTypeData() && window.HTMLDialogElement && window.fetch); +} + +function setColumnTypeActionLabel(column) { + var columnConfig = getSetColumnTypeConfig(column); + if (!columnConfig) { + return null; + } + return columnConfig.current + ? `Custom type: ${columnConfig.current.type}` + : "Set custom type"; +} + +function createSetColumnTypeOption(value, name, description, checked) { + var label = document.createElement("label"); + label.className = "set-column-type-option"; + + var input = document.createElement("input"); + input.type = "radio"; + input.name = "set-column-type-choice"; + input.value = value; + input.checked = checked; + + var content = document.createElement("span"); + content.className = "set-column-type-option-content"; + + var title = document.createElement("span"); + title.className = "set-column-type-option-name"; + title.textContent = name; + + var detail = document.createElement("span"); + detail.className = "set-column-type-option-description"; + detail.textContent = description; + + content.appendChild(title); + content.appendChild(detail); + label.appendChild(input); + label.appendChild(content); + return label; +} + +function setSetColumnTypeDialogBusy(state, isBusy) { + state.isBusy = isBusy; + state.saveButton.disabled = isBusy; + state.cancelButton.disabled = isBusy; + Array.from( + state.optionsWrap.querySelectorAll('input[name="set-column-type-choice"]'), + ).forEach(function (input) { + input.disabled = isBusy; + }); + state.saveButton.textContent = isBusy ? "Saving..." : "Save"; +} + +function clearSetColumnTypeDialogError(state) { + state.error.hidden = true; + state.error.textContent = ""; +} + +function showSetColumnTypeDialogError(state, message) { + state.error.hidden = false; + state.error.textContent = message; +} + +function ensureSetColumnTypeDialog() { + if (setColumnTypeDialogState) { + return setColumnTypeDialogState; + } + if (!window.HTMLDialogElement) { + return null; + } + + var dialog = document.createElement("dialog"); + dialog.id = SET_COLUMN_TYPE_DIALOG_ID; + dialog.className = "set-column-type-dialog"; + dialog.setAttribute("aria-labelledby", "set-column-type-title"); + dialog.innerHTML = ` +
+ Set custom type + +
+

+ +
+
+ + + +
+ `; + document.body.appendChild(dialog); + + setColumnTypeDialogState = { + dialog: dialog, + meta: dialog.querySelector(".modal-meta"), + status: dialog.querySelector(".set-column-type-status"), + error: dialog.querySelector(".set-column-type-error"), + optionsWrap: dialog.querySelector(".set-column-type-options"), + footerInfo: dialog.querySelector(".footer-info"), + cancelButton: dialog.querySelector(".set-column-type-cancel"), + saveButton: dialog.querySelector(".set-column-type-save"), + currentColumn: null, + currentConfig: null, + isBusy: false, + }; + + setColumnTypeDialogState.cancelButton.addEventListener("click", function () { + if (!setColumnTypeDialogState.isBusy) { + dialog.close(); + } + }); + + dialog.addEventListener("click", function (ev) { + if (ev.target === dialog && !setColumnTypeDialogState.isBusy) { + dialog.close(); + } + }); + + dialog.addEventListener("cancel", function (ev) { + if (setColumnTypeDialogState.isBusy) { + ev.preventDefault(); + } + }); + + dialog.addEventListener("close", function () { + clearSetColumnTypeDialogError(setColumnTypeDialogState); + setSetColumnTypeDialogBusy(setColumnTypeDialogState, false); + }); + + setColumnTypeDialogState.saveButton.addEventListener("click", async function () { + var state = setColumnTypeDialogState; + var selected = state.dialog.querySelector( + 'input[name="set-column-type-choice"]:checked', + ); + var selectedType = selected ? selected.value : ""; + var currentType = state.currentConfig.current + ? state.currentConfig.current.type + : ""; + + if (selectedType === currentType) { + state.dialog.close(); + return; + } + + clearSetColumnTypeDialogError(state); + setSetColumnTypeDialogBusy(state, true); + + var payload = { + column: state.currentColumn, + column_type: selectedType ? { type: selectedType } : null, + }; + + try { + var response = await fetch(getSetColumnTypeData().path, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(payload), + }); + var data = await response.json(); + if (!response.ok || data.ok === false) { + var message = (data.errors || ["Request failed"]).join(" "); + throw new Error(message); + } + location.reload(); + } catch (error) { + setSetColumnTypeDialogBusy(state, false); + showSetColumnTypeDialogError(state, error.message || "Request failed"); + } + }); + + return setColumnTypeDialogState; +} + +function openSetColumnTypeDialog(th) { + var column = th.dataset.column; + var columnConfig = getSetColumnTypeConfig(column); + if (!columnConfig) { + return; + } + + var state = ensureSetColumnTypeDialog(); + if (!state) { + return; + } + + clearSetColumnTypeDialogError(state); + setSetColumnTypeDialogBusy(state, false); + state.currentColumn = column; + state.currentConfig = columnConfig; + state.status.textContent = `Column: ${column}`; + state.meta.textContent = getColumnTypeText(th) || "Type unavailable"; + state.footerInfo.textContent = columnConfig.current + ? `Current custom type: ${columnConfig.current.type}` + : "No custom type set."; + state.optionsWrap.innerHTML = ""; + + var currentType = columnConfig.current ? columnConfig.current.type : ""; + state.optionsWrap.appendChild( + createSetColumnTypeOption( + "", + "No custom type", + "Use standard Datasette rendering without a custom type.", + currentType === "", + ), + ); + + columnConfig.options.forEach(function (option) { + state.optionsWrap.appendChild( + createSetColumnTypeOption( + option.name, + option.name, + option.description, + option.name === currentType, + ), + ); + }); + + if (!columnConfig.options.length) { + var emptyState = document.createElement("p"); + emptyState.className = "set-column-type-empty"; + emptyState.textContent = + "No registered custom types are compatible with this SQLite type."; + state.optionsWrap.appendChild(emptyState); + } + + if (!state.dialog.open) { + state.dialog.showModal(); + } + var selectedOption = state.dialog.querySelector( + 'input[name="set-column-type-choice"]:checked', + ); + if (selectedOption) { + selectedOption.focus(); + } else { + state.saveButton.focus(); + } +} + function canChooseColumns() { return !!( document.querySelector("column-chooser") && window._columnChooserData @@ -171,6 +427,21 @@ function buildColumnActionItems(manager, th, options) { }); } + if (canSetColumnType() && getSetColumnTypeConfig(column)) { + columnActions.push({ + label: setColumnTypeActionLabel(column), + href: "#", + onClick: + options.onSetColumnType || + function (ev) { + ev.preventDefault(); + window.setTimeout(function () { + openSetColumnTypeDialog(th); + }, 0); + }, + }); + } + if (th.dataset.isPk !== "1" && hasMultipleVisibleColumns(manager)) { columnActions.push({ label: "Hide this column", @@ -281,6 +552,13 @@ const initDatasetteTable = function (manager) { closeMenu(); openColumnChooser(); }, + onSetColumnType: function (ev) { + ev.preventDefault(); + closeMenu(); + window.setTimeout(function () { + openSetColumnTypeDialog(th); + }, 0); + }, }); var menuList = menu.querySelector("ul.dropdown-actions"); menuList.innerHTML = ""; diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 0df08a94..2919d306 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -154,6 +154,11 @@ window._columnChooserData = {{ {"allColumns": all_columns, "selectedColumns": display_columns|map(attribute='name')|list, "primaryKeys": primary_keys}|tojson }}; {% endif %} +{% if set_column_type_ui %} + +{% endif %} {% include custom_table_templates %} diff --git a/datasette/views/table.py b/datasette/views/table.py index e7a226af..5643858d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1721,6 +1721,47 @@ async def table_view_data( for col_name, ct in ct_map.items() } + async def extra_set_column_type_ui(): + "Column type UI metadata for this table" + if is_view: + return None + + if not await datasette.allowed( + action="set-column-type", + resource=TableResource(database=database_name, table=table_name), + actor=request.actor, + ): + return None + + column_details = await datasette._get_resource_column_details( + database_name, table_name + ) + ct_map = await datasette.get_column_types(database_name, table_name) + columns = {} + for column_name, column_detail in column_details.items(): + current = ct_map.get(column_name) + columns[column_name] = { + "current": ( + {"type": current.name, "config": current.config} + if current is not None + else None + ), + "options": [ + { + "name": name, + "description": ct_cls.description, + } + for name, ct_cls in sorted(datasette._column_types.items()) + if datasette._column_type_is_applicable(ct_cls, column_detail) + ], + } + return { + "path": "{}/-/set-column-type".format( + datasette.urls.table(database_name, table_name) + ), + "columns": columns, + } + async def extra_metadata(): "Metadata about the table and database" tablemetadata = await datasette.get_resource_metadata(database_name, table_name) @@ -1903,6 +1944,7 @@ async def table_view_data( "all_columns", "expandable_columns", "form_hidden_args", + "set_column_type_ui", ] } @@ -1931,6 +1973,7 @@ async def table_view_data( extra_request, extra_query, extra_column_types, + extra_set_column_type_ui, extra_metadata, extra_extras, extra_database, diff --git a/tests/test_column_types.py b/tests/test_column_types.py index 4fd30812..68b92a39 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -1,5 +1,7 @@ +import json import logging +from bs4 import BeautifulSoup as Soup from datasette.app import Datasette from datasette.column_types import ( ColumnType, @@ -56,6 +58,49 @@ def ds_ct(tmp_path_factory): database.close() +@pytest.fixture +def ds_ct_editor_permission(tmp_path_factory): + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "data.db") + db = sqlite3.connect(str(db_path)) + db.execute("vacuum") + db.execute( + "create table posts (id integer primary key, title text, body text, " + "author_email text, website text, metadata text)" + ) + db.execute( + "insert into posts values (1, 'Hello', '# World', 'test@example.com', " + "'https://example.com', '{\"key\": \"value\"}')" + ) + db.commit() + ds = Datasette( + [db_path], + config={ + "databases": { + "data": { + "tables": { + "posts": { + "permissions": {"set-column-type": {"id": "editor"}}, + "column_types": { + "body": "markdown", + "author_email": "email", + "website": "url", + "metadata": "json", + }, + } + } + } + } + }, + ) + ds.root_enabled = True + yield ds + db.close() + for database in ds.databases.values(): + if not database.is_memory: + database.close() + + def write_token(ds, actor_id="root", permissions=None): to_sign = {"a": actor_id, "token": "dstok", "t": int(time.time())} if permissions: @@ -70,6 +115,19 @@ def _headers(token): } +def _window_data_from_html(html, variable_name): + soup = Soup(html, "html.parser") + scripts = soup.find_all("script") + matching_scripts = [ + script for script in scripts if variable_name in (script.string or "") + ] + assert len(matching_scripts) == 1 + script_text = matching_scripts[0].string.strip() + prefix = f"window.{variable_name} = " + assert script_text.startswith(prefix) + return json.loads(script_text[len(prefix) :].rstrip(";")) + + # --- Internal DB and config loading --- @@ -860,6 +918,50 @@ async def test_html_table_page_rendering(ds_ct): assert 'href="https://example.com"' in html +@pytest.mark.asyncio +async def test_set_column_type_ui_data_hidden_without_permission(ds_ct): + await ds_ct.invoke_startup() + response = await ds_ct.client.get("/data/posts") + assert response.status_code == 200 + assert "window._setColumnTypeData" not in response.text + + +@pytest.mark.asyncio +async def test_set_column_type_ui_data_includes_applicable_types( + ds_ct_editor_permission, +): + await ds_ct_editor_permission.invoke_startup() + response = await ds_ct_editor_permission.client.get( + "/data/posts", + cookies={ + "ds_actor": ds_ct_editor_permission.client.actor_cookie({"id": "editor"}) + }, + ) + assert response.status_code == 200 + data = _window_data_from_html(response.text, "_setColumnTypeData") + assert data["path"] == "/data/posts/-/set-column-type" + assert data["columns"]["id"] == { + "current": None, + "options": [], + } + assert data["columns"]["title"] == { + "current": None, + "options": [ + {"name": "email", "description": "Email address"}, + {"name": "json", "description": "JSON data"}, + {"name": "url", "description": "URL"}, + ], + } + assert data["columns"]["author_email"] == { + "current": {"type": "email", "config": None}, + "options": [ + {"name": "email", "description": "Email address"}, + {"name": "json", "description": "JSON data"}, + {"name": "url", "description": "URL"}, + ], + } + + # --- Validation on upsert --- From c673ee981900284df078e5b6e933ef102e8c1008 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 15:07:07 -0700 Subject: [PATCH 012/156] Update docs for async def resources_sql(cls, datasette, actor=None) signature --- datasette/permissions.py | 2 +- docs/plugin_hooks.rst | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/datasette/permissions.py b/datasette/permissions.py index b868d025..917c58ab 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -105,7 +105,7 @@ class Resource(ABC): @classmethod @abstractmethod - def resources_sql(cls, datasette, actor=None) -> str: + async def resources_sql(cls, datasette, actor=None) -> str: """ Return SQL query that returns all resources of this type. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index e375707d..fdc392cb 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -890,13 +890,15 @@ Actions define what operations can be performed on resources (like viewing a tab """A collection of documents.""" name = "document-collection" - parent_name = None + parent_class = None def __init__(self, collection: str): super().__init__(parent=collection, child=None) @classmethod - def resources_sql(cls) -> str: + async def resources_sql( + cls, datasette, actor=None + ) -> str: return """ SELECT collection_name AS parent, NULL AS child FROM document_collections @@ -907,13 +909,15 @@ Actions define what operations can be performed on resources (like viewing a tab """A document in a collection.""" name = "document" - parent_name = "document-collection" + parent_class = DocumentCollectionResource def __init__(self, collection: str, document: str): super().__init__(parent=collection, child=document) @classmethod - def resources_sql(cls) -> str: + async def resources_sql( + cls, datasette, actor=None + ) -> str: return """ SELECT collection_name AS parent, document_id AS child FROM documents @@ -959,13 +963,15 @@ The fields of the ``Action`` dataclass are as follows: - Define a ``name`` class attribute (e.g., ``"document"``) - Define a ``parent_class`` class attribute (``None`` for top-level resources like databases, or the parent ``Resource`` subclass for child resources) - - Implement a ``resources_sql()`` classmethod that returns SQL returning all resources as ``(parent, child)`` columns + - Implement an async ``resources_sql(cls, datasette, actor=None)`` classmethod that returns SQL returning all resources as ``(parent, child)`` columns - Have an ``__init__`` method that accepts appropriate parameters and calls ``super().__init__(parent=..., child=...)`` -The ``resources_sql()`` method -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _plugin_resources_sql: -The ``resources_sql()`` classmethod returns a SQL query that lists all resources of that type that exist in the system. +The ``resources_sql(datasette, actor)`` method +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``resources_sql()`` classmethod returns a SQL query that lists all resources of that type that exist in the system. It can be async because Datasette calls it with ``await``, and it receives the current ``datasette`` instance plus an optional ``actor`` argument. This query is used by Datasette to efficiently check permissions across multiple resources at once. When a user requests a list of resources (like tables, documents, or other entities), Datasette uses this SQL to: @@ -984,7 +990,7 @@ For example, if you're building a document management plugin with collections an .. code-block:: python @classmethod - def resources_sql(cls) -> str: + async def resources_sql(cls, datasette, actor=None) -> str: return """ SELECT collection_name AS parent, document_id AS child FROM documents From 4fcf474088faaa6d4e915d1b872683f01cad4822 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 18 Mar 2026 15:13:37 -0700 Subject: [PATCH 013/156] Release 1.0a26 Refs #1592, #2661, #2664, #2666, #2669, #2670, #2671, #2672 --- datasette/version.py | 2 +- docs/changelog.rst | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index 2907e537..add192f6 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a25" +__version__ = "1.0a26" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3d7d8405..adb62d42 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,37 @@ Changelog ========= +.. _v1_0_a26: + +1.0a26 (2026-03-18) +------------------- + +New ``column_types`` system +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Table columns can now have custom column types assigned to them, using the new ``column_types`` table configuration option or at runtime using a new UI and ``POST //
/-/set-column-type`` JSON API. + +Built-in column types include ``url``, ``email``, and ``json``, and plugins can register additional types using the new :ref:`register_column_types() ` plugin hook. (:issue:`2664`, :issue:`2671`) + +Column types can customize HTML rendering, validate values written through the insert, update, and upsert APIs, and transform values returned by the JSON API. They can optionally restrict themselves to specific SQLite column types using ``sqlite_types``. This feature also introduces a new :ref:`set-column-type ` permission for assigning column types to a table. (:issue:`2672`) + +The :ref:`render_cell() ` plugin hook now receives a ``column_type`` argument containing the assigned type instance, and a column type's own ``render_cell()`` method takes priority over the plugin hook chain. + +The `datasette-files `__ plugin will be the first to use this new feature. + +UI for selecting columns and their order +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Table and view pages now include a dialog for selecting and re-ordering visible columns. (:issue:`2661`) + +Other changes +~~~~~~~~~~~~~ + +- Fixed ``allowed_resources("view-query", actor)`` so actor-specific canned queries are returned correctly. Any plugin that defines a ``resources_sql()`` method on a ``Resource`` subclass needs to update to the new signature, see :ref:`the resources_sql() method` documentation for details. +- Column actions can now be accessed in mobile view via a new "Column actions" button. Previously they were not available on mobile because table headers are not displayed there. (:issue:`2669`, :issue:`2670`) +- Row pages now render foreign key values as links to the referenced row. (:issue:`1592`) +- The ``startup()`` plugin hook now fires after metadata and internal schema tables have been populated, so plugins can reliably inspect that state during startup. (:issue:`2666`) + .. _v1_0_a25: 1.0a25 (2026-02-25) @@ -437,7 +468,7 @@ Configuration - The ``-s/--setting`` option can now be used to set plugin configuration as well. See :ref:`configuration_cli` for details. (:issue:`2252`) The above YAML configuration example using ``-s/--setting`` looks like this: - + .. code-block:: bash datasette mydatabase.db \ From c479e7dec90521a02033cc386b23af8bc0c9463d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Mar 2026 10:44:10 -0700 Subject: [PATCH 014/156] Document call_with_supported_arguments as a supported public API (#2678) * Document call_with_supported_arguments as a supported public API Mark both call_with_supported_arguments and async_call_with_supported_arguments with the @documented decorator and add documentation to docs/internals.rst so plugin authors can use these dependency injection utilities in their own code. https://claude.ai/code/session_01DKogZpHwzCTrbeG4XjXmNc --- datasette/utils/__init__.py | 23 +++++++++++++++++++++++ docs/internals.rst | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index c6973d06..7fb81f02 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1086,12 +1086,35 @@ def _gather_arguments(fn, kwargs): return call_with +@documented def call_with_supported_arguments(fn, **kwargs): + """ + Call ``fn`` with the subset of ``**kwargs`` matching its signature. + + This implements dependency injection: the caller provides all available + keyword arguments and the function receives only the ones it declares + as parameters. + + :param fn: A callable (sync function) + :param kwargs: All available keyword arguments + :returns: The return value of ``fn`` + """ call_with = _gather_arguments(fn, kwargs) return fn(*call_with) +@documented async def async_call_with_supported_arguments(fn, **kwargs): + """ + Async version of :func:`call_with_supported_arguments`. + + Calls ``await fn(...)`` with the subset of ``**kwargs`` matching its + signature. + + :param fn: An async callable + :param kwargs: All available keyword arguments + :returns: The return value of ``await fn(...)`` + """ call_with = _gather_arguments(fn, kwargs) return await fn(*call_with) diff --git a/docs/internals.rst b/docs/internals.rst index 704643cc..829b4dd4 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2115,6 +2115,43 @@ Note that the space character is a special case: it will be replaced with a ``+` .. autofunction:: datasette.utils.tilde_decode +.. _internals_utils_call_with_supported_arguments: + +call_with_supported_arguments(fn, **kwargs) +------------------------------------------- + +Call ``fn``, passing it only those keyword arguments that match its function signature. This implements a dependency injection pattern - the caller provides all available arguments, and the function receives only the ones it declares as parameters. + +This is useful in plugins that want to define callback functions that only declare the arguments they need. For example: + +.. code-block:: python + + from datasette.utils import call_with_supported_arguments + + + def my_callback(request, datasette): ... + + + # This will pass only request and datasette, ignoring other kwargs: + call_with_supported_arguments( + my_callback, + request=request, + datasette=datasette, + database=database, + table=table, + ) + +.. autofunction:: datasette.utils.call_with_supported_arguments + +.. _internals_utils_async_call_with_supported_arguments: + +await async_call_with_supported_arguments(fn, **kwargs) +------------------------------------------------------- + +Async version of :ref:`call_with_supported_arguments `. Use this for ``async def`` callback functions. + +.. autofunction:: datasette.utils.async_call_with_supported_arguments + .. _internals_tracer: datasette.tracer From 1a64d5e55e92a979e68062534407fc38d7f7b65d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:54:34 -0700 Subject: [PATCH 015/156] Bump picomatch from 2.3.1 to 2.3.2 (#2679) Bumps [picomatch](https://github.com/micromatch/picomatch) from 2.3.1 to 2.3.2. - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/2.3.1...2.3.2) --- updated-dependencies: - dependency-name: picomatch dependency-version: 2.3.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 35709001..271b3343 100644 --- a/package-lock.json +++ b/package-lock.json @@ -380,9 +380,9 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "engines": { "node": ">=8.6" }, @@ -776,9 +776,9 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==" }, "prettier": { "version": "3.6.2", From 9b5cb1347c62a5c6759a2f71565c22d4a96bf1d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 10:54:48 -0700 Subject: [PATCH 016/156] Bump rollup from 3.29.5 to 3.30.0 (#2651) Bumps [rollup](https://github.com/rollup/rollup) from 3.29.5 to 3.30.0. - [Release notes](https://github.com/rollup/rollup/releases) - [Changelog](https://github.com/rollup/rollup/blob/v3.30.0/CHANGELOG.md) - [Commits](https://github.com/rollup/rollup/compare/v3.29.5...v3.30.0) --- updated-dependencies: - dependency-name: rollup dependency-version: 3.30.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 271b3343..213999c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "@rollup/plugin-node-resolve": "^15.0.1", "@rollup/plugin-terser": "^0.1.0", "codemirror": "^6.0.1", - "rollup": "^3.29.5" + "rollup": "^3.30.0" }, "devDependencies": { "prettier": "^3.0.0" @@ -423,9 +423,9 @@ } }, "node_modules/rollup": { - "version": "3.29.5", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.29.5.tgz", - "integrity": "sha512-GVsDdsbJzzy4S/v3dqWPJ7EfvZJfCHiDqe80IyrF59LYuP+e6U1LJoUqeuqRbwAWoMNoXivMNeNAOf5E22VA1w==", + "version": "3.30.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.30.0.tgz", + "integrity": "sha512-kQvGasUgN+AlWGliFn2POSajRQEsULVYFGTvOZmK06d7vCD+YhZztt70kGk3qaeAXeWYL5eO7zx+rAubBc55eA==", "bin": { "rollup": "dist/bin/rollup" }, @@ -797,9 +797,9 @@ } }, "rollup": { - "version": "3.29.5", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.29.5.tgz", - "integrity": "sha512-GVsDdsbJzzy4S/v3dqWPJ7EfvZJfCHiDqe80IyrF59LYuP+e6U1LJoUqeuqRbwAWoMNoXivMNeNAOf5E22VA1w==", + "version": "3.30.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.30.0.tgz", + "integrity": "sha512-kQvGasUgN+AlWGliFn2POSajRQEsULVYFGTvOZmK06d7vCD+YhZztt70kGk3qaeAXeWYL5eO7zx+rAubBc55eA==", "requires": { "fsevents": "~2.3.2" } diff --git a/package.json b/package.json index 16453896..27abd0cd 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,6 @@ "@rollup/plugin-node-resolve": "^15.0.1", "@rollup/plugin-terser": "^0.1.0", "codemirror": "^6.0.1", - "rollup": "^3.29.5" + "rollup": "^3.30.0" } } From 312f41b0c28eea66c76ab8dfac11db76aaf0000a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Mar 2026 11:20:46 -0700 Subject: [PATCH 017/156] RenameTableEvent, plus write connection track_event() mechanism (#2682) * Add track_event callback to execute_write_fn and write_wrapper Allows write functions and write_wrapper generators to queue events during a write operation that are dispatched after successful commit. The fn or wrapper can optionally accept a `track_event` parameter (detected via call_with_supported_arguments). Events are discarded if the write raises an exception. Does not yet handle the block=False (non-blocking) case - events queued during non-blocking writes are currently silently discarded. Refs https://github.com/simonw/datasette/issues/2681 * Dispatch track_event events for non-blocking (block=False) writes Spawns a background asyncio task that awaits the write thread's reply queue and dispatches pending events after a successful non-blocking write. Events are still discarded if the write raises an exception. Refs https://github.com/simonw/datasette/issues/2681 * Warn that events won't fire for other processes Refs https://github.com/simonw/datasette/issues/2681#issuecomment-4157118662 --- datasette/database.py | 67 ++++++--- datasette/events.py | 58 ++++++++ datasette/hookspecs.py | 18 ++- docs/events.md | 2 + docs/internals.rst | 30 ++++ docs/plugin_hooks.rst | 4 +- docs/plugins.rst | 3 +- tests/test_write_wrapper.py | 265 ++++++++++++++++++++++++++++++++++++ 8 files changed, 423 insertions(+), 24 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index fcf69c7f..ffbbebba 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -10,6 +10,7 @@ import uuid from .tracer import trace from .utils import ( + call_with_supported_arguments, detect_fts, detect_primary_keys, detect_spatialite, @@ -190,7 +191,12 @@ class Database: return await self._send_to_write_thread(fn, isolated_connection=True) async def execute_write_fn(self, fn, block=True, transaction=True, request=None): - fn = self._wrap_fn_with_hooks(fn, request, transaction) + pending_events = [] + + def track_event(event): + pending_events.append(event) + + fn = self._wrap_fn_with_hooks(fn, request, transaction, track_event) if self.ds.executor is None: # non-threaded mode if self._write_connection is None: @@ -198,17 +204,44 @@ class Database: self.ds._prepare_connection(self._write_connection, self.name) if transaction: with self._write_connection: - return fn(self._write_connection) + result = fn(self._write_connection) else: - return fn(self._write_connection) + result = fn(self._write_connection) else: - return await self._send_to_write_thread( + result = await self._send_to_write_thread( fn, block=block, transaction=transaction ) + if block: + for event in pending_events: + await self.ds.track_event(event) + else: + # For non-blocking writes, spawn a background task to + # dispatch events after the write thread completes + task_id, reply_queue = result - def _wrap_fn_with_hooks(self, fn, request, transaction): + async def _dispatch_events_after_write(): + write_result = await reply_queue.async_q.get() + if not isinstance(write_result, Exception): + for event in pending_events: + await self.ds.track_event(event) + + asyncio.ensure_future(_dispatch_events_after_write()) + result = task_id + return result + + def _wrap_fn_with_hooks(self, fn, request, transaction, track_event): from .plugins import pm + # Wrap fn so it receives track_event if its signature supports it + original_fn = fn + + def fn_with_track_event(conn): + return call_with_supported_arguments( + original_fn, conn=conn, track_event=track_event + ) + + fn = fn_with_track_event + wrappers = pm.hook.write_wrapper( datasette=self.ds, database=self.name, @@ -220,10 +253,9 @@ class Database: return fn # Build the wrapped fn by nesting context manager generators. # The first wrapper returned by pluggy is outermost. - original_fn = fn for wrapper_factory in reversed(wrappers): - original_fn = _apply_write_wrapper(original_fn, wrapper_factory) - return original_fn + fn = _apply_write_wrapper(fn, wrapper_factory, track_event) + return fn async def _send_to_write_thread( self, fn, block=True, isolated_connection=False, transaction=True @@ -250,7 +282,7 @@ class Database: else: return result else: - return task_id + return task_id, reply_queue def _execute_writes(self): # Infinite looping thread that protects the single write connection @@ -682,18 +714,21 @@ class Database: return f"" -def _apply_write_wrapper(fn, wrapper_factory): +def _apply_write_wrapper(fn, wrapper_factory, track_event): """Apply a single write_wrapper context manager around fn. - ``wrapper_factory`` is a callable that takes ``(conn)`` and returns a - generator that yields exactly once. Code before the yield runs before - ``fn(conn)``, code after the yield runs after. The result of - ``fn(conn)`` is sent into the generator via ``.send()``, and any - exception raised by ``fn(conn)`` is thrown via ``.throw()``. + ``wrapper_factory`` is a callable that takes ``(conn)`` and optionally + ``track_event``, and returns a generator that yields exactly once. + Code before the yield runs before ``fn(conn)``, code after the yield + runs after. The result of ``fn(conn)`` is sent into the generator + via ``.send()``, and any exception raised by ``fn(conn)`` is thrown + via ``.throw()``. """ def wrapped(conn): - gen = wrapper_factory(conn) + gen = call_with_supported_arguments( + wrapper_factory, conn=conn, track_event=track_event + ) # Advance to the yield point (run "before" code) try: next(gen) diff --git a/datasette/events.py b/datasette/events.py index 5cd5ba3d..e8786da9 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -199,6 +199,27 @@ class UpdateRowEvent(Event): pks: list +@dataclass +class RenameTableEvent(Event): + """ + Event name: ``rename-table`` + + A table has been renamed. + + :ivar database: The name of the database containing the renamed table. + :type database: str + :ivar old_table: The previous name of the table. + :type old_table: str + :ivar new_table: The new name of the table. + :type new_table: str + """ + + name = "rename-table" + database: str + old_table: str + new_table: str + + @dataclass class DeleteRowEvent(Event): """ @@ -219,6 +240,42 @@ class DeleteRowEvent(Event): pks: list +@hookimpl +def write_wrapper(datasette, database, request, transaction): + def wrapper(conn, track_event): + # Snapshot rootpage -> name before the write + before = { + row[1]: row[0] + for row in conn.execute( + "select name, rootpage from sqlite_master" + " where type='table' and rootpage != 0" + ).fetchall() + } + yield + # Snapshot rootpage -> name after the write + after = { + row[1]: row[0] + for row in conn.execute( + "select name, rootpage from sqlite_master" + " where type='table' and rootpage != 0" + ).fetchall() + } + # Detect renames: same rootpage, different name + for rootpage, old_name in before.items(): + new_name = after.get(rootpage) + if new_name and new_name != old_name: + track_event( + RenameTableEvent( + actor=request.actor if request else None, + database=database, + old_table=old_name, + new_table=new_name, + ) + ) + + return wrapper + + @hookimpl def register_events(): return [ @@ -227,6 +284,7 @@ def register_events(): CreateTableEvent, CreateTokenEvent, AlterTableEvent, + RenameTableEvent, DropTableEvent, InsertRowsEvent, UpsertRowsEvent, diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 2ab9d0c5..7af9cbce 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -246,12 +246,18 @@ def register_token_handler(datasette): def write_wrapper(datasette, database, request, transaction): """Called when a write function is about to execute. - Return a generator function that accepts a ``conn`` argument. - The generator should ``yield`` exactly once: code before the - ``yield`` runs before the write, code after the ``yield`` runs - after the write completes. The result of the write is sent - back through the ``yield``, so you can capture it with - ``result = yield``. + Return a generator function that accepts a ``conn`` argument and + optionally a ``track_event`` argument. The generator should + ``yield`` exactly once: code before the ``yield`` runs before + the write, code after the ``yield`` runs after the write + completes. The result of the write is sent back through the + ``yield``, so you can capture it with ``result = yield``. + + If your generator accepts ``track_event``, you can call + ``track_event(event)`` to queue an event that will be dispatched + via ``datasette.track_event()`` after the write commits + successfully. Events are discarded if the write raises an + exception. If the write raises an exception, it is thrown into the generator so you can handle it with a try/except around the ``yield``. diff --git a/docs/events.md b/docs/events.md index 399317e9..f63d1893 100644 --- a/docs/events.md +++ b/docs/events.md @@ -5,6 +5,8 @@ Datasette includes a mechanism for tracking events that occur while the software The core Datasette application triggers events when certain things happen. This page describes those events. +Note that these events will *not* fire for changes made to a SQLite database by a process other than Datasette itself. + Plugins can listen for events using the {ref}`plugin_hook_track_event` plugin hook, which will be called with instances of the following classes - or additional classes {ref}`registered by other plugins `. ```{eval-rst} diff --git a/docs/internals.rst b/docs/internals.rst index 829b4dd4..3b65d57a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1739,6 +1739,36 @@ For example: except Exception as e: print("An error occurred:", e) +Your function can optionally accept a ``track_event`` parameter in addition to ``conn``. If it does, it will be passed a callable that can be used to queue events for dispatch after the write transaction commits successfully. Events queued this way are discarded if the write raises an exception. + +.. code-block:: python + + from datasette.events import AlterTableEvent + + + def my_write(conn, track_event): + before_schema = conn.execute( + "select sql from sqlite_master where name = 'my_table'" + ).fetchone()[0] + conn.execute( + "alter table my_table add column new_col text" + ) + after_schema = conn.execute( + "select sql from sqlite_master where name = 'my_table'" + ).fetchone()[0] + track_event( + AlterTableEvent( + actor=None, + database="mydb", + table="my_table", + before_schema=before_schema, + after_schema=after_schema, + ) + ) + + + await database.execute_write_fn(my_write) + The value returned from ``await database.execute_write_fn(...)`` will be the return value from your function. If your function raises an exception that exception will be propagated up to the ``await`` line. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index fdc392cb..79b3e669 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -78,12 +78,14 @@ write_wrapper(datasette, database, request, transaction) ``transaction`` - bool ``True`` if the write will be wrapped in a database transaction. -Return a generator function that accepts a ``conn`` argument (a SQLite connection object). The generator should ``yield`` exactly once. Code before the ``yield`` runs before the write function executes; code after the ``yield`` runs after it completes. +Return a generator function that accepts a ``conn`` argument (a SQLite connection object) and optionally a ``track_event`` argument. The generator should ``yield`` exactly once. Code before the ``yield`` runs before the write function executes; code after the ``yield`` runs after it completes. The result of the write function is sent back through the ``yield``, so you can capture it with ``result = yield``. If the write function raises an exception, it is thrown into the generator so you can handle it with a ``try`` / ``except`` around the ``yield``. +If your generator accepts ``track_event``, you can call ``track_event(event)`` to queue an event that will be dispatched via :ref:`datasette.track_event() ` after the write commits successfully. Events are discarded if the write raises an exception. + Return ``None`` to skip wrapping for this particular write. This example logs every write operation: diff --git a/docs/plugins.rst b/docs/plugins.rst index 03cbedeb..eb7b06e1 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -261,7 +261,8 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "templates": false, "version": null, "hooks": [ - "register_events" + "register_events", + "write_wrapper" ] }, { diff --git a/tests/test_write_wrapper.py b/tests/test_write_wrapper.py index 55e0461e..c2ceb344 100644 --- a/tests/test_write_wrapper.py +++ b/tests/test_write_wrapper.py @@ -2,7 +2,9 @@ Tests for the write_wrapper plugin hook. """ +from dataclasses import dataclass from datasette.app import Datasette +from datasette.events import Event from datasette.hookspecs import hookimpl from datasette.plugins import pm import pytest @@ -10,6 +12,12 @@ import sqlite3 import time +@dataclass +class DummyEvent(Event): + name = "dummy" + message: str + + @pytest.fixture def datasette(tmp_path): db_path = str(tmp_path / "test.db") @@ -477,3 +485,260 @@ async def test_write_wrapper_set_authorizer(datasette, actor, table, should_deny assert result.rows[0][0] == "test" finally: pm.unregister(name="test_set_authorizer") + + +# --- Tests for track_event callback --- + + +@pytest.fixture +def ds_with_event_tracking(tmp_path): + """Datasette instance that records tracked events and registers DummyEvent.""" + db_path = str(tmp_path / "test.db") + ds = Datasette([db_path]) + ds._tracked_events = [] + # Set event_classes directly to avoid needing invoke_startup + ds.event_classes = (DummyEvent,) + + async def recording_track_event(event): + ds._tracked_events.append(event) + + ds.track_event = recording_track_event + + yield ds + + +@pytest.mark.asyncio +async def test_track_event_in_write_fn(ds_with_event_tracking): + """fn(conn, track_event) can queue events that are dispatched after commit.""" + ds = ds_with_event_tracking + db = ds.get_database("test") + + def my_write(conn, track_event): + conn.execute("create table if not exists te1 (id integer primary key)") + track_event(DummyEvent(actor=None, message="hello")) + + await db.execute_write_fn(my_write) + assert len(ds._tracked_events) == 1 + assert ds._tracked_events[0].message == "hello" + + +@pytest.mark.asyncio +async def test_track_event_discarded_on_exception(ds_with_event_tracking): + """Events are discarded if the write fn raises an exception.""" + ds = ds_with_event_tracking + db = ds.get_database("test") + + def my_write(conn, track_event): + track_event(DummyEvent(actor=None, message="should not fire")) + raise ValueError("deliberate error") + + with pytest.raises(ValueError, match="deliberate"): + await db.execute_write_fn(my_write) + assert len(ds._tracked_events) == 0 + + +@pytest.mark.asyncio +async def test_track_event_existing_fn_signature_still_works(ds_with_event_tracking): + """Existing fn(conn) signatures continue to work without track_event.""" + ds = ds_with_event_tracking + db = ds.get_database("test") + + await db.execute_write_fn( + lambda conn: conn.execute( + "create table if not exists te2 (id integer primary key)" + ) + ) + # No events, no errors + assert len(ds._tracked_events) == 0 + + +@pytest.mark.asyncio +async def test_track_event_in_write_wrapper(ds_with_event_tracking): + """write_wrapper generator with (conn, track_event) can queue events.""" + ds = ds_with_event_tracking + db = ds.get_database("test") + + class Plugin: + __name__ = "Plugin" + + @staticmethod + @hookimpl + def write_wrapper(datasette, database, request, transaction): + def wrapper(conn, track_event): + track_event(DummyEvent(actor=None, message="from wrapper before")) + yield + track_event(DummyEvent(actor=None, message="from wrapper after")) + + return wrapper + + pm.register(Plugin(), name="test_track_wrapper") + try: + await db.execute_write_fn( + lambda conn: conn.execute( + "create table if not exists te3 (id integer primary key)" + ) + ) + assert len(ds._tracked_events) == 2 + assert ds._tracked_events[0].message == "from wrapper before" + assert ds._tracked_events[1].message == "from wrapper after" + finally: + pm.unregister(name="test_track_wrapper") + + +@pytest.mark.asyncio +async def test_track_event_shared_between_fn_and_wrapper(ds_with_event_tracking): + """Both fn and wrapper can queue events, all dispatched in order.""" + ds = ds_with_event_tracking + db = ds.get_database("test") + + class Plugin: + __name__ = "Plugin" + + @staticmethod + @hookimpl + def write_wrapper(datasette, database, request, transaction): + def wrapper(conn, track_event): + track_event(DummyEvent(actor=None, message="wrapper-before")) + yield + track_event(DummyEvent(actor=None, message="wrapper-after")) + + return wrapper + + pm.register(Plugin(), name="test_track_shared") + try: + + def my_write(conn, track_event): + conn.execute("create table if not exists te4 (id integer primary key)") + track_event(DummyEvent(actor=None, message="from-fn")) + + await db.execute_write_fn(my_write) + messages = [e.message for e in ds._tracked_events] + assert messages == ["wrapper-before", "from-fn", "wrapper-after"] + finally: + pm.unregister(name="test_track_shared") + + +@pytest.mark.asyncio +async def test_track_event_with_block_false(ds_with_event_tracking): + """Events are dispatched even when block=False (non-blocking writes).""" + ds = ds_with_event_tracking + db = ds.get_database("test") + + def my_write(conn, track_event): + conn.execute("create table if not exists te5 (id integer primary key)") + track_event(DummyEvent(actor=None, message="non-blocking")) + + task_id = await db.execute_write_fn(my_write, block=False) + assert task_id is not None + + # Give the background task time to complete + import asyncio + + for _ in range(50): + if ds._tracked_events: + break + await asyncio.sleep(0.01) + + assert len(ds._tracked_events) == 1 + assert ds._tracked_events[0].message == "non-blocking" + + +# --- Tests for RenameTableEvent detection --- + + +@pytest.fixture +def ds_for_rename(tmp_path): + """Datasette instance that records tracked events for rename detection tests.""" + from datasette.events import RenameTableEvent + + db_path = str(tmp_path / "test.db") + ds = Datasette([db_path]) + ds._tracked_events = [] + ds.event_classes = (RenameTableEvent,) + + async def recording_track_event(event): + ds._tracked_events.append(event) + + ds.track_event = recording_track_event + return ds + + +@pytest.mark.asyncio +async def test_rename_table_fires_event(ds_for_rename): + """Renaming a table via ALTER TABLE fires a RenameTableEvent.""" + from datasette.events import RenameTableEvent + + ds = ds_for_rename + db = ds.get_database("test") + + await db.execute_write("create table old_name (id integer primary key)") + + def rename(conn): + conn.execute("alter table old_name rename to new_name") + + await db.execute_write_fn(rename) + + rename_events = [e for e in ds._tracked_events if isinstance(e, RenameTableEvent)] + assert len(rename_events) == 1 + assert rename_events[0].old_table == "old_name" + assert rename_events[0].new_table == "new_name" + assert rename_events[0].database == "test" + + +@pytest.mark.asyncio +async def test_no_rename_event_for_regular_writes(ds_for_rename): + """Regular writes (CREATE, INSERT) do not fire RenameTableEvent.""" + from datasette.events import RenameTableEvent + + ds = ds_for_rename + db = ds.get_database("test") + + await db.execute_write("create table t (id integer primary key)") + await db.execute_write_fn(lambda conn: conn.execute("insert into t values (1)")) + + rename_events = [e for e in ds._tracked_events if isinstance(e, RenameTableEvent)] + assert len(rename_events) == 0 + + +@pytest.mark.asyncio +async def test_no_rename_event_on_rollback(ds_for_rename): + """RenameTableEvent is not fired if the write raises an exception.""" + from datasette.events import RenameTableEvent + + ds = ds_for_rename + db = ds.get_database("test") + + await db.execute_write("create table rollback_test (id integer primary key)") + + def rename_then_fail(conn): + conn.execute("alter table rollback_test rename to renamed") + raise ValueError("deliberate error") + + with pytest.raises(ValueError, match="deliberate"): + await db.execute_write_fn(rename_then_fail) + + rename_events = [e for e in ds._tracked_events if isinstance(e, RenameTableEvent)] + assert len(rename_events) == 0 + + +@pytest.mark.asyncio +async def test_multiple_renames_in_one_write(ds_for_rename): + """Multiple renames in a single write fire multiple RenameTableEvents.""" + from datasette.events import RenameTableEvent + + ds = ds_for_rename + db = ds.get_database("test") + + await db.execute_write("create table alpha (id integer primary key)") + await db.execute_write("create table beta (id integer primary key)") + + def rename_both(conn): + conn.execute("alter table alpha rename to alpha2") + conn.execute("alter table beta rename to beta2") + + await db.execute_write_fn(rename_both) + + rename_events = [e for e in ds._tracked_events if isinstance(e, RenameTableEvent)] + assert len(rename_events) == 2 + names = {(e.old_table, e.new_table) for e in rename_events} + assert names == {("alpha", "alpha2"), ("beta", "beta2")} From 94d14e3d3706b7919babd6d07ad56849895eaa07 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Mar 2026 16:11:06 -0700 Subject: [PATCH 018/156] Warning note about VACUUM and RenameTableEvent I noticed that VACUUM can update the rootpage for tables in a way that could confuse our rename table detection logic - but using the execute_isolated_fn() method to run VACUUM avoids this problem. Refs #2681 --- docs/internals.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index 3b65d57a..367ec223 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1788,6 +1788,8 @@ The :ref:`prepare_connection() ` plugin hook is This allows plugins to execute database operations that might conflict with how database connections are usually configured. For example, running a ``VACUUM`` operation while bypassing any restrictions placed by the `datasette-sqlite-authorizer `__ plugin. +Running ``VACUUM`` using this method also ensures it won't trigger incorrect :class:`~datasette.events.RenameTableEvent` events, since ``execute_isolated_fn()`` does not trigger the Datasette mechanism that detects renamed tables in a way that can be confused by a ``VACUUM``. + Plugins can also use this method to load potentially dangerous SQLite extensions, use them to perform an operation and then have them safely unloaded at the end of the call, without risk of exposing them to other connections. Functions run using ``execute_isolated_fn()`` share the same queue as ``execute_write_fn()``, which guarantees that no writes can be executed at the same time as the isolated function is executing. From fc1794719a99812103aa27ad5bf46b4449828642 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 30 Mar 2026 21:03:21 -0700 Subject: [PATCH 019/156] Database(is_temp_disk=True) option, used for internal database (#2684) Closes #2683 * Add is_temp_disk option to Database for temp file-backed databases Replace the default in-memory internal database with a temporary file-backed database using WAL mode. This fixes concurrent read/write locking errors that occur with named in-memory SQLite databases. The new is_temp_disk parameter on Database creates a temp file via tempfile.mkstemp, connects to it as a regular file-based database with WAL mode enabled, and cleans it up on close() and via atexit. https://claude.ai/code/session_01TteLrUjpDcARjnP1GMRqz2 --- datasette/app.py | 2 +- datasette/database.py | 37 ++++++++++++++++++++++++++++++-- docs/internals.rst | 14 ++++++++++-- tests/test_internals_database.py | 29 +++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 5 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 4c98e521..ed62c528 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -381,7 +381,7 @@ class Datasette: self.internal_db_created = False if internal is None: - self._internal_database = Database(self, memory_name=secrets.token_hex()) + self._internal_database = Database(self, is_temp_disk=True) else: self._internal_database = Database(self, path=internal, mode="rwc") self._internal_database.name = INTERNAL_DB_NAME diff --git a/datasette/database.py b/datasette/database.py index ffbbebba..8b824462 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,10 +1,13 @@ import asyncio +import atexit from collections import namedtuple +import os from pathlib import Path import janus import queue import sqlite_utils import sys +import tempfile import threading import uuid @@ -43,6 +46,7 @@ class Database: is_memory=False, memory_name=None, mode=None, + is_temp_disk=False, ): self.name = None self._thread_local_id = f"x{self._thread_local_id_counter}" @@ -53,8 +57,19 @@ class Database: self.is_mutable = is_mutable self.is_memory = is_memory self.memory_name = memory_name + self.is_temp_disk = is_temp_disk if memory_name is not None: self.is_memory = True + if is_temp_disk: + fd, temp_path = tempfile.mkstemp(suffix=".db", prefix="datasette_temp_") + os.close(fd) + self.path = temp_path + self.is_mutable = True + self.mode = "rwc" + self._wal_enabled = False + atexit.register(self._cleanup_temp_file) + else: + self._wal_enabled = False self.cached_hash = None self.cached_size = None self._cached_table_counts = None @@ -65,7 +80,8 @@ class Database: self._write_connection = None # This is used to track all file connections so they can be closed self._all_file_connections = [] - self.mode = mode + if not is_temp_disk: + self.mode = mode @property def cached_table_counts(self): @@ -86,6 +102,8 @@ class Database: return md5_not_usedforsecurity(self.name)[:6] def suggest_name(self): + if self.is_temp_disk: + return "_temp_disk" if self.path: return Path(self.path).stem elif self.memory_name: @@ -124,12 +142,25 @@ class Database: f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs ) self._all_file_connections.append(conn) + if self.is_temp_disk and not self._wal_enabled: + conn.execute("PRAGMA journal_mode=WAL") + self._wal_enabled = True return conn def close(self): # Close all connections - useful to avoid running out of file handles in tests for connection in self._all_file_connections: connection.close() + if self.is_temp_disk: + self._cleanup_temp_file() + + def _cleanup_temp_file(self): + if self.is_temp_disk and self.path: + for suffix in ("", "-wal", "-shm"): + try: + os.unlink(self.path + suffix) + except OSError: + pass async def execute_write(self, sql, params=None, block=True, request=None): def _inner(conn): @@ -405,7 +436,7 @@ class Database: def hash(self): if self.cached_hash is not None: return self.cached_hash - elif self.is_mutable or self.is_memory: + elif self.is_mutable or self.is_memory or self.is_temp_disk: return None elif self.ds.inspect_data and self.ds.inspect_data.get(self.name): self.cached_hash = self.ds.inspect_data[self.name]["hash"] @@ -704,6 +735,8 @@ class Database: tags.append("mutable") if self.is_memory: tags.append("memory") + if self.is_temp_disk: + tags.append("temp_disk") if self.hash: tags.append(f"hash={self.hash}") if self.size is not None: diff --git a/docs/internals.rst b/docs/internals.rst index 367ec223..06a6b348 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1552,8 +1552,8 @@ Instances of the ``Database`` class can be used to execute queries against attac .. _database_constructor: -Database(ds, path=None, is_mutable=True, is_memory=False, memory_name=None) ---------------------------------------------------------------------------- +Database(ds, path=None, is_mutable=True, is_memory=False, memory_name=None, is_temp_disk=False) +----------------------------------------------------------------------------------------------- The ``Database()`` constructor can be used by plugins, in conjunction with :ref:`datasette_add_database`, to create and register new databases. @@ -1574,6 +1574,13 @@ The arguments are as follows: ``memory_name`` - string or ``None`` Use this to create a named in-memory database. Unlike regular memory databases these can be accessed by multiple threads and will persist an changes made to them for the lifetime of the Datasette server process. +``is_temp_disk`` - boolean + Set this to ``True`` to create a temporary file-backed database. This creates a SQLite database in a temporary file on disk (using Python's ``tempfile.mkstemp()``) with WAL mode enabled for better concurrent read/write performance. The temporary file is automatically cleaned up when the database is closed or when the process exits. + + Unlike named in-memory databases (``memory_name``), temporary disk databases support concurrent readers and writers without locking errors, because WAL mode allows readers and writers to operate simultaneously. This makes them suitable for use cases like the internal database where concurrent access is common. + + When ``is_temp_disk=True``, the ``path``, ``is_mutable``, and ``mode`` parameters are set automatically and should not be provided. + The first argument is the ``datasette`` instance you are attaching to, the second is a ``path=``, then ``is_mutable`` and ``is_memory`` are both optional arguments. .. _database_hash: @@ -1825,6 +1832,9 @@ The ``Database`` class also provides properties and methods for introspecting th ``db.is_memory`` - boolean Is this database an in-memory database? +``db.is_temp_disk`` - boolean + Is this database a temporary file-backed database? See :ref:`database_constructor` for details. Temporary disk databases report ``hash`` as ``None`` but have real values for ``size`` and ``mtime_ns`` since they are backed by a file on disk. + ``await db.attached_databases()`` - list of named tuples Returns a list of additional databases that have been connected to this database using the SQLite ATTACH command. Each named tuple has fields ``seq``, ``name`` and ``file``. diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 5e3459cd..9a83dd4f 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -767,3 +767,32 @@ async def test_replace_database(tmpdir): db2 = datasette.get_database("data1") count = (await db2.execute("select count(*) from t")).first()[0] assert count == 1 + + +@pytest.mark.parametrize( + "kwargs,expected_repr", + [ + ({"is_memory": True}, ""), + ({"memory_name": "my_mem"}, ""), + ( + {"is_memory": True, "is_mutable": False}, + "", + ), + ], + ids=["memory", "named_memory", "immutable_memory"], +) +def test_repr(app_client, kwargs, expected_repr): + db = Database(app_client.ds, **kwargs) + db.name = "test_db" + assert repr(db) == expected_repr + + +def test_repr_temp_disk(app_client): + db = Database(app_client.ds, is_temp_disk=True) + db.name = "test_db" + r = repr(db) + assert r.startswith("") + assert isinstance(db.size, int) + assert isinstance(db.mtime_ns, int) + db.close() From 0b639a8122a702b0a99646b06e732dc666cebd66 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Apr 2026 17:11:36 -0700 Subject: [PATCH 020/156] Replace token-based CSRF with Sec-Fetch-Site header protection (#2689) - New CSRF protection middleware inspired by Go 1.25 and research by Filippo Valsorda - https://words.filippo.io/csrf/ - this replaces the old CSRF token based protection. - Removes all instances of `` in the templates - they are no longer needed. - Removes the `def skip_csrf(datasette, scope):` plugin hook defined in `datasette/hookspecs.py` and its documentation and tests. - Updated CSRF protection documentation to describe the new approach. - Upgrade guide now describes the CSRF change. --- datasette/app.py | 30 +- datasette/csrf.py | 126 ++++++++ datasette/default_permissions/__init__.py | 12 +- datasette/hookspecs.py | 5 - datasette/templates/base.html | 1 - datasette/templates/create_token.html | 1 - datasette/templates/csrf_error.html | 4 +- .../debug_permissions_playground.html | 1 - datasette/templates/logout.html | 1 - datasette/templates/messages_debug.html | 1 - datasette/templates/query.html | 1 - datasette/utils/testing.py | 11 +- docs/internals.rst | 15 +- docs/plugin_hooks.rst | 25 -- docs/plugins.rst | 3 +- docs/testing_plugins.rst | 3 +- docs/upgrade_guide.md | 60 ++++ pyproject.toml | 1 - tests/conftest.py | 11 + tests/fixtures.py | 1 - tests/plugins/my_plugin.py | 5 - tests/test_canned_queries.py | 32 +-- tests/test_csrf_middleware.py | 270 ++++++++++++++++++ tests/test_html.py | 5 +- tests/test_permissions.py | 5 - tests/test_plugins.py | 39 +-- 26 files changed, 506 insertions(+), 163 deletions(-) create mode 100644 datasette/csrf.py create mode 100644 tests/test_csrf_middleware.py diff --git a/datasette/app.py b/datasette/app.py index ed62c528..6ab32f9e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1,6 +1,5 @@ from __future__ import annotations -from asgi_csrf import Errors import asyncio import contextvars from typing import TYPE_CHECKING, Any, Dict, Iterable, List @@ -8,7 +7,6 @@ from typing import TYPE_CHECKING, Any, Dict, Iterable, List if TYPE_CHECKING: from datasette.permissions import Resource from datasette.tokens import TokenRestrictions -import asgi_csrf import collections import dataclasses import datetime @@ -120,6 +118,7 @@ from .utils.asgi import ( asgi_send_file, asgi_send_redirect, ) +from .csrf import CrossOriginProtectionMiddleware from .utils.internal_db import init_internal_db, populate_schema_tables from .utils.sqlite import ( sqlite3, @@ -2003,7 +2002,11 @@ class Datasette: "extra_js_urls", template, context, request, view_name ), "base_url": self.setting("base_url"), - "csrftoken": request.scope["csrftoken"] if request else lambda: "", + "csrftoken": ( + request.scope["csrftoken"] + if request and "csrftoken" in request.scope + else lambda: "" + ), "datasette_version": __version__, }, **extra_template_vars, @@ -2306,26 +2309,7 @@ class Datasette: if not database.is_mutable: await database.table_counts(limit=60 * 60 * 1000) - async def custom_csrf_error(scope, send, message_id): - await asgi_send( - send, - content=await self.render_template( - "csrf_error.html", - {"message_id": message_id, "message_name": Errors(message_id).name}, - ), - status=403, - content_type="text/html; charset=utf-8", - ) - - asgi = asgi_csrf.asgi_csrf( - DatasetteRouter(self, routes), - signing_secret=self._secret, - cookie_name="ds_csrftoken", - skip_if_scope=lambda scope: any( - pm.hook.skip_csrf(datasette=self, scope=scope) - ), - send_csrf_failed=custom_csrf_error, - ) + asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) asgi = AsgiLifespan(asgi) diff --git a/datasette/csrf.py b/datasette/csrf.py new file mode 100644 index 00000000..dd968a4d --- /dev/null +++ b/datasette/csrf.py @@ -0,0 +1,126 @@ +""" +Header-based CSRF (Cross-Origin) protection. + +Datasette uses the Sec-Fetch-Site + Origin header approach described in +Filippo Valsorda's article (https://words.filippo.io/csrf/) and implemented +in Go 1.25's http.CrossOriginProtection. This replaces the previous +token-based asgi-csrf mechanism. +""" + +from __future__ import annotations + +import secrets +import urllib.parse + +from .utils.asgi import asgi_send + +SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) + + +def _install_legacy_csrftoken(scope): + """ + Populate ``scope["csrftoken"]`` with a callable returning a per-request + random token. Provided for plugin compatibility only - core no longer + uses this value for CSRF enforcement. + """ + + def csrftoken(): + if "_datasette_legacy_csrftoken" not in scope: + scope["_datasette_legacy_csrftoken"] = secrets.token_urlsafe(32) + return scope["_datasette_legacy_csrftoken"] + + scope["csrftoken"] = csrftoken + + +class CrossOriginProtectionMiddleware: + """ + Modern CSRF protection using the Sec-Fetch-Site and Origin headers. + + Based on Filippo Valsorda's algorithm, as implemented in Go 1.25's + http.CrossOriginProtection. See https://words.filippo.io/csrf/ + + Unsafe-method requests are allowed through only if they look same-origin. + Non-browser clients (curl, etc.) send neither Sec-Fetch-Site nor Origin + and are passed through unchanged - CSRF is a browser-only attack. + """ + + SAFE_METHODS = SAFE_METHODS + + def __init__(self, app, datasette): + self.app = app + self.datasette = datasette + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + _install_legacy_csrftoken(scope) + + if scope.get("method", "GET") in self.SAFE_METHODS: + await self.app(scope, receive, send) + return + + headers = dict(scope.get("headers") or []) + + # Bearer-token requests are not ambient browser credentials, so they + # are not CSRF-vulnerable. Narrowly exempt them from the header check + # before evaluating Sec-Fetch-Site / Origin. Only "Bearer" is exempt; + # schemes like Basic or Digest can be browser-managed and ambient. + authorization = headers.get(b"authorization", b"").decode("latin-1") + if authorization: + scheme = authorization.split(None, 1)[0].lower() + if scheme == "bearer": + await self.app(scope, receive, send) + return + + origin_bytes = headers.get(b"origin") + sec_fetch_site_bytes = headers.get(b"sec-fetch-site") + host_bytes = headers.get(b"host", b"") + origin = origin_bytes.decode("latin-1") if origin_bytes else None + sec_fetch_site = ( + sec_fetch_site_bytes.decode("latin-1") if sec_fetch_site_bytes else None + ) + host = host_bytes.decode("latin-1") + + # Primary defense: Sec-Fetch-Site (set by browsers, unforgeable from JS) + if sec_fetch_site is not None: + if sec_fetch_site in ("same-origin", "none"): + await self.app(scope, receive, send) + return + await self._forbid( + send, + "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format( + sec_fetch_site + ), + ) + return + + # No Sec-Fetch-Site and no Origin -> non-browser client (curl, API, etc.) + if origin is None: + await self.app(scope, receive, send) + return + + # Fallback for older browsers: Origin host must match Host header + parsed = urllib.parse.urlparse(origin) + origin_host = parsed.hostname or "" + if parsed.port: + origin_host = "{}:{}".format(origin_host, parsed.port) + if origin_host == host: + await self.app(scope, receive, send) + return + + await self._forbid( + send, + "Origin {!r} does not match Host {!r}".format(origin, host), + ) + + async def _forbid(self, send, reason): + await asgi_send( + send, + content=await self.datasette.render_template( + "csrf_error.html", {"reason": reason} + ), + status=403, + content_type="text/html; charset=utf-8", + ) diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index 4ebe6147..9e3bb648 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -17,7 +17,7 @@ UNION/INTERSECT operations. The order of evaluation is: from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: from datasette.app import Datasette @@ -39,16 +39,6 @@ from .defaults import ( ) -@hookimpl -def skip_csrf(scope) -> Optional[bool]: - """Skip CSRF check for JSON content-type requests.""" - if scope["type"] == "http": - headers = scope.get("headers") or {} - if dict(headers).get(b"content-type") == b"application/json": - return True - return None - - @hookimpl def canned_queries(datasette: "Datasette", database: str, actor) -> dict: """Return canned queries defined in datasette.yaml configuration.""" diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 7af9cbce..27e20bd4 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -187,11 +187,6 @@ def homepage_actions(datasette, actor, request): """Links for the homepage actions menu""" -@hookspec -def skip_csrf(datasette, scope): - """Mechanism for skipping CSRF checks for certain requests""" - - @hookspec def handle_exception(datasette, request, exception): """Handle an uncaught exception. Can return a Response or None.""" diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 0d89e11c..21f8c693 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -38,7 +38,6 @@ {% endif %} {% if show_logout %}
- {% endif %} diff --git a/datasette/templates/create_token.html b/datasette/templates/create_token.html index ad7c71b6..270d9c86 100644 --- a/datasette/templates/create_token.html +++ b/datasette/templates/create_token.html @@ -50,7 +50,6 @@ -
diff --git a/datasette/templates/csrf_error.html b/datasette/templates/csrf_error.html index 7cd4b42b..b84749d3 100644 --- a/datasette/templates/csrf_error.html +++ b/datasette/templates/csrf_error.html @@ -1,5 +1,5 @@ {% extends "base.html" %} -{% block title %}CSRF check failed){% endblock %} +{% block title %}CSRF check failed{% endblock %} {% block content %}

Form origin check failed

@@ -7,7 +7,7 @@
Technical details

Developers: consult Datasette's CSRF protection documentation.

-

Error code is {{ message_name }}.

+

Reason: {{ reason }}

{% endblock %} diff --git a/datasette/templates/debug_permissions_playground.html b/datasette/templates/debug_permissions_playground.html index 91ce1fcf..4410a677 100644 --- a/datasette/templates/debug_permissions_playground.html +++ b/datasette/templates/debug_permissions_playground.html @@ -52,7 +52,6 @@ textarea {
-
diff --git a/datasette/templates/logout.html b/datasette/templates/logout.html index c8fc642a..a99870e6 100644 --- a/datasette/templates/logout.html +++ b/datasette/templates/logout.html @@ -10,7 +10,6 @@
-
diff --git a/datasette/templates/messages_debug.html b/datasette/templates/messages_debug.html index 2940cd69..891cf915 100644 --- a/datasette/templates/messages_debug.html +++ b/datasette/templates/messages_debug.html @@ -19,7 +19,6 @@
-
diff --git a/datasette/templates/query.html b/datasette/templates/query.html index a6e9a3aa..8b405da5 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -65,7 +65,6 @@ {% endif %}

{% if not hide_sql %}{% endif %} - {% if canned_query_write %}{% endif %} {{ show_hide_hidden }} {% if canned_query and edit_sql_url %}Edit SQL{% endif %} diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index 1606da05..de7e94af 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -95,15 +95,8 @@ class TestClient: cookies = cookies or {} post_data = post_data or {} assert not (post_data and body), "Provide one or other of body= or post_data=" - # Maybe fetch a csrftoken first - if csrftoken_from is not None: - assert body is None, "body= is not compatible with csrftoken_from=" - if csrftoken_from is True: - csrftoken_from = path - token_response = await self._request(csrftoken_from, cookies=cookies) - csrftoken = token_response.cookies["ds_csrftoken"] - cookies["ds_csrftoken"] = csrftoken - post_data["csrftoken"] = csrftoken + # csrftoken_from is accepted for backward compatibility but is now a no-op. + # Datasette no longer uses CSRF tokens - see CrossOriginProtectionMiddleware. if post_data: body = urlencode(post_data, doseq=True) return await self._request( diff --git a/docs/internals.rst b/docs/internals.rst index 06a6b348..1693a241 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1941,19 +1941,16 @@ The ``Database`` class also provides properties and methods for introspecting th CSRF protection =============== -Datasette uses `asgi-csrf `__ to guard against CSRF attacks on form POST submissions. Users receive a ``ds_csrftoken`` cookie which is compared against the ``csrftoken`` form field (or ``x-csrftoken`` HTTP header) for every incoming request. +Datasette protects against Cross-Site Request Forgery by inspecting the browser-set ``Sec-Fetch-Site`` and ``Origin`` headers on every unsafe (non-``GET``/``HEAD``/``OPTIONS``) request, following the approach described in `Filippo Valsorda's article `__ and implemented in Go 1.25's ``http.CrossOriginProtection``. -If your plugin implements a ``
`` anywhere you will need to include that token. You can do so with the following template snippet: +A request is rejected with a ``403`` response if: -.. code-block:: html +- It carries ``Sec-Fetch-Site`` with any value other than ``same-origin`` or ``none``, or +- It has no ``Sec-Fetch-Site`` header but does carry an ``Origin`` header whose host does not match the request ``Host``. - +Requests from non-browser clients (``curl``, server-to-server scripts, etc.) do not send ``Sec-Fetch-Site`` or ``Origin`` and pass through unchanged - CSRF is a browser-only attack. -If you are rendering templates using the :ref:`datasette_render_template` method the ``csrftoken()`` helper will only work if you provide the ``request=`` argument to that method. If you forget to do this you will see the following error:: - - form-urlencoded POST field did not match cookie - -You can selectively disable CSRF protection using the :ref:`plugin_hook_skip_csrf` hook. +No token, cookie, or hidden form field is needed. Any ```` inside Datasette or a plugin will be accepted from the same origin without modification. .. _internals_internal: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 79b3e669..54dde20c 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1837,31 +1837,6 @@ This example logs an error to `Sentry `__ and then renders a Example: `datasette-sentry `_ -.. _plugin_hook_skip_csrf: - -skip_csrf(datasette, scope) ---------------------------- - -``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. - -``scope`` - dictionary - The `ASGI scope `__ for the incoming HTTP request. - -This hook can be used to skip :ref:`internals_csrf` for a specific incoming request. For example, you might have a custom path at ``/submit-comment`` which is designed to accept comments from anywhere, whether or not the incoming request originated on the site and has an accompanying CSRF token. - -This example will disable CSRF protection for that specific URL path: - -.. code-block:: python - - from datasette import hookimpl - - - @hookimpl - def skip_csrf(scope): - return scope["path"] == "/submit-comment" - -If any of the currently active ``skip_csrf()`` plugin hooks return ``True``, CSRF protection will be skipped for the request. .. _plugin_hook_menu_links: diff --git a/docs/plugins.rst b/docs/plugins.rst index eb7b06e1..d9938dba 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -241,8 +241,7 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "version": null, "hooks": [ "canned_queries", - "permission_resources_sql", - "skip_csrf" + "permission_resources_sql" ] }, { diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index b0713e7c..1b10c132 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -235,9 +235,8 @@ As an example, here's a very simple plugin which executes an HTTP response and r if request.method == "GET": return Response.html(""" - - """.format(request.scope["csrftoken"]())) + """) vars = await request.post_vars() url = vars["url"] return Response.text(httpx.get(url).text) diff --git a/docs/upgrade_guide.md b/docs/upgrade_guide.md index b67eb054..33a8343b 100644 --- a/docs/upgrade_guide.md +++ b/docs/upgrade_guide.md @@ -155,3 +155,63 @@ token = await datasette.create_token( ``` The `datasette create-token` CLI command is unchanged. + +(upgrade_guide_csrf)= +### CSRF protection is now header-based + +Datasette's Cross-Site Request Forgery protection no longer uses tokens. The previous `asgi-csrf` mechanism - which set a `ds_csrftoken` cookie and required a matching `` in every form - has been replaced with an ASGI middleware that inspects the browser-set `Sec-Fetch-Site` and `Origin` headers, following the approach described in [Filippo Valsorda's research](https://words.filippo.io/csrf/) and implemented in Go 1.25's `http.CrossOriginProtection`. + +This works identically on HTTPS, HTTP, and localhost. Non-browser clients (curl, Python `requests`, server-to-server scripts) do not send `Sec-Fetch-Site` or `Origin` and are passed through unchanged - CSRF is a browser-only attack. + +Requests that carry an explicit `Authorization: Bearer ...` header are also exempt from the CSRF check, because bearer tokens are not ambient browser credentials: a malicious cross-origin page cannot cause the browser to attach a target site's bearer token unless the attacker's JavaScript already possesses it. This exemption is narrow - it covers the `Bearer` scheme only, not `Basic` or `Digest` - and it does not depend on the `--cors` setting. The exemption is about CSRF classification, not browser read access; CORS still controls the latter. + +#### What you can remove + +You can now delete any of the following from your plugins and custom templates: + +- Hidden CSRF form fields: + + ```html + + ``` + + The `csrftoken()` template helper (and `request.scope["csrftoken"]()` for plugins that call it from Python) still exists as a compatibility shim. It now returns a per-request random string rather than a cookie-bound signed value. Datasette no longer validates this token, and no `ds_csrftoken` cookie is set. + + **Important for plugin authors:** if your plugin previously used `request.scope["csrftoken"]()` or the `ds_csrftoken` cookie as a security primitive (for example, signing a URL and later comparing it to the cookie), the invariant that the token equals `request.cookies["ds_csrftoken"]` no longer holds. Replace those flows with signed, short-lived action URLs or explicit non-ambient credentials. + +- Manual CSRF token extraction in tests, e.g.: + + ```python + # No longer needed + csrftoken = response.cookies["ds_csrftoken"] + cookies["ds_csrftoken"] = csrftoken + post_data["csrftoken"] = csrftoken + ``` + + The `ds_csrftoken` cookie is no longer set at all. The `csrftoken_from=` argument of the Datasette test client's `.post()` method is now a no-op and can be removed from your test code. + +#### Breaking changes + +- **The `skip_csrf` plugin hook has been removed.** Existing plugins that still declare a `skip_csrf` hookimpl will continue to load - pluggy silently ignores unknown hook names - but the hook is no longer consulted by core, so the flows it previously unlocked will now be blocked (or allowed) purely on the basis of the new header check. + + The new middleware already covers the common cases that `skip_csrf` was written for: + + - Browser-initiated JSON POSTs automatically get `Sec-Fetch-Site: same-origin` and pass the check. + - Non-browser API clients (curl, `requests`, server-to-server scripts) do not send browser security headers and are passed through. + - Requests with an explicit `Authorization: Bearer ...` header are exempt from the CSRF check (see above). + + If your plugin previously used `skip_csrf` to accept cross-origin browser POSTs, replace that flow with an authentication mechanism that does **not** rely on ambient browser credentials. Safe patterns include: + + - Requiring an `Authorization: Bearer ...` API token on the endpoint. + - Requiring a non-ambient credential in the request body (a webhook secret, HMAC signature, signed capability URL, OAuth client credential, or similar). + - Issuing a short-lived signed URL that encodes the actor, the action, and an expiry, and verifying the signature on request. + + Do not rely on the `ds_csrftoken` cookie for your own plugin's security checks - Datasette no longer sets or validates it, and the `request.scope["csrftoken"]()` compatibility shim now returns a fresh random value each request rather than the signed cookie-bound value it used to. + +- **The `asgi-csrf` dependency has been dropped.** Any plugin that imported from `asgi_csrf` directly will need to be updated. + +- **The `csrf_error.html` template now receives a `reason` context variable** instead of `message_id` and `message_name`. Custom overrides of this template should be updated. + +#### Security properties + +For defense-in-depth the `ds_actor` and `ds_messages` cookies continue to be set with `SameSite=Lax` (Datasette's long-standing default). This means a genuine cross-site POST from an attacker's page would arrive without the user's authentication cookie even if the header check somehow failed. diff --git a/pyproject.toml b/pyproject.toml index 2ab2ce10..a0ee050c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ dependencies = [ "uvicorn>=0.11", "aiofiles>=0.4", "janus>=0.6.2", - "asgi-csrf>=0.10", "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", diff --git a/tests/conftest.py b/tests/conftest.py index efa02c0a..1a9b940f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,17 @@ def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): raise AssertionError("Timed out waiting for {} to respond".format(url)) +@pytest.fixture +def bare_ds(): + """ + Minimal Datasette with no plugins, data, metadata, or config - for tests + that want to exercise core behavior (e.g. middleware) in isolation. + """ + from datasette.app import Datasette + + return Datasette(memory=True) + + @pytest_asyncio.fixture async def ds_client(): from datasette.app import Datasette diff --git a/tests/fixtures.py b/tests/fixtures.py index 1f6c491d..713e6c17 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -54,7 +54,6 @@ EXPECTED_PLUGINS = [ "register_token_handler", "render_cell", "row_actions", - "skip_csrf", "startup", "table_actions", "view_actions", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 77079557..4e401c07 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -444,11 +444,6 @@ def homepage_actions(datasette, actor, request): ] -@hookimpl -def skip_csrf(scope): - return scope["path"] == "/skip-csrf" - - @hookimpl def register_actions(datasette): extras_old = datasette.plugin_config("datasette-register-permissions") or {} diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index ed6202a4..5e36a87a 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -109,31 +109,12 @@ def test_insert(canned_write_client): assert response.headers["Location"] == "/data/add_name?success" -@pytest.mark.parametrize( - "query_name,expect_csrf_hidden_field", - [ - ("canned_read", False), - ("add_name_specify_id", True), - ("add_name", True), - ], -) -def test_canned_query_form_csrf_hidden_field( - canned_write_client, query_name, expect_csrf_hidden_field -): - response = canned_write_client.get(f"/data/{query_name}") - html = response.text - fragment = '' in response.text -def test_vary_header(canned_write_client): - # These forms embed a csrftoken so they should be served with Vary: Cookie +def test_canned_query_pages_no_vary_header(canned_write_client): + # These pages no longer embed per-cookie CSRF tokens, so they must not + # set Vary: Cookie - they should be cacheable across users. assert "vary" not in canned_write_client.get("/data").headers - assert "Cookie" == canned_write_client.get("/data/update_name").headers["vary"] + assert "vary" not in canned_write_client.get("/data/update_name").headers def test_json_post_body(canned_write_client): diff --git a/tests/test_csrf_middleware.py b/tests/test_csrf_middleware.py new file mode 100644 index 00000000..07ff598e --- /dev/null +++ b/tests/test_csrf_middleware.py @@ -0,0 +1,270 @@ +""" +Tests for the header-based CSRF (Cross-Origin) protection middleware. + +Datasette uses the Sec-Fetch-Site + Origin header approach described in +Filippo Valsorda's article (https://words.filippo.io/csrf/) and implemented +in Go 1.25's http.CrossOriginProtection. This replaces the previous +token-based asgi-csrf mechanism. +""" + +import pluggy +import pytest + +from datasette import hookimpl +from datasette.csrf import CrossOriginProtectionMiddleware, _install_legacy_csrftoken + + +async def _post(bare_ds, **kwargs): + kwargs.setdefault("data", {"message": "hello", "message_class": "info"}) + return await bare_ds.client.post("/-/messages", **kwargs) + + +async def _run_middleware(scope): + """ + Run CrossOriginProtectionMiddleware against a scope and return + ("allowed",) if the inner app was called, or ("blocked", status) + if the middleware sent a response itself. + """ + + class FakeDs: + async def render_template(self, name, ctx): + return "BLOCKED" + + inner_called = [] + + async def app(scope, receive, send): + inner_called.append(True) + + sent = [] + + async def send(msg): + sent.append(msg) + + mw = CrossOriginProtectionMiddleware(app, FakeDs()) + await mw(scope, None, send) + if inner_called: + return ("allowed",) + start = [m for m in sent if m["type"] == "http.response.start"][0] + return ("blocked", start["status"]) + + +def _http_scope(headers, method="POST"): + return { + "type": "http", + "method": method, + "headers": [(k.encode(), v.encode()) for k, v in headers.items()], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["GET", "HEAD", "OPTIONS"]) +async def test_safe_methods_always_pass(bare_ds, method): + # Safe methods bypass CSRF entirely, even with hostile headers + response = await bare_ds.client.request( + method, + "/-/messages", + headers={"sec-fetch-site": "cross-site", "origin": "http://evil.example"}, + ) + assert response.status_code != 403 or "origin" not in response.text.lower() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sec_fetch_site", ["same-origin", "none"]) +async def test_post_with_trusted_sec_fetch_site_allowed(bare_ds, sec_fetch_site): + # "same-origin" = first-party; "none" = user-initiated direct navigation + response = await _post(bare_ds, headers={"sec-fetch-site": sec_fetch_site}) + assert response.status_code != 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sec_fetch_site", ["cross-site", "same-site", "cross-origin"]) +async def test_post_with_untrusted_sec_fetch_site_blocked(bare_ds, sec_fetch_site): + # same-site is blocked too: different subdomains must not bypass CSRF + response = await _post( + bare_ds, data={"message": "hi"}, headers={"sec-fetch-site": sec_fetch_site} + ) + assert response.status_code == 403 + assert response.headers["content-type"].startswith("text/html") + + +@pytest.mark.asyncio +async def test_post_with_no_browser_headers_allowed(bare_ds): + # curl / requests / server-to-server: no Sec-Fetch-Site, no Origin. + # CSRF is browser-specific so these pass through. + response = await _post(bare_ds) + assert response.status_code != 403 + + +@pytest.mark.asyncio +async def test_post_with_matching_origin_allowed(bare_ds): + # Fallback for older browsers without Sec-Fetch-Site: Origin must match Host + response = await _post(bare_ds, headers={"origin": "http://localhost"}) + assert response.status_code != 403 + + +@pytest.mark.asyncio +async def test_post_with_mismatched_origin_blocked(bare_ds): + response = await _post( + bare_ds, data={"message": "hi"}, headers={"origin": "http://evil.example.com"} + ) + assert response.status_code == 403 + + +@pytest.mark.asyncio +async def test_csrf_error_page_renders(bare_ds): + response = await _post( + bare_ds, data={"message": "hi"}, headers={"sec-fetch-site": "cross-site"} + ) + assert response.status_code == 403 + assert "origin" in response.text.lower() + + +@pytest.mark.asyncio +async def test_csrf_error_page_title_has_no_typo(bare_ds): + response = await _post( + bare_ds, data={"message": "hi"}, headers={"sec-fetch-site": "cross-site"} + ) + assert "CSRF check failed" in response.text + assert "CSRF check failed)" not in response.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope_type", ["websocket", "lifespan"]) +async def test_non_http_scope_passes_through(scope_type): + called = [] + + async def app(scope, receive, send): + called.append(scope["type"]) + + mw = CrossOriginProtectionMiddleware(app, datasette=None) + await mw({"type": scope_type}, None, None) + assert called == [scope_type] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "label,headers,expected", + [ + ( + "plain cross-site blocked", + {"sec-fetch-site": "cross-site", "host": "example.com"}, + ("blocked", 403), + ), + ( + "basic auth does not bypass", + { + "sec-fetch-site": "cross-site", + "host": "example.com", + "authorization": "Basic dXNlcjpwYXNz", + }, + ("blocked", 403), + ), + ( + "bearer auth bypasses", + { + "sec-fetch-site": "cross-site", + "origin": "https://evil.example", + "host": "example.com", + "authorization": "Bearer dstok_abc", + }, + ("allowed",), + ), + ( + "bearer scheme case-insensitive", + { + "sec-fetch-site": "cross-site", + "host": "example.com", + "authorization": "bearer dstok_abc", + }, + ("allowed",), + ), + ( + "non-browser (no Sec-Fetch-Site, no Origin) allowed", + {"host": "example.com"}, + ("allowed",), + ), + ], +) +async def test_middleware_unit(label, headers, expected): + assert await _run_middleware(_http_scope(headers)) == expected + + +def test_legacy_csrftoken_scope_value_nonempty(app_client): + # GET /post/ calls request.scope["csrftoken"]() - must not 500 + response = app_client.get("/post/") + assert response.status == 200 + assert response.text.strip() != "" + assert len(response.text.strip()) >= 20 + + +def test_legacy_csrftoken_no_ds_csrftoken_cookie(app_client): + response = app_client.get("/post/") + assert "ds_csrftoken" not in response.cookies + + +def test_legacy_csrftoken_varies_across_requests(app_client): + r1 = app_client.get("/post/").text.strip() + r2 = app_client.get("/post/").text.strip() + assert r1 != r2 + + +def test_legacy_csrftoken_stable_within_request(): + # Two calls in the same request return the same value + scope = {} + _install_legacy_csrftoken(scope) + assert scope["csrftoken"]() == scope["csrftoken"]() + + +@pytest.mark.asyncio +async def test_cross_site_post_blocked_even_with_ds_csrftoken_cookie(bare_ds): + # A stale ds_csrftoken cookie + csrftoken body field must NOT bypass + # the header-based CSRF check. + response = await _post( + bare_ds, + data={"message": "hi", "message_class": "info", "csrftoken": "abc"}, + headers={"sec-fetch-site": "cross-site"}, + cookies={"ds_csrftoken": "abc"}, + ) + assert response.status_code == 403 + + +@pytest.mark.asyncio +async def test_bearer_invalid_token_not_csrf_error(bare_ds): + # Cross-site POST with bogus bearer must pass CSRF and be rejected + # by auth/permission handling, not by the CSRF middleware. + response = await _post( + bare_ds, + headers={ + "sec-fetch-site": "cross-site", + "authorization": "Bearer totally-invalid-token", + }, + ) + if response.status_code == 403: + assert "origin" not in response.text.lower() + assert "sec-fetch-site" not in response.text.lower() + + +@pytest.mark.asyncio +async def test_cross_site_post_without_auth_still_blocked(bare_ds): + response = await _post( + bare_ds, data={"message": "hi"}, headers={"sec-fetch-site": "cross-site"} + ) + assert response.status_code == 403 + + +def test_legacy_skip_csrf_hookimpl_does_not_break_loading(): + # Plugins that still define skip_csrf must load cleanly - pluggy ignores + # unknown hook implementations - even though the hook is no longer + # consulted by core. Use a throwaway PluginManager so that registering + # this hookimpl does not leak a _HookCaller onto the real datasette.pm. + class LegacyPlugin: + __name__ = "legacy-skip-csrf-plugin" + + @hookimpl + def skip_csrf(self, datasette, scope): + return True + + throwaway = pluggy.PluginManager("datasette") + plugin = LegacyPlugin() + throwaway.register(plugin, name=LegacyPlugin.__name__) + assert throwaway.is_registered(plugin) diff --git a/tests/test_html.py b/tests/test_html.py index 39249c19..4fc144ab 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1202,11 +1202,12 @@ async def test_custom_csrf_error(ds_client): data={ "message": "A message", }, - cookies={"csrftoken": "x"}, + headers={"sec-fetch-site": "cross-site"}, ) assert response.status_code == 403 assert response.headers["content-type"] == "text/html; charset=utf-8" - assert "Error code is FORM_URLENCODED_MISMATCH." in response.text + assert "Reason:" in response.text + assert "cross-site" in response.text @pytest.mark.asyncio diff --git a/tests/test_permissions.py b/tests/test_permissions.py index f9303759..04195f75 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -711,10 +711,6 @@ async def test_actor_restricted_permissions( perms_ds.pdb = True perms_ds.root_enabled = True # Allow root actor to access /-/permissions cookies = {"ds_actor": perms_ds.sign({"a": {"id": "root"}}, "actor")} - csrftoken = (await perms_ds.client.get("/-/permissions", cookies=cookies)).cookies[ - "ds_csrftoken" - ] - cookies["ds_csrftoken"] = csrftoken response = await perms_ds.client.post( "/-/permissions", data={ @@ -722,7 +718,6 @@ async def test_actor_restricted_permissions( "permission": permission, "resource_1": resource_1, "resource_2": resource_2, - "csrftoken": csrftoken, }, cookies=cookies, ) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 4ce2c7c0..c9de1c57 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -812,21 +812,25 @@ def test_hook_register_routes_override(): def test_hook_register_routes_post(app_client): - response = app_client.post("/post/", {"this is": "post data"}, csrftoken_from=True) + response = app_client.post("/post/", {"this is": "post data"}) assert response.status_code == 200 - assert "csrftoken" in response.json assert response.json["this is"] == "post data" def test_hook_register_routes_csrftoken(restore_working_directory, tmpdir_factory): + # csrftoken() is a legacy compatibility shim that returns a + # per-request random value - it is no longer used for CSRF enforcement. templates = tmpdir_factory.mktemp("templates") (templates / "csrftoken_form.html").write_text( - "CSRFTOKEN: {{ csrftoken() }}", "utf-8" + "CSRFTOKEN:{{ csrftoken() }}:END", "utf-8" ) with make_app_client(template_dir=templates) as client: response = client.get("/csrftoken-form/") - expected_token = client.ds._last_request.scope["csrftoken"]() - assert f"CSRFTOKEN: {expected_token}" == response.text + assert response.text.startswith("CSRFTOKEN:") + assert response.text.endswith(":END") + token = response.text[len("CSRFTOKEN:") : -len(":END")] + assert len(token) >= 20 + assert "ds_csrftoken" not in response.cookies @pytest.mark.asyncio @@ -1125,31 +1129,6 @@ async def test_hook_homepage_actions(ds_client): ] -def test_hook_skip_csrf(app_client): - cookie = app_client.actor_cookie({"id": "test"}) - csrf_response = app_client.post( - "/post/", - post_data={"this is": "post data"}, - csrftoken_from=True, - cookies={"ds_actor": cookie}, - ) - assert csrf_response.status_code == 200 - missing_csrf_response = app_client.post( - "/post/", post_data={"this is": "post data"}, cookies={"ds_actor": cookie} - ) - assert missing_csrf_response.status_code == 403 - # But "/skip-csrf" should allow - allow_csrf_response = app_client.post( - "/skip-csrf", post_data={"this is": "post data"}, cookies={"ds_actor": cookie} - ) - assert allow_csrf_response.status_code == 405 # Method not allowed - # /skip-csrf-2 should not - second_missing_csrf_response = app_client.post( - "/skip-csrf-2", post_data={"this is": "post data"}, cookies={"ds_actor": cookie} - ) - assert second_missing_csrf_response.status_code == 403 - - def _extract_commands(output): lines = output.split("Commands:\n", 1)[1].split("\n") return {line.split()[0].replace("*", "") for line in lines if line.strip()} From 9c164572d3e515226e16ec15582cfa369f8bc904 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Apr 2026 18:31:57 -0700 Subject: [PATCH 021/156] Add actor= parameter to datasette.client methods (#2688) `datasette.client.get(path, actor={"id": "root"}` now makes the internal request with that actor as `request.actor` - same for the other HTTP verb methods on `datasette.client`. Upgraded relevant tests to use the new `actor=` mechanism. --- datasette/app.py | 13 +++++ docs/internals.rst | 22 +++++++ tests/test_api.py | 3 +- tests/test_column_types.py | 4 +- tests/test_html.py | 20 +++---- tests/test_internals_datasette_client.py | 73 ++++++++++++++++++++++++ tests/test_permission_endpoints.py | 22 ++++--- tests/test_permissions.py | 23 ++++---- tests/test_plugins.py | 8 +-- tests/test_schema_endpoints.py | 4 +- tests/test_search_tables.py | 10 ++-- 11 files changed, 149 insertions(+), 53 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 6ab32f9e..16545cff 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2654,9 +2654,21 @@ class DatasetteClient: path = f"http://localhost{path}" return path + def _apply_actor(self, kwargs): + """If ``actor=`` was supplied, convert it into a signed ds_actor cookie.""" + actor = kwargs.pop("actor", None) + if actor is None: + return + cookies = dict(kwargs.get("cookies") or {}) + if "ds_actor" in cookies: + raise TypeError("Cannot pass both actor= and a ds_actor cookie") + cookies["ds_actor"] = self.actor_cookie(actor) + kwargs["cookies"] = cookies + async def _request(self, method, path, skip_permission_checks=False, **kwargs): from datasette.permissions import SkipPermissions + self._apply_actor(kwargs) with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): @@ -2722,6 +2734,7 @@ class DatasetteClient: from datasette.permissions import SkipPermissions avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) + self._apply_actor(kwargs) with _DatasetteClientContext(): if skip_permission_checks: with SkipPermissions(): diff --git a/docs/internals.rst b/docs/internals.rst index 1693a241..ba9d3131 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1312,6 +1312,28 @@ These methods can be used with :ref:`internals_datasette_urls` - for example: For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation `__. +.. _internals_datasette_client_actor: + +Authenticating as an actor +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All ``datasette.client`` methods accept an optional ``actor=`` parameter. When set to a dictionary describing an actor, the request is made with a signed ``ds_actor`` cookie identifying that actor — as if the request had been made by a user who is signed in as that actor. + +This is a convenient shorthand equivalent to signing the cookie manually using ``datasette.client.actor_cookie()``. + +Example usage: + +.. code-block:: python + + response = await datasette.client.get( + "/-/actor.json", actor={"id": "root"} + ) + assert response.json() == {"actor": {"id": "root"}} + +This parameter works with all HTTP methods (``get``, ``post``, ``put``, ``patch``, ``delete``, ``options``, ``head``) and the generic ``request`` method. + +Passing both ``actor=`` and a ``ds_actor`` cookie via ``cookies=`` raises a ``TypeError``. Other unrelated cookies can be combined with ``actor=``. + Bypassing permission checks ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/test_api.py b/tests/test_api.py index 95958a72..3676c1fb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -553,8 +553,7 @@ async def test_actions_json(ds_client): original_root_enabled = ds_client.ds.root_enabled try: ds_client.ds.root_enabled = True - cookies = {"ds_actor": ds_client.actor_cookie({"id": "root"})} - response = await ds_client.get("/-/actions.json", cookies=cookies) + response = await ds_client.get("/-/actions.json", actor={"id": "root"}) data = response.json() finally: ds_client.ds.root_enabled = original_root_enabled diff --git a/tests/test_column_types.py b/tests/test_column_types.py index 68b92a39..6e89acb9 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -933,9 +933,7 @@ async def test_set_column_type_ui_data_includes_applicable_types( await ds_ct_editor_permission.invoke_startup() response = await ds_ct_editor_permission.client.get( "/data/posts", - cookies={ - "ds_actor": ds_ct_editor_permission.client.actor_cookie({"id": "editor"}) - }, + actor={"id": "editor"}, ) assert response.status_code == 200 data = _window_data_from_html(response.text, "_setColumnTypeData") diff --git a/tests/test_html.py b/tests/test_html.py index 4fc144ab..e38898da 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1003,10 +1003,10 @@ async def test_navigation_menu_links( # Enable root user if testing with root actor if actor_id == "root": ds_client.ds.root_enabled = True - cookies = {} + kwargs = {} if actor_id: - cookies = {"ds_actor": ds_client.actor_cookie({"id": actor_id})} - html = (await ds_client.get("/", cookies=cookies)).text + kwargs["actor"] = {"id": actor_id} + html = (await ds_client.get("/", **kwargs)).text soup = Soup(html, "html.parser") details = soup.find("nav").find("details") if not actor_id: @@ -1215,8 +1215,7 @@ async def test_actions_page(ds_client): original_root_enabled = ds_client.ds.root_enabled try: ds_client.ds.root_enabled = True - cookies = {"ds_actor": ds_client.actor_cookie({"id": "root"})} - response = await ds_client.get("/-/actions", cookies=cookies) + response = await ds_client.get("/-/actions", actor={"id": "root"}) assert response.status_code == 200 assert "Registered actions" in response.text assert "

" in response.text @@ -1233,8 +1232,7 @@ async def test_actions_page_does_not_display_none_string(ds_client): original_root_enabled = ds_client.ds.root_enabled try: ds_client.ds.root_enabled = True - cookies = {"ds_actor": ds_client.actor_cookie({"id": "root"})} - response = await ds_client.get("/-/actions", cookies=cookies) + response = await ds_client.get("/-/actions", actor={"id": "root"}) assert response.status_code == 200 assert "None" not in response.text finally: @@ -1247,11 +1245,11 @@ async def test_permission_debug_tabs_with_query_string(ds_client): original_root_enabled = ds_client.ds.root_enabled try: ds_client.ds.root_enabled = True - cookies = {"ds_actor": ds_client.actor_cookie({"id": "root"})} + actor = {"id": "root"} # Test /-/allowed with query string response = await ds_client.get( - "/-/allowed?action=view-table&page_size=50", cookies=cookies + "/-/allowed?action=view-table&page_size=50", actor=actor ) assert response.status_code == 200 # Check that Rules and Check tabs have the query string @@ -1263,7 +1261,7 @@ async def test_permission_debug_tabs_with_query_string(ds_client): # Test /-/rules with query string response = await ds_client.get( - "/-/rules?action=view-database&parent=test", cookies=cookies + "/-/rules?action=view-database&parent=test", actor=actor ) assert response.status_code == 200 # Check that Allowed and Check tabs have the query string @@ -1271,7 +1269,7 @@ async def test_permission_debug_tabs_with_query_string(ds_client): assert 'href="/-/check?action=view-database&parent=test"' in response.text # Test /-/check with query string - response = await ds_client.get("/-/check?action=execute-sql", cookies=cookies) + response = await ds_client.get("/-/check?action=execute-sql", actor=actor) assert response.status_code == 200 # Check that Allowed and Rules tabs have the query string assert 'href="/-/allowed?action=execute-sql"' in response.text diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py index 326fcdc0..ccac280b 100644 --- a/tests/test_internals_datasette_client.py +++ b/tests/test_internals_datasette_client.py @@ -311,3 +311,76 @@ async def test_in_client_with_skip_permission_checks(): assert all(in_client_values), f"Expected all True, got {in_client_values}" finally: ds.pm.unregister(name="test_in_client_skip_plugin") + + +@pytest.mark.asyncio +async def test_actor_parameter_sets_cookie(datasette): + """Passing actor= should sign a ds_actor cookie and authenticate the request.""" + response = await datasette.client.get("/-/actor.json", actor={"id": "root"}) + assert response.status_code == 200 + assert response.json() == {"actor": {"id": "root"}} + + +@pytest.mark.asyncio +async def test_actor_parameter_works_with_request_method(datasette): + response = await datasette.client.request( + "GET", "/-/actor.json", actor={"id": "root"} + ) + assert response.status_code == 200 + assert response.json() == {"actor": {"id": "root"}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method", ["get", "post", "options", "head", "put", "patch", "delete"] +) +async def test_actor_parameter_all_http_methods(datasette, method): + """actor= should not cause errors on any HTTP verb wrapper.""" + client_method = getattr(datasette.client, method) + # Just verify no TypeError about unexpected 'actor' kwarg + response = await client_method("/", actor={"id": "root"}) + assert isinstance(response, httpx.Response) + + +@pytest.mark.asyncio +async def test_actor_parameter_conflicts_with_ds_actor_cookie(datasette): + """Passing both actor= and a ds_actor cookie should raise TypeError.""" + with pytest.raises(TypeError, match="actor"): + await datasette.client.get( + "/-/actor.json", + actor={"id": "root"}, + cookies={"ds_actor": datasette.client.actor_cookie({"id": "other"})}, + ) + + +@pytest.mark.asyncio +async def test_actor_parameter_merges_with_other_cookies(datasette): + """actor= should coexist with unrelated cookies.""" + response = await datasette.client.get( + "/-/actor.json", + actor={"id": "root"}, + cookies={"unrelated": "value"}, + ) + assert response.status_code == 200 + assert response.json() == {"actor": {"id": "root"}} + + +@pytest.mark.asyncio +async def test_actor_parameter_with_skip_permission_checks( + datasette_with_permissions, +): + """actor= should be compatible with skip_permission_checks.""" + ds = datasette_with_permissions + # Non-admin actor with skip_permission_checks=True should get 200 + response = await ds.client.get( + "/test_db.json", + actor={"id": "user"}, + skip_permission_checks=True, + ) + assert response.status_code == 200 + # Admin actor on its own should also get 200 + response = await ds.client.get("/test_db.json", actor={"id": "admin"}) + assert response.status_code == 200 + # Non-admin actor should get 403 + response = await ds.client.get("/test_db.json", actor={"id": "user"}) + assert response.status_code == 403 diff --git a/tests/test_permission_endpoints.py b/tests/test_permission_endpoints.py index 84f3370f..e25be23e 100644 --- a/tests/test_permission_endpoints.py +++ b/tests/test_permission_endpoints.py @@ -117,9 +117,7 @@ async def test_allowed_json_with_actor(ds_with_permissions): """Test /-/allowed.json includes actor information.""" response = await ds_with_permissions.client.get( "/-/allowed.json?action=view-table", - cookies={ - "ds_actor": ds_with_permissions.client.actor_cookie({"id": "test_user"}) - }, + actor={"id": "test_user"}, ) assert response.status_code == 200 data = response.json() @@ -252,7 +250,7 @@ async def test_rules_json_basic( # Use root actor for rules endpoint (requires permissions-debug) response = await ds_with_permissions.client.get( path, - cookies={"ds_actor": ds_with_permissions.client.actor_cookie({"id": "root"})}, + actor={"id": "root"}, ) assert response.status_code == expected_status data = response.json() @@ -264,7 +262,7 @@ async def test_rules_json_response_structure(ds_with_permissions): """Test that /-/rules.json returns the expected structure.""" response = await ds_with_permissions.client.get( "/-/rules.json?action=view-instance", - cookies={"ds_actor": ds_with_permissions.client.actor_cookie({"id": "root"})}, + actor={"id": "root"}, ) assert response.status_code == 200 data = response.json() @@ -294,7 +292,7 @@ async def test_rules_json_includes_all_rules(ds_with_permissions): # Root user should see rules for everything response = await ds_with_permissions.client.get( "/-/rules.json?action=view-table", - cookies={"ds_actor": ds_with_permissions.client.actor_cookie({"id": "root"})}, + actor={"id": "root"}, ) assert response.status_code == 200 data = response.json() @@ -326,7 +324,7 @@ async def test_rules_json_pagination(): # Test basic pagination structure - just verify it returns paginated results response = await ds.client.get( "/-/rules.json?action=view-table&page_size=2&page=1", - cookies={"ds_actor": ds.client.actor_cookie({"id": "root"})}, + actor={"id": "root"}, ) assert response.status_code == 200 data = response.json() @@ -343,7 +341,7 @@ async def test_rules_json_with_actor(ds_with_permissions): # Use root actor (rules endpoint requires permissions-debug) response = await ds_with_permissions.client.get( "/-/rules.json?action=view-table", - cookies={"ds_actor": ds_with_permissions.client.actor_cookie({"id": "root"})}, + actor={"id": "root"}, ) assert response.status_code == 200 data = response.json() @@ -374,7 +372,7 @@ async def test_root_user_respects_settings_deny(): # Root user should NOT see the denied database response = await ds.client.get( "/-/allowed.json?action=view-database", - cookies={"ds_actor": ds.client.actor_cookie({"id": "root"})}, + actor={"id": "root"}, ) assert response.status_code == 200 data = response.json() @@ -415,7 +413,7 @@ async def test_root_user_respects_settings_deny_tables(): # Root user should NOT see tables from the content database response = await ds.client.get( "/-/allowed.json?action=view-table", - cookies={"ds_actor": ds.client.actor_cookie({"id": "root"})}, + actor={"id": "root"}, ) assert response.status_code == 200 data = response.json() @@ -475,7 +473,7 @@ async def test_execute_sql_requires_view_database(): # User should NOT have execute-sql permission because view-database is denied response = await ds.client.get( "/-/allowed.json?action=execute-sql", - cookies={"ds_actor": ds.client.actor_cookie({"id": "test_user"})}, + actor={"id": "test_user"}, ) assert response.status_code == 200 data = response.json() @@ -491,7 +489,7 @@ async def test_execute_sql_requires_view_database(): # (may be 403 or 302 redirect to login/error page depending on middleware) response = await ds.client.get( "/secret?sql=SELECT+1", - cookies={"ds_actor": ds.client.actor_cookie({"id": "test_user"})}, + actor={"id": "test_user"}, ) assert response.status_code in (302, 403), ( f"Expected 302 or 403 when trying to execute SQL without view-database permission, " diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 04195f75..0c09e773 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1171,10 +1171,10 @@ async def test_api_explorer_visibility( try: prev_config = perms_ds.config perms_ds.config = config or {} - cookies = {} + kwargs = {} if is_logged_in: - cookies = {"ds_actor": perms_ds.client.actor_cookie({"id": "user"})} - response = await perms_ds.client.get("/-/api", cookies=cookies) + kwargs["actor"] = {"id": "user"} + response = await perms_ds.client.get("/-/api", **kwargs) if expected_visible_tables: assert response.status_code == 200 # Search HTML for stuff matching: @@ -1208,8 +1208,7 @@ async def test_view_table_token_cannot_gain_access_without_base_permission(perms # Restricted token claims access to perms_ds_two/t1 only "_r": {"r": {"perms_ds_two": {"t1": ["vt"]}}}, } - cookies = {"ds_actor": perms_ds.client.actor_cookie(actor)} - response = await perms_ds.client.get("/perms_ds_two/t1.json", cookies=cookies) + response = await perms_ds.client.get("/perms_ds_two/t1.json", actor=actor) assert response.status_code == 403 finally: perms_ds.config = previous_config @@ -1328,7 +1327,7 @@ async def test_actor_restrictions( if restrictions: actor["_r"] = restrictions method = getattr(perms_ds.client, verb) - kwargs = {"cookies": {"ds_actor": perms_ds.client.actor_cookie(actor)}} + kwargs = {"actor": actor} if body: kwargs["json"] = body perms_ds._permission_checks.clear() @@ -1459,7 +1458,7 @@ async def test_actor_restrictions_do_not_expand_allowed_resources(perms_ds): # And explicit permission checks should still deny response = await perms_ds.client.get( "/perms_ds_one/t1.json", - cookies={"ds_actor": perms_ds.client.actor_cookie(actor)}, + actor=actor, ) assert response.status_code == 403 finally: @@ -1527,18 +1526,17 @@ async def test_actor_restrictions_json_endpoints_show_filtered_listings(perms_ds """Test that /.json and /db.json show correct filtered listings - issue #2534""" actor = {"id": "user", "_r": {"r": {"perms_ds_one": {"t1": ["vt"]}}}} - cookies = {"ds_actor": perms_ds.client.actor_cookie(actor)} # /.json should be 403 (no view-instance permission) - response = await perms_ds.client.get("/.json", cookies=cookies) + response = await perms_ds.client.get("/.json", actor=actor) assert response.status_code == 403 # /perms_ds_one.json should be 403 (no view-database permission) - response = await perms_ds.client.get("/perms_ds_one.json", cookies=cookies) + response = await perms_ds.client.get("/perms_ds_one.json", actor=actor) assert response.status_code == 403 # /perms_ds_one/t1.json should be 200 - response = await perms_ds.client.get("/perms_ds_one/t1.json", cookies=cookies) + response = await perms_ds.client.get("/perms_ds_one/t1.json", actor=actor) assert response.status_code == 200 @@ -1547,10 +1545,9 @@ async def test_actor_restrictions_view_instance_only(perms_ds): """Test actor restricted to view-instance only - issue #2534""" actor = {"id": "user", "_r": {"a": ["vi"]}} - cookies = {"ds_actor": perms_ds.client.actor_cookie(actor)} # /.json should be 200 (has view-instance permission) - response = await perms_ds.client.get("/.json", cookies=cookies) + response = await perms_ds.client.get("/.json", actor=actor) assert response.status_code == 200 # But no databases should be visible (no view-database permission) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index c9de1c57..083e23a0 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1024,7 +1024,7 @@ async def test_hook_view_actions(ds_client): assert get_actions_links(response.text) == [] response_2 = await ds_client.get( "/fixtures/simple_view", - cookies={"ds_actor": ds_client.actor_cookie({"id": "bob"})}, + actor={"id": "bob"}, ) assert ">View actions<" in response_2.text assert sorted( @@ -1088,7 +1088,7 @@ async def test_hook_row_actions(ds_client): response_2 = await ds_client.get( "/fixtures/facet_cities/1", - cookies={"ds_actor": ds_client.actor_cookie({"id": "sam"})}, + actor={"id": "sam"}, ) assert get_actions_links(response_2.text) == [ { @@ -1116,9 +1116,7 @@ async def test_hook_homepage_actions(ds_client): # No button for anonymous users assert "Homepage actions" not in response.text # Signed in user gets an action - response2 = await ds_client.get( - "/", cookies={"ds_actor": ds_client.actor_cookie({"id": "troy"})} - ) + response2 = await ds_client.get("/", actor={"id": "troy"}) assert "Homepage actions" in response2.text assert get_actions_links(response2.text) == [ { diff --git a/tests/test_schema_endpoints.py b/tests/test_schema_endpoints.py index 50742df2..c95d8614 100644 --- a/tests/test_schema_endpoints.py +++ b/tests/test_schema_endpoints.py @@ -151,7 +151,7 @@ async def test_schema_permission_enforcement(schema_ds, url): # Authenticated user with permission should succeed response = await schema_ds.client.get( url, - cookies={"ds_actor": schema_ds.client.actor_cookie({"id": "root"})}, + actor={"id": "root"}, ) assert response.status_code == 200 @@ -171,7 +171,7 @@ async def test_instance_schema_respects_database_permissions(schema_ds): # Authenticated user should see all databases response = await schema_ds.client.get( "/-/schema.json", - cookies={"ds_actor": schema_ds.client.actor_cookie({"id": "root"})}, + actor={"id": "root"}, ) assert response.status_code == 200 data = response.json() diff --git a/tests/test_search_tables.py b/tests/test_search_tables.py index 34b37706..b901c0b3 100644 --- a/tests/test_search_tables.py +++ b/tests/test_search_tables.py @@ -86,7 +86,7 @@ async def test_tables_search_with_auth(ds_with_tables): # Editor user should see content.articles response = await ds_with_tables.client.get( "/-/tables.json?q=articles", - cookies={"ds_actor": ds_with_tables.client.actor_cookie({"id": "editor"})}, + actor={"id": "editor"}, ) assert response.status_code == 200 data = response.json() @@ -104,7 +104,7 @@ async def test_tables_search_partial_match(ds_with_tables): # Search for "com" should match "comments" response = await ds_with_tables.client.get( "/-/tables.json?q=com", - cookies={"ds_actor": ds_with_tables.client.actor_cookie({"id": "user"})}, + actor={"id": "user"}, ) assert response.status_code == 200 data = response.json() @@ -120,7 +120,7 @@ async def test_tables_search_respects_database_permissions(ds_with_tables): # Even authenticated users shouldn't see it because database is denied response = await ds_with_tables.client.get( "/-/tables.json?q=secrets", - cookies={"ds_actor": ds_with_tables.client.actor_cookie({"id": "user"})}, + actor={"id": "user"}, ) assert response.status_code == 200 data = response.json() @@ -135,7 +135,7 @@ async def test_tables_search_respects_table_permissions(ds_with_tables): # Regular authenticated user searching for "users" response = await ds_with_tables.client.get( "/-/tables.json?q=users", - cookies={"ds_actor": ds_with_tables.client.actor_cookie({"id": "regular"})}, + actor={"id": "regular"}, ) assert response.status_code == 200 data = response.json() @@ -150,7 +150,7 @@ async def test_tables_search_response_structure(ds_with_tables): """Test that response has correct structure.""" response = await ds_with_tables.client.get( "/-/tables.json?q=users", - cookies={"ds_actor": ds_with_tables.client.actor_cookie({"id": "user"})}, + actor={"id": "user"}, ) assert response.status_code == 200 data = response.json() From f02484c3defcbb8a777d65c920db778496d49457 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Dec 2025 22:38:04 -0800 Subject: [PATCH 022/156] From 409 warnings down to 52 warnings. By closing unclosed database connections. Refs #2614 --- datasette/app.py | 1 + datasette/cli.py | 4 +++- datasette/utils/__init__.py | 5 +++++ datasette/utils/sqlite.py | 17 +++++++++-------- tests/conftest.py | 7 ++++--- tests/fixtures.py | 4 ++++ tests/test_api_write.py | 2 ++ tests/test_cli.py | 20 +++++++++++++++----- tests/test_config_dir.py | 3 +++ tests/test_crossdb.py | 1 + tests/test_internals_database.py | 12 +++++++++--- tests/test_plugins.py | 7 ++++--- tests/test_utils.py | 15 ++++++++++++--- tests/utils.py | 4 +++- 14 files changed, 75 insertions(+), 27 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 16545cff..0f417ec9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1819,6 +1819,7 @@ class Datasette: break except importlib.metadata.PackageNotFoundError: pass + conn.close() return info def _plugins(self, request=None, all=False): diff --git a/datasette/cli.py b/datasette/cli.py index 32a4d898..93aa22ef 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -615,7 +615,9 @@ def serve( for file in file_paths: if not pathlib.Path(file).exists(): if create: - sqlite3.connect(file).execute("vacuum") + conn = sqlite3.connect(file) + conn.execute("vacuum") + conn.close() else: raise click.ClickException( "Invalid value for '[FILES]...': Path '{}' does not exist.".format( diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 7fb81f02..1fea992e 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -681,13 +681,18 @@ def detect_fts_sql(table): def detect_json1(conn=None): + close_conn = False if conn is None: conn = sqlite3.connect(":memory:") + close_conn = True try: conn.execute("SELECT json('{}')") return True except Exception: return False + finally: + if close_conn: + conn.close() def table_columns(conn, table): diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index 342ff3fa..d0a2d783 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -20,15 +20,16 @@ def sqlite_version(): def _sqlite_version(): - return tuple( - map( - int, - sqlite3.connect(":memory:") - .execute("select sqlite_version()") - .fetchone()[0] - .split("."), + conn = sqlite3.connect(":memory:") + try: + return tuple( + map( + int, + conn.execute("select sqlite_version()").fetchone()[0].split("."), + ) ) - ) + finally: + conn.close() def supports_table_xinfo(): diff --git a/tests/conftest.py b/tests/conftest.py index 1a9b940f..3a3203fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -100,9 +100,10 @@ async def ds_client(): def pytest_report_header(config): - return "SQLite: {}".format( - sqlite3.connect(":memory:").execute("select sqlite_version()").fetchone()[0] - ) + conn = sqlite3.connect(":memory:") + version = conn.execute("select sqlite_version()").fetchone()[0] + conn.close() + return "SQLite: {}".format(version) def pytest_configure(config): diff --git a/tests/fixtures.py b/tests/fixtures.py index 713e6c17..f61ec0c7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -187,6 +187,8 @@ def app_client(): def app_client_no_files(): ds = Datasette([]) yield TestClient(ds) + for db in ds.databases.values(): + db.close() @pytest.fixture(scope="session") @@ -822,6 +824,7 @@ def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename for sql, params in TABLE_PARAMETERIZED_SQL: with conn: conn.execute(sql, params) + conn.close() print(f"Test tables written to {db_filename}") if metadata: with open(metadata, "w") as fp: @@ -850,6 +853,7 @@ def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename pathlib.Path(extra_db_filename).unlink() conn = sqlite3.connect(extra_db_filename) conn.executescript(EXTRA_DATABASE_SQL) + conn.close() print(f"Test tables written to {extra_db_filename}") diff --git a/tests/test_api_write.py b/tests/test_api_write.py index e59c4295..adf8d310 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -17,6 +17,8 @@ def ds_write(tmp_path_factory): db.execute( "create table docs (id integer primary key, title text, score float, age integer)" ) + db1.close() + db2.close() ds = Datasette([db_path], immutables=[db_path_immutable]) ds.root_enabled = True yield ds diff --git a/tests/test_cli.py b/tests/test_cli.py index 7673c3f3..1d3a2b28 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -472,7 +472,9 @@ def test_serve_duplicate_database_names(tmpdir): nested.mkdir() db_2_path = str(tmpdir / "nested" / "db.db") for path in (db_1_path, db_2_path): - sqlite3.connect(path).execute("vacuum") + conn = sqlite3.connect(path) + conn.execute("vacuum") + conn.close() result = runner.invoke(cli, [db_1_path, db_2_path, "--get", "/-/databases.json"]) assert result.exit_code == 0, result.output databases = json.loads(result.output) @@ -486,7 +488,9 @@ def test_weird_database_names(tmpdir, filename): # https://github.com/simonw/datasette/issues/1181 runner = CliRunner() db_path = str(tmpdir / filename) - sqlite3.connect(db_path).execute("vacuum") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.close() result1 = runner.invoke(cli, [db_path, "--get", "/"]) assert result1.exit_code == 0, result1.output filename_no_stem = filename.rsplit(".", 1)[0] @@ -523,7 +527,9 @@ def test_duplicate_database_files_error(tmpdir): """Test that passing the same database file multiple times raises an error""" runner = CliRunner() db_path = str(tmpdir / "test.db") - sqlite3.connect(db_path).execute("vacuum") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.close() # Test with exact duplicate result = runner.invoke(cli, ["serve", db_path, db_path, "--get", "/"]) @@ -542,7 +548,9 @@ def test_duplicate_database_files_error(tmpdir): config_dir = tmpdir / "config" config_dir.mkdir() config_db_path = str(config_dir / "data.db") - sqlite3.connect(config_db_path).execute("vacuum") + conn = sqlite3.connect(config_db_path) + conn.execute("vacuum") + conn.close() result3 = runner.invoke( cli, ["serve", config_db_path, str(config_dir), "--get", "/"] @@ -553,7 +561,9 @@ def test_duplicate_database_files_error(tmpdir): # Test that mixing a file NOT in the directory with a directory works fine other_db_path = str(tmpdir / "other.db") - sqlite3.connect(other_db_path).execute("vacuum") + conn = sqlite3.connect(other_db_path) + conn.execute("vacuum") + conn.close() result4 = runner.invoke( cli, ["serve", other_db_path, str(config_dir), "--get", "/-/databases.json"] diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index ae7fe500..0a9b30d8 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -60,6 +60,7 @@ def config_dir(tmp_path_factory): (1, 'San Francisco') ; """) + db.close() # Mark "immutable.db" as immutable (config_dir / "inspect-data.json").write_text( @@ -95,6 +96,8 @@ def test_invalid_settings(config_dir): def config_dir_client(config_dir): ds = Datasette([], config_dir=config_dir) yield _TestClient(ds) + for db in ds.databases.values(): + db.close() def test_settings(config_dir_client): diff --git a/tests/test_crossdb.py b/tests/test_crossdb.py index 7807cd5d..11e53224 100644 --- a/tests/test_crossdb.py +++ b/tests/test_crossdb.py @@ -43,6 +43,7 @@ def test_crossdb_warning_if_too_many_databases(tmp_path_factory): path = str(db_dir / "db_{}.db".format(i)) conn = sqlite3.connect(path) conn.execute("vacuum") + conn.close() dbs.append(path) runner = CliRunner() result = runner.invoke( diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 9a83dd4f..e3d35f57 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -543,7 +543,9 @@ async def test_execute_write_fn_exception(db): @pytest.mark.timeout(1) async def test_execute_write_fn_connection_exception(tmpdir, app_client): path = str(tmpdir / "immutable.db") - sqlite3.connect(path).execute("vacuum") + conn = sqlite3.connect(path) + conn.execute("vacuum") + conn.close() db = Database(app_client.ds, path=path, is_mutable=False) app_client.ds.add_database(db, name="immutable-db") @@ -747,15 +749,19 @@ async def test_replace_database(tmpdir): path1 = str(tmpdir / "data1.db") (tmpdir / "two").mkdir() path2 = str(tmpdir / "two" / "data1.db") - sqlite3.connect(path1).executescript(""" + conn1 = sqlite3.connect(path1) + conn1.executescript(""" create table t (id integer primary key); insert into t (id) values (1); insert into t (id) values (2); """) - sqlite3.connect(path2).executescript(""" + conn1.close() + conn2 = sqlite3.connect(path2) + conn2.executescript(""" create table t (id integer primary key); insert into t (id) values (1); """) + conn2.close() datasette = Datasette([path1]) db = datasette.get_database("data1") count = (await db.execute("select count(*) from t")).first()[0] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 083e23a0..7ebd57f3 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -422,9 +422,9 @@ def test_plugins_async_template_function(restore_working_directory): .select("pre.extra_from_awaitable_function")[0] .text ) - expected = ( - sqlite3.connect(":memory:").execute("select sqlite_version()").fetchone()[0] - ) + conn = sqlite3.connect(":memory:") + expected = conn.execute("select sqlite_version()").fetchone()[0] + conn.close() assert expected == extra_from_awaitable_function @@ -466,6 +466,7 @@ def view_names_client(tmp_path_factory): db_path = str(tmpdir / "fixtures.db") conn = sqlite3.connect(db_path) conn.executescript(TABLES) + conn.close() return _TestClient( Datasette([db_path], template_dir=str(templates), plugins_dir=str(plugins)) ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 85ab9e6b..3fcb623e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -208,6 +208,7 @@ def test_detect_fts(open_quote, close_quote): assert None is utils.detect_fts(conn, "Test_View") assert None is utils.detect_fts(conn, "r") assert "Street_Tree_List_fts" == utils.detect_fts(conn, "Street_Tree_List") + conn.close() @pytest.mark.parametrize("table", ("regular", "has'single quote")) @@ -222,6 +223,7 @@ def test_detect_fts_different_table_names(table): conn = utils.sqlite3.connect(":memory:") conn.executescript(sql) assert "{table}_fts".format(table=table) == utils.detect_fts(conn, table) + conn.close() @pytest.mark.parametrize( @@ -359,6 +361,7 @@ def test_table_columns(): create table places (id integer primary key, name text, bob integer) """) assert ["id", "name", "bob"] == utils.table_columns(conn, "places") + conn.close() @pytest.mark.parametrize( @@ -433,11 +436,13 @@ def test_check_connection_spatialite_raises(): conn = sqlite3.connect(path) with pytest.raises(utils.SpatialiteConnectionProblem): utils.check_connection(conn) + conn.close() def test_check_connection_passes(): conn = sqlite3.connect(":memory:") utils.check_connection(conn) + conn.close() def test_call_with_supported_arguments(): @@ -564,10 +569,14 @@ def test_display_actor(actor, expected): async def test_initial_path_for_datasette(tmp_path_factory, dbs, expected_path): db_dir = tmp_path_factory.mktemp("dbs") one_table = str(db_dir / "one.db") - sqlite3.connect(one_table).execute("create table one (id integer primary key)") + conn1 = sqlite3.connect(one_table) + conn1.execute("create table one (id integer primary key)") + conn1.close() two_tables = str(db_dir / "two.db") - sqlite3.connect(two_tables).execute("create table two (id integer primary key)") - sqlite3.connect(two_tables).execute("create table three (id integer primary key)") + conn2 = sqlite3.connect(two_tables) + conn2.execute("create table two (id integer primary key)") + conn2.execute("create table three (id integer primary key)") + conn2.close() datasette = Datasette( [{"one_table": one_table, "two_tables": two_tables}[db] for db in dbs] ) diff --git a/tests/utils.py b/tests/utils.py index e2d9339a..808feea7 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -34,7 +34,9 @@ def inner_html(soup): def has_load_extension(): conn = sqlite3.connect(":memory:") - return hasattr(conn, "enable_load_extension") + result = hasattr(conn, "enable_load_extension") + conn.close() + return result def cookie_was_deleted(response, cookie): From 028cc2446f54b766b84b0d1e1db4237b1699a3ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Apr 2026 19:23:21 -0700 Subject: [PATCH 023/156] Don't allow cookies with Authorization: Bearer to bypass CSRF Refs #2689 --- datasette/csrf.py | 5 ++++- tests/test_csrf_middleware.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/datasette/csrf.py b/datasette/csrf.py index dd968a4d..845c8fb4 100644 --- a/datasette/csrf.py +++ b/datasette/csrf.py @@ -68,7 +68,10 @@ class CrossOriginProtectionMiddleware: # before evaluating Sec-Fetch-Site / Origin. Only "Bearer" is exempt; # schemes like Basic or Digest can be browser-managed and ambient. authorization = headers.get(b"authorization", b"").decode("latin-1") - if authorization: + cookie_header = headers.get(b"cookie") + # If the request also carries a Cookie header, ambient cookie auth + # could be in play, so do NOT treat it as exempt. + if authorization and not cookie_header: scheme = authorization.split(None, 1)[0].lower() if scheme == "bearer": await self.app(scope, receive, send) diff --git a/tests/test_csrf_middleware.py b/tests/test_csrf_middleware.py index 07ff598e..820df1e7 100644 --- a/tests/test_csrf_middleware.py +++ b/tests/test_csrf_middleware.py @@ -252,6 +252,20 @@ async def test_cross_site_post_without_auth_still_blocked(bare_ds): assert response.status_code == 403 +@pytest.mark.asyncio +async def test_bearer_with_cookie_does_not_bypass(): + # Bearer + Cookie => ambient cookie auth is in play, not exempt. + scope = _http_scope( + { + "sec-fetch-site": "cross-site", + "host": "example.com", + "authorization": "Bearer dstok_abc", + "cookie": "ds_actor=anything", + } + ) + assert await _run_middleware(scope) == ("blocked", 403) + + def test_legacy_skip_csrf_hookimpl_does_not_break_loading(): # Plugins that still define skip_csrf must load cleanly - pluggy ignores # unknown hook implementations - even though the hook is no longer From a973e3ffa119c2563c7fb7cca0feef4797eb5b8a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 14 Apr 2026 19:24:31 -0700 Subject: [PATCH 024/156] Normalize headers in CSRF checks, refs #2689 --- datasette/csrf.py | 71 +++++++++++++++++++++++++++++------ tests/test_csrf_middleware.py | 50 ++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 11 deletions(-) diff --git a/datasette/csrf.py b/datasette/csrf.py index 845c8fb4..df239aee 100644 --- a/datasette/csrf.py +++ b/datasette/csrf.py @@ -16,6 +16,38 @@ from .utils.asgi import asgi_send SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) +DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443} + + +def _normalize_headers(raw_headers): + """Lowercase header names; for duplicates, last value wins.""" + result = {} + for name, value in raw_headers: + if isinstance(name, str): + name = name.encode("latin-1") + if isinstance(value, str): + value = value.encode("latin-1") + result[name.lower()] = value + return result + + +def _origin_tuple(value): + """ + Parse an origin-like string into ``(scheme, host, port)`` with default + ports filled in. Raises ``ValueError`` for malformed input. + """ + parsed = urllib.parse.urlsplit(value) + scheme = (parsed.scheme or "").lower() + host = (parsed.hostname or "").lower() + if not scheme or not host: + raise ValueError("missing scheme or host in {!r}".format(value)) + port = parsed.port # may raise ValueError on bad ports + if port is None: + port = DEFAULT_PORTS.get(scheme) + if port is None: + raise ValueError("unknown default port for scheme {!r}".format(scheme)) + return scheme, host, port + def _install_legacy_csrftoken(scope): """ @@ -61,19 +93,19 @@ class CrossOriginProtectionMiddleware: await self.app(scope, receive, send) return - headers = dict(scope.get("headers") or []) + headers = _normalize_headers(scope.get("headers") or []) + authorization = headers.get(b"authorization", b"").decode("latin-1") + cookie_header = headers.get(b"cookie") # Bearer-token requests are not ambient browser credentials, so they # are not CSRF-vulnerable. Narrowly exempt them from the header check # before evaluating Sec-Fetch-Site / Origin. Only "Bearer" is exempt; # schemes like Basic or Digest can be browser-managed and ambient. - authorization = headers.get(b"authorization", b"").decode("latin-1") - cookie_header = headers.get(b"cookie") # If the request also carries a Cookie header, ambient cookie auth # could be in play, so do NOT treat it as exempt. if authorization and not cookie_header: - scheme = authorization.split(None, 1)[0].lower() - if scheme == "bearer": + parts = authorization.split(None, 1) + if parts and parts[0].lower() == "bearer": await self.app(scope, receive, send) return @@ -104,12 +136,20 @@ class CrossOriginProtectionMiddleware: await self.app(scope, receive, send) return - # Fallback for older browsers: Origin host must match Host header - parsed = urllib.parse.urlparse(origin) - origin_host = parsed.hostname or "" - if parsed.port: - origin_host = "{}:{}".format(origin_host, parsed.port) - if origin_host == host: + # Fallback for older browsers: Origin must match the request's own + # scheme + host + port. Compare full origin tuples, not host alone. + request_scheme = self._request_scheme(scope) + try: + origin_tuple = _origin_tuple(origin) + expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host)) + except ValueError: + await self._forbid( + send, + "Malformed Origin {!r} or Host {!r}".format(origin, host), + ) + return + + if origin_tuple == expected_tuple: await self.app(scope, receive, send) return @@ -118,6 +158,15 @@ class CrossOriginProtectionMiddleware: "Origin {!r} does not match Host {!r}".format(origin, host), ) + def _request_scheme(self, scope): + if self.datasette is not None: + try: + if self.datasette.setting("force_https_urls"): + return "https" + except Exception: + pass + return scope.get("scheme") or "http" + async def _forbid(self, send, reason): await asgi_send( send, diff --git a/tests/test_csrf_middleware.py b/tests/test_csrf_middleware.py index 820df1e7..2fcfb216 100644 --- a/tests/test_csrf_middleware.py +++ b/tests/test_csrf_middleware.py @@ -266,6 +266,56 @@ async def test_bearer_with_cookie_does_not_bypass(): assert await _run_middleware(scope) == ("blocked", 403) +@pytest.mark.asyncio +async def test_origin_scheme_must_match(): + # http Origin against an https request must be blocked even when host matches. + scope = _http_scope({"origin": "http://example.com", "host": "example.com"}) + scope["scheme"] = "https" + assert await _run_middleware(scope) == ("blocked", 403) + + +@pytest.mark.asyncio +async def test_origin_port_must_match(): + scope = _http_scope({"origin": "http://example.com:8001", "host": "example.com"}) + scope["scheme"] = "http" + assert await _run_middleware(scope) == ("blocked", 403) + + +@pytest.mark.asyncio +async def test_origin_default_port_normalized(): + # http://example.com:80 == http://example.com + scope = _http_scope({"origin": "http://example.com:80", "host": "example.com"}) + scope["scheme"] = "http" + assert await _run_middleware(scope) == ("allowed",) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "headers", + [ + {"authorization": " ", "host": "example.com", "origin": "http://evil"}, + {"origin": "http://example.com:notaport", "host": "example.com"}, + {"origin": "not-a-url", "host": "example.com"}, + ], +) +async def test_malformed_headers_do_not_500(headers): + # Should be a clean 403, not an unhandled exception. + result = await _run_middleware(_http_scope(headers)) + assert result[0] == "blocked" + assert result[1] == 403 + + +@pytest.mark.asyncio +async def test_uppercase_header_names_normalized(): + # ASGI servers should lowercase, but middleware normalizes defensively. + scope = { + "type": "http", + "method": "POST", + "headers": [(b"Sec-Fetch-Site", b"same-origin")], + } + assert await _run_middleware(scope) == ("allowed",) + + def test_legacy_skip_csrf_hookimpl_does_not_break_loading(): # Plugins that still define skip_csrf must load cleanly - pluggy ignores # unknown hook implementations - even though the hook is no longer From 4922fc2e39ba3e6033ed3c1f6fccef60fa76d316 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 15 Apr 2026 15:11:18 -0700 Subject: [PATCH 025/156] Disallow null primary keys in upsert Refs https://github.com/simonw/datasette/issues/1936#issuecomment-1341849496 --- datasette/views/table.py | 7 +++++++ tests/test_api_write.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/datasette/views/table.py b/datasette/views/table.py index 5643858d..7027bb10 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -461,6 +461,13 @@ class TableInsertView(BaseView): i, '", "'.join(missing_pks) ) ) + null_pks = [pk for pk in pks_list if pk in row and row[pk] is None] + if null_pks: + errors.append( + 'Row {} has null primary key column(s): "{}"'.format( + i, '", "'.join(null_pks) + ) + ) invalid_columns = set(row.keys()) - columns if invalid_columns and not extras.get("alter"): errors.append( diff --git a/tests/test_api_write.py b/tests/test_api_write.py index adf8d310..91a88606 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -299,6 +299,14 @@ async def test_insert_rows(ds_write, return_rows): 400, ['Row 0 is missing primary key column(s): "id"'], ), + # null primary key + ( + "/data/docs/-/upsert", + {"rows": [{"id": None, "title": "Null PK"}]}, + None, + 400, + ['Row 0 has null primary key column(s): "id"'], + ), # Upsert does not support ignore or replace ( "/data/docs/-/upsert", From 73f338b9f3d510dc8fa60f6697b276cb121a0ec5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 15 Apr 2026 15:29:59 -0700 Subject: [PATCH 026/156] Better example in API explorer for /-/upsert, closes #1936 --- datasette/views/special.py | 10 ++++++++-- tests/test_api_write.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/datasette/views/special.py b/datasette/views/special.py index dbe5eab1..1df74f07 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -813,9 +813,15 @@ class ApiExplorerView(BaseView): "json": { "rows": [ { - column: None + column: "<{}{}>".format( + column, + ( + " (primary key)" + if column in pks + else "" + ), + ) for column in await db.table_columns(table) - if column not in pks } ] }, diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 91a88606..f600d4f5 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -44,6 +44,22 @@ def _headers(token): } +@pytest.mark.asyncio +async def test_api_explorer_upsert_example_json(ds_write): + response = await ds_write.client.get("/-/api", actor={"id": "root"}) + print("STATUS", response.status_code) + assert response.status_code == 200 + import urllib.parse + + text = urllib.parse.unquote_plus(response.text) + upsert_idx = text.index("/data/docs/-/upsert") + upsert_chunk = text[upsert_idx : upsert_idx + 500] + assert '"id": ""' in upsert_chunk + assert '"title": ""' in upsert_chunk + assert '"score": "<score>"' in upsert_chunk + assert '"age": "<age>"' in upsert_chunk + + @pytest.mark.asyncio @pytest.mark.parametrize( "content_type", From 5f39036b9bebcece55e6657446c69105ef8322c6 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 15 Apr 2026 15:44:06 -0700 Subject: [PATCH 027/156] ok: true in /db.json for consistency --- datasette/views/database.py | 1 + docs/changelog.rst | 6 ++++++ docs/pages.rst | 2 ++ tests/test_api.py | 1 + 4 files changed, 10 insertions(+) diff --git a/datasette/views/database.py b/datasette/views/database.py index 916cdbc1..faf870d0 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -130,6 +130,7 @@ class DatabaseView(View): actor=request.actor, ) json_data = { + "ok": True, "database": database, "private": private, "path": datasette.urls.database(database), diff --git a/docs/changelog.rst b/docs/changelog.rst index adb62d42..005e5d7f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,6 +3,12 @@ ========= Changelog ========= +.. dev: + +dev +--- + +- The ``/<database>.json`` endpoint now includes an ``"ok": true`` key, for consistency with other JSON API responses. .. _v1_0_a26: diff --git a/docs/pages.rst b/docs/pages.rst index a8a25fa7..34c851a5 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -40,6 +40,8 @@ The JSON version of this page provides programmatic access to the underlying dat * `fivethirtyeight.datasettes.com/fivethirtyeight.json <https://fivethirtyeight.datasettes.com/fivethirtyeight.json>`_ * `datasette.io/global-power-plants.json <https://datasette.io/global-power-plants.json>`_ +The returned object includes an ``"ok": true`` key alongside keys such as ``"database"``, ``"tables"``, ``"views"``, ``"queries"`` and ``"metadata"``. + .. _DatabaseView_hidden: Hidden tables diff --git a/tests/test_api.py b/tests/test_api.py index 3676c1fb..392030d6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -59,6 +59,7 @@ async def test_database_page(ds_client): response = await ds_client.get("/fixtures.json") assert response.status_code == 200 data = response.json() + assert data["ok"] is True assert data["database"] == "fixtures" # Build lookup for easier assertions From 1a7030d66824d25323ab032aa4e86c1ff571d8e0 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 15 Apr 2026 15:47:48 -0700 Subject: [PATCH 028/156] API explorer special case for rowid in /-/upsert Refs #1936 --- datasette/views/special.py | 7 +++++-- tests/test_api_write.py | 21 ++++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/datasette/views/special.py b/datasette/views/special.py index 1df74f07..b28e9257 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -817,11 +817,14 @@ class ApiExplorerView(BaseView): column, ( " (primary key)" - if column in pks + if column in (pks or ["rowid"]) else "" ), ) - for column in await db.table_columns(table) + for column in ( + (["rowid"] if not pks else []) + + await db.table_columns(table) + ) } ] }, diff --git a/tests/test_api_write.py b/tests/test_api_write.py index f600d4f5..9ba08848 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -47,7 +47,6 @@ def _headers(token): @pytest.mark.asyncio async def test_api_explorer_upsert_example_json(ds_write): response = await ds_write.client.get("/-/api", actor={"id": "root"}) - print("STATUS", response.status_code) assert response.status_code == 200 import urllib.parse @@ -60,6 +59,26 @@ async def test_api_explorer_upsert_example_json(ds_write): assert '"age": "<age>"' in upsert_chunk +@pytest.mark.asyncio +async def test_api_explorer_upsert_example_json_rowid_table(tmp_path_factory): + db_path = str(tmp_path_factory.mktemp("dbs") / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("create table things (title text, score float)") + conn.close() + ds = Datasette([db_path]) + ds.root_enabled = True + response = await ds.client.get("/-/api", actor={"id": "root"}) + assert response.status_code == 200 + import urllib.parse + + text = urllib.parse.unquote_plus(response.text) + upsert_idx = text.index("/data/things/-/upsert") + upsert_chunk = text[upsert_idx : upsert_idx + 500] + assert '"rowid": "<rowid (primary key)>"' in upsert_chunk + assert '"title": "<title>"' in upsert_chunk + assert '"score": "<score>"' in upsert_chunk + + @pytest.mark.asyncio @pytest.mark.parametrize( "content_type", From 67349e0e020bc61ae1291722b9ac3758987ea7f6 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 15 Apr 2026 16:03:58 -0700 Subject: [PATCH 029/156] New :pr:`ID` shortcut for docs --- docs/changelog.rst | 86 +++++++++++++++++++++++----------------------- docs/conf.py | 1 + 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 005e5d7f..cff580da 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -49,7 +49,7 @@ Other changes ``write_wrapper()`` plugin hook for intercepting write operations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A new :ref:`write_wrapper() <plugin_hook_write_wrapper>` plugin hook allows plugins to intercept and wrap database write operations. (`#2636 <https://github.com/simonw/datasette/pull/2636>`__) +A new :ref:`write_wrapper() <plugin_hook_write_wrapper>` plugin hook allows plugins to intercept and wrap database write operations. (:pr:`2636`) Plugins implement the hook as a generator-based context manager: @@ -67,20 +67,20 @@ Plugins implement the hook as a generator-based context manager: ``register_token_handler()`` plugin hook for custom API token backends ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A new :ref:`register_token_handler() <plugin_hook_register_token_handler>` plugin hook allows plugins to provide custom token backends for API authentication. (`#2650 <https://github.com/simonw/datasette/pull/2650>`__) +A new :ref:`register_token_handler() <plugin_hook_register_token_handler>` plugin hook allows plugins to provide custom token backends for API authentication. (:pr:`2650`) This includes a **backwards incompatible change**: the ``datasette.create_token()`` internal method is now an ``async`` method. Consult the :ref:`upgrade guide <upgrade_guide_v1_a25>` for details on how to update your code. ``render_cell()`` now receives a ``pks`` parameter ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook now receives a ``pks`` parameter containing the list of primary key column names for the table being rendered. This avoids plugins needing to make redundant async calls to look up primary keys. (`#2641 <https://github.com/simonw/datasette/pull/2641>`__) +The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook now receives a ``pks`` parameter containing the list of primary key column names for the table being rendered. This avoids plugins needing to make redundant async calls to look up primary keys. (:pr:`2641`) Other changes ~~~~~~~~~~~~~ - Facets defined in metadata now preserve their configured order, instead of being sorted by result count. Request-based facets added via the ``_facet`` parameter are still sorted by result count and appear after metadata-defined facets. (:issue:`2647`) -- Fixed ``--reload`` incorrectly interpreting the ``serve`` command as a file argument. Thanks, `Daniel Bates <https://github.com/danielalanbates>`__. (`#2646 <https://github.com/simonw/datasette/pull/2646>`__) +- Fixed ``--reload`` incorrectly interpreting the ``serve`` command as a file argument. Thanks, `Daniel Bates <https://github.com/danielalanbates>`__. (:pr:`2646`) .. _v1_0_a24: @@ -90,7 +90,7 @@ Other changes ``request.form()`` method for POST data and file uploads ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Datasette now includes a ``request.form()`` method for parsing form submissions, including handling file uploads. (`#2626 <https://github.com/simonw/datasette/pull/2626>`__) +Datasette now includes a ``request.form()`` method for parsing form submissions, including handling file uploads. (:pr:`2626`) This supports both ``application/x-www-form-urlencoded`` and ``multipart/form-data`` content types, and uses a new streaming multipart parser that processes uploads without buffering entire request bodies in memory. @@ -241,7 +241,7 @@ Other changes - Fixed bug where ``link:`` HTTP headers used invalid syntax. (:issue:`2470`) - No longer tested against Python 3.8. Now tests against Python 3.13. - FTS tables are now hidden by default if they correspond to a content table. (:issue:`2477`) -- Fixed bug with foreign key links to rows in databases with filenames containing a special character. Thanks, `Jack Stratton <https://github.com/phroa>`__. (`#2476 <https://github.com/simonw/datasette/pull/2476>`__) +- Fixed bug with foreign key links to rows in databases with filenames containing a special character. Thanks, `Jack Stratton <https://github.com/phroa>`__. (:pr:`2476`) .. _v1_0_a17: @@ -284,7 +284,7 @@ Other changes This release focuses on performance, in particular against large tables, and introduces some minor breaking changes for CSS styling in Datasette plugins. - Removed the unit conversions feature and its dependency, Pint. This means Datasette is now compatible with the upcoming Python 3.13. (:issue:`2400`, :issue:`2320`) -- The ``datasette --pdb`` option now uses the `ipdb <https://github.com/gotcha/ipdb>`__ debugger if it is installed. You can install it using ``datasette install ipdb``. Thanks, `Tiago Ilieve <https://github.com/myhro>`__. (`#2342 <https://github.com/simonw/datasette/pull/2342>`__) +- The ``datasette --pdb`` option now uses the `ipdb <https://github.com/gotcha/ipdb>`__ debugger if it is installed. You can install it using ``datasette install ipdb``. Thanks, `Tiago Ilieve <https://github.com/myhro>`__. (:pr:`2342`) - Fixed a confusing error that occurred if ``metadata.json`` contained nested objects. (:issue:`2403`) - Fixed a bug with ``?_trace=1`` where it returned a blank page if the response was larger than 256KB. (:issue:`2404`) - Tracing mechanism now also displays SQL queries that returned errors or ran out of time. `datasette-pretty-traces 0.5 <https://github.com/simonw/datasette-pretty-traces/releases/tag/0.5>`__ includes support for displaying this new type of trace. (:issue:`2405`) @@ -314,7 +314,7 @@ This release focuses on performance, in particular against large tables, and int - Failed CSRF checks now display a more user-friendly error page. (:issue:`2390`) - Fixed a bug where the ``json1`` extension was not correctly detected on the ``/-/versions`` page. Thanks, `Seb Bacon <https://github.com/sebbacon>`__. (:issue:`2326`) - Fixed a bug where the Datasette write API did not correctly accept ``Content-Type: application/json; charset=utf-8``. (:issue:`2384`) -- Fixed a bug where Datasette would fail to start if ``metadata.yml`` contained a ``queries`` block. (`#2386 <https://github.com/simonw/datasette/pull/2386>`__) +- Fixed a bug where Datasette would fail to start if ``metadata.yml`` contained a ``queries`` block. (:pr:`2386`) .. _v1_0_a14: @@ -327,13 +327,13 @@ This alpha introduces significant changes to Datasette's :ref:`metadata` system, - Metadata about tables, databases, instances and columns is now stored in :ref:`internals_internal`. Thanks, Alex Garcia. (:issue:`2341`) - Database write connections now execute using the ``IMMEDIATE`` isolation level for SQLite. This should help avoid a rare ``SQLITE_BUSY`` error that could occur when a transaction upgraded to a write mid-flight. (:issue:`2358`) - Fix for a bug where canned queries with named parameters could fail against SQLite 3.46. (:issue:`2353`) -- Datasette now serves ``E-Tag`` headers for static files. Thanks, `Agustin Bacigalup <https://github.com/redraw>`__. (`#2306 <https://github.com/simonw/datasette/pull/2306>`__) +- Datasette now serves ``E-Tag`` headers for static files. Thanks, `Agustin Bacigalup <https://github.com/redraw>`__. (:pr:`2306`) - Dropdown menus now use a ``z-index`` that should avoid them being hidden by plugins. (:issue:`2311`) - Incorrect table and row names are no longer reflected back on the resulting 404 page. (:issue:`2359`) - Improved documentation for async usage of the :ref:`plugin_hook_track_event` hook. (:issue:`2319`) - Fixed some HTTPX deprecation warnings. (:issue:`2307`) - Datasette now serves a ``<html lang="en">`` attribute. Thanks, `Charles Nepote <https://github.com/CharlesNepote>`__. (:issue:`2348`) -- Datasette's automated tests now run against the maximum and minimum supported versions of SQLite: 3.25 (from September 2018) and 3.46 (from May 2024). Thanks, Alex Garcia. (`#2352 <https://github.com/simonw/datasette/pull/2352>`__) +- Datasette's automated tests now run against the maximum and minimum supported versions of SQLite: 3.25 (from September 2018) and 3.46 (from May 2024). Thanks, Alex Garcia. (:pr:`2352`) - Fixed an issue where clicking twice on the URL output by ``datasette --root`` produced a confusing error. (:issue:`2375`) .. _v0_64_8: @@ -497,7 +497,7 @@ This provides two initial hooks, with more to come in the future: - :ref:`makeAboveTablePanelConfigs() <javascript_plugins_makeAboveTablePanelConfigs>` can add additional panels to the top of the table page. - :ref:`makeColumnActions() <javascript_plugins_makeColumnActions>` can add additional actions to the column menu. -Thanks `Cameron Yick <https://github.com/hydrosquall>`__ for contributing this feature. (`#2052 <https://github.com/simonw/datasette/pull/2052>`__) +Thanks `Cameron Yick <https://github.com/hydrosquall>`__ for contributing this feature. (:pr:`2052`) Plugin hooks ~~~~~~~~~~~~ @@ -563,7 +563,7 @@ Minor fixes - Datasette now checks if the user has permission to view a table linked to by a foreign key before turning that foreign key into a clickable link. (:issue:`2178`) - The ``execute-sql`` permission now implies that the actor can also view the database and instance. (:issue:`2169`) - Documentation describing a pattern for building plugins that themselves :ref:`define further hooks <writing_plugins_extra_hooks>` for other plugins. (:issue:`1765`) -- Datasette is now tested against the Python 3.12 preview. (`#2175 <https://github.com/simonw/datasette/pull/2175>`__) +- Datasette is now tested against the Python 3.12 preview. (:pr:`2175`) .. _v1_0_a5: @@ -766,11 +766,11 @@ Features ~~~~~~~~ - Now tested against Python 3.11. Docker containers used by ``datasette publish`` and ``datasette package`` both now use that version of Python. (:issue:`1853`) -- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 <https://github.com/simonw/datasette/pull/1789>`__) +- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (:pr:`1789`) - Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) - The :ref:`setting_truncate_cells_html` setting now also affects long URLs in columns. (:issue:`1805`) - The non-JavaScript SQL editor textarea now increases height to fit the SQL query. (:issue:`1786`) -- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 <https://github.com/simonw/datasette/pull/1794>`__) +- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (:pr:`1794`) - The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) - SQL queries can now include leading SQL comments, using ``/* ... */`` or ``-- ...`` syntax. Thanks, Charles Nepote. (:issue:`1860`) - SQL query is now re-displayed when terminated with a time limit error. (:issue:`1819`) @@ -792,7 +792,7 @@ Documentation - New tutorial: `Cleaning data with sqlite-utils and Datasette <https://datasette.io/tutorials/clean-data>`__. - Screenshots in the documentation are now maintained using `shot-scraper <https://shot-scraper.datasette.io/>`__, as described in `Automating screenshots for the Datasette documentation using shot-scraper <https://simonwillison.net/2022/Oct/14/automating-screenshots/>`__. (:issue:`1844`) - More detailed command descriptions on the :ref:`CLI reference <cli_reference>` page. (:issue:`1787`) -- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (`#1825 <https://github.com/simonw/datasette/pull/1825>`__) +- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (:pr:`1825`) .. _v0_62: @@ -808,14 +808,14 @@ Features - Datasette is now compatible with `Pyodide <https://pyodide.org/>`__. This is the enabling technology behind `Datasette Lite <https://lite.datasette.io/>`__. (:issue:`1733`) - Database file downloads now implement conditional GET using ETags. (:issue:`1739`) -- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (`#1759 <https://github.com/simonw/datasette/pull/1759>`__) +- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (:pr:`1759`) - Datasette now runs some SQL queries in parallel. This has limited impact on performance, see `this research issue <https://github.com/simonw/datasette/issues/1727>`__ for details. - New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`) - Spaces in the database names in URLs are now encoded as ``+`` rather than ``~20``. (:issue:`1701`) - ``<Binary: 2427344 bytes>`` is now displayed as ``<Binary: 2,427,344 bytes>`` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`) - The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image <https://hub.docker.com/datasetteproject/datasette>`__ has been upgraded to ``3.10.6-slim-bullseye``. (:issue:`1768`) - Canned writable queries against immutable databases now show a warning message. (:issue:`1728`) -- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (`#1717 <https://github.com/simonw/datasette/pull/1717>`__) +- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (:pr:`1717`) - ``datasette publish cloudrun`` has new ``--min-instances`` and ``--max-instances`` options. (:issue:`1779`) Plugin hooks @@ -823,7 +823,7 @@ Plugin hooks - New plugin hook: :ref:`handle_exception() <plugin_hook_handle_exception>`, for custom handling of exceptions caught by Datasette. (:issue:`1770`) - The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook is now also passed a ``row`` argument, representing the ``sqlite3.Row`` object that is being rendered. (:issue:`1300`) -- The :ref:`configuration directory <config_dir>` is now stored in ``datasette.config_dir``, making it available to plugins. Thanks, Chris Amico. (`#1766 <https://github.com/simonw/datasette/pull/1766>`__) +- The :ref:`configuration directory <config_dir>` is now stored in ``datasette.config_dir``, making it available to plugins. Thanks, Chris Amico. (:pr:`1766`) Bug fixes ~~~~~~~~~ @@ -876,7 +876,7 @@ Datasette also now requires Python 3.7 or higher. - Common Datasette symbols can now be imported directly from the top-level ``datasette`` package, see :ref:`internals_shortcuts`. Those symbols are ``Response``, ``Forbidden``, ``NotFound``, ``hookimpl``, ``actor_matches_allow``. (:issue:`957`) - ``/-/versions`` page now returns additional details for libraries used by SpatiaLite. (:issue:`1607`) - Documentation now links to the `Datasette Tutorials <https://datasette.io/tutorials>`__. -- Datasette will now also look for SpatiaLite in ``/opt/homebrew`` - thanks, Dan Peterson. (`#1649 <https://github.com/simonw/datasette/pull/1649>`__) +- Datasette will now also look for SpatiaLite in ``/opt/homebrew`` - thanks, Dan Peterson. (:pr:`1649`) - Fixed bug where :ref:`custom pages <custom_pages>` did not work on Windows. Thanks, Robert Christie. (:issue:`1545`) - Fixed error caused when a table had a column named ``n``. (:issue:`1228`) @@ -975,7 +975,7 @@ Other small fixes - New :ref:`register_commands() <plugin_hook_register_commands>` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`) - Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`) - Upgraded dependency `httpx 0.20 <https://github.com/encode/httpx/releases/tag/0.20.0>`__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defaults to ``False`` where it previously defaulted to ``True``. (:issue:`1488`) -- The ``--cors`` option now causes Datasette to return the ``Access-Control-Allow-Headers: Authorization`` header, in addition to ``Access-Control-Allow-Origin: *``. (`#1467 <https://github.com/simonw/datasette/pull/1467>`__) +- The ``--cors`` option now causes Datasette to return the ``Access-Control-Allow-Headers: Authorization`` header, in addition to ``Access-Control-Allow-Origin: *``. (:pr:`1467`) - Code that figures out which named parameters a SQL query takes in order to display form fields for them is no longer confused by strings that contain colon characters. (:issue:`1421`) - Renamed ``--help-config`` option to ``--help-settings``. (:issue:`1431`) - ``datasette.databases`` property is now a documented API. (:issue:`1443`) @@ -1004,7 +1004,7 @@ Other small fixes - New ``datasette --uds /tmp/datasette.sock`` option for binding Datasette to a Unix domain socket, see :ref:`proxy documentation <deploying_proxy>` (:issue:`1388`) - ``"searchmode": "raw"`` table metadata option for defaulting a table to executing SQLite full-text search syntax without first escaping it, see :ref:`full_text_search_advanced_queries`. (:issue:`1389`) - New plugin hook: ``get_metadata()``, for returning custom metadata for an instance, database or table. Thanks, Brandon Roberts! (:issue:`1384`) -- New plugin hook: :ref:`plugin_hook_skip_csrf`, for opting out of CSRF protection based on the incoming request. (:issue:`1377`) +- New plugin hook: ``skip_csrf``, for opting out of CSRF protection based on the incoming request. (:issue:`1377`) - The :ref:`menu_links() <plugin_hook_menu_links>`, :ref:`table_actions() <plugin_hook_table_actions>` and :ref:`database_actions() <plugin_hook_database_actions>` plugin hooks all gained a new optional ``request`` argument providing access to the current request. (:issue:`1371`) - Major performance improvement for Datasette faceting. (:issue:`1394`) - Improved documentation for :ref:`deploying_proxy` to recommend using ``ProxyPreservehost On`` with Apache. (:issue:`1387`) @@ -1074,8 +1074,8 @@ Documentation improvements, bug fixes and support for SpatiaLite 5. - The :ref:`Response.asgi_send() <internals_response_asgi_send>` method is now documented. (:issue:`1266`) - The official Datasette Docker image now bundles SpatiaLite version 5. (:issue:`1278`) - Fixed a ``no such table: pragma_database_list`` bug when running Datasette against SQLite versions prior to SQLite 3.16.0. (:issue:`1276`) -- HTML lists displayed in table cells are now styled correctly. Thanks, Bob Whitelock. (:issue:`1141`, `#1252 <https://github.com/simonw/datasette/pull/1252>`__) -- Configuration directory mode now correctly serves immutable databases that are listed in ``inspect-data.json``. Thanks Campbell Allen and Frankie Robertson. (`#1031 <https://github.com/simonw/datasette/pull/1031>`__, `#1229 <https://github.com/simonw/datasette/pull/1229>`__) +- HTML lists displayed in table cells are now styled correctly. Thanks, Bob Whitelock. (:issue:`1141`, :pr:`1252`) +- Configuration directory mode now correctly serves immutable databases that are listed in ``inspect-data.json``. Thanks Campbell Allen and Frankie Robertson. (:pr:`1031`, :pr:`1229`) .. _v0_55: @@ -1252,7 +1252,7 @@ A new visual design, plugin hooks for adding navigation options, better handling New visual design ~~~~~~~~~~~~~~~~~ -Datasette is no longer white and grey with blue and purple links! `Natalie Downe <https://twitter.com/natbat>`__ has been working on a visual refresh, the first iteration of which is included in this release. (`#1056 <https://github.com/simonw/datasette/pull/1056>`__) +Datasette is no longer white and grey with blue and purple links! `Natalie Downe <https://twitter.com/natbat>`__ has been working on a visual refresh, the first iteration of which is included in this release. (:pr:`1056`) .. image:: datasette-0.51.png :width: 740px @@ -1329,7 +1329,7 @@ New :ref:`deploying` documentation with guides for deploying Datasette on a Linu Other improvements in this release: -- :ref:`publish_cloud_run` documentation now covers Google Cloud SDK options. Thanks, Geoffrey Hing. (`#995 <https://github.com/simonw/datasette/pull/995>`__) +- :ref:`publish_cloud_run` documentation now covers Google Cloud SDK options. Thanks, Geoffrey Hing. (:pr:`995`) - New ``datasette -o`` option which opens your browser as soon as Datasette starts up. (:issue:`970`) - Datasette now sets ``sqlite3.enable_callback_tracebacks(True)`` so that errors in custom SQL functions will display tracebacks. (:issue:`891`) - Fixed two rendering bugs with column headers in portrait mobile view. (:issue:`978`, :issue:`980`) @@ -1433,7 +1433,7 @@ See also `Datasette 0.49: The annotated release notes <https://simonwillison.net - ``tests`` are now excluded from the Datasette package properly - thanks, abeyerpath. (:issue:`456`) - The Datasette package published to PyPI now includes ``sdist`` as well as ``bdist_wheel``. - Better titles for canned query pages. (:issue:`887`) -- Now only loads Python files from a directory passed using the ``--plugins-dir`` option - thanks, Amjith Ramanujam. (`#890 <https://github.com/simonw/datasette/pull/890>`__) +- Now only loads Python files from a directory passed using the ``--plugins-dir`` option - thanks, Amjith Ramanujam. (:pr:`890`) - New documentation section on :ref:`publish_vercel`. .. _v0_45: @@ -1702,7 +1702,7 @@ Also in this release: * Datasette now has a *pattern portfolio* at ``/-/patterns`` - e.g. https://latest.datasette.io/-/patterns. This is a page that shows every Datasette user interface component in one place, to aid core development and people building custom CSS themes. (:issue:`151`) * SQLite `PRAGMA functions <https://www.sqlite.org/pragma.html#pragfunc>`__ such as ``pragma_table_info(tablename)`` are now allowed in Datasette SQL queries. (:issue:`761`) * Datasette pages now consistently return a ``content-type`` of ``text/html; charset=utf-8"``. (:issue:`752`) -* Datasette now handles an ASGI ``raw_path`` value of ``None``, which should allow compatibility with the `Mangum <https://github.com/erm/mangum>`__ adapter for running ASGI apps on AWS Lambda. Thanks, Colin Dellow. (`#719 <https://github.com/simonw/datasette/pull/719>`__) +* Datasette now handles an ASGI ``raw_path`` value of ``None``, which should allow compatibility with the `Mangum <https://github.com/erm/mangum>`__ adapter for running ASGI apps on AWS Lambda. Thanks, Colin Dellow. (:pr:`719`) * Installation documentation now covers how to :ref:`installation_pipx`. (:issue:`756`) * Improved the documentation for :ref:`full_text_search`. (:issue:`748`) @@ -1755,7 +1755,7 @@ Also in this release: ----------------- * Plugins now have a supported mechanism for writing to a database, using the new ``.execute_write()`` and ``.execute_write_fn()`` methods. :ref:`Documentation <database_execute_write>`. (:issue:`682`) -* Immutable databases that have had their rows counted using the ``inspect`` command now use the calculated count more effectively - thanks, Kevin Keogh. (`#666 <https://github.com/simonw/datasette/pull/666>`__) +* Immutable databases that have had their rows counted using the ``inspect`` command now use the calculated count more effectively - thanks, Kevin Keogh. (:pr:`666`) * ``--reload`` no longer restarts the server if a database file is modified, unless that database was opened immutable mode with ``-i``. (:issue:`494`) * New ``?_searchmode=raw`` option turns off escaping for FTS queries in ``?_search=`` allowing full use of SQLite's `FTS5 query syntax <https://www.sqlite.org/fts5.html#full_text_query_syntax>`__. (:issue:`676`) @@ -1776,7 +1776,7 @@ Also in this release: * Added five new plugins and one new conversion tool to the :ref:`ecosystem`. * The ``Datasette`` class has a new ``render_template()`` method which can be used by plugins to render templates using Datasette's pre-configured `Jinja <https://jinja.palletsprojects.com/>`__ templating library. -* You can now execute SQL queries that start with a ``-- comment`` - thanks, Jay Graves (`#653 <https://github.com/simonw/datasette/pull/653>`__) +* You can now execute SQL queries that start with a ``-- comment`` - thanks, Jay Graves (:pr:`653`) .. _v0_34: @@ -1784,7 +1784,7 @@ Also in this release: ----------------- * ``_search=`` queries are now correctly escaped using a new ``escape_fts()`` custom SQL function. This means you can now run searches for strings like ``park.`` without seeing errors. (:issue:`651`) -* `Google Cloud Run <https://cloud.google.com/run/>`__ is no longer in beta, so ``datasette publish cloudrun`` has been updated to work even if the user has not installed the ``gcloud`` beta components package. Thanks, Katie McLaughlin (`#660 <https://github.com/simonw/datasette/pull/660>`__) +* `Google Cloud Run <https://cloud.google.com/run/>`__ is no longer in beta, so ``datasette publish cloudrun`` has been updated to work even if the user has not installed the ``gcloud`` beta components package. Thanks, Katie McLaughlin (:pr:`660`) * ``datasette package`` now accepts a ``--port`` option for specifying which port the resulting Docker container should listen on. (:issue:`661`) .. _v0_33: @@ -1822,7 +1822,7 @@ Datasette now renders templates using `Jinja async mode <https://jinja.palletspr 0.31.1 (2019-11-12) ------------------- -- Deployments created using ``datasette publish`` now use ``python:3.8`` base Docker image (`#629 <https://github.com/simonw/datasette/pull/629>`__) +- Deployments created using ``datasette publish`` now use ``python:3.8`` base Docker image (:pr:`629`) .. _v0_31: @@ -1835,10 +1835,10 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you pip install datasette==0.30.2 -- Format SQL button now works with read-only SQL queries - thanks, Tobias Kunze (`#602 <https://github.com/simonw/datasette/pull/602>`__) +- Format SQL button now works with read-only SQL queries - thanks, Tobias Kunze (:pr:`602`) - New ``?column__notin=x,y,z`` filter for table views (:issue:`614`) - Table view now uses ``select col1, col2, col3`` instead of ``select *`` -- Database filenames can now contain spaces - thanks, Tobias Kunze (`#590 <https://github.com/simonw/datasette/pull/590>`__) +- Database filenames can now contain spaces - thanks, Tobias Kunze (:pr:`590`) - Removed obsolete ``?_group_count=col`` feature (:issue:`504`) - Improved user interface and documentation for ``datasette publish cloudrun`` (:issue:`608`) - Tables with indexes now show the ``CREATE INDEX`` statements on the table page (:issue:`618`) @@ -1874,7 +1874,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you - Allow ``EXPLAIN WITH...`` (:issue:`583`) - Button to format SQL - thanks, Tobias Kunze (:issue:`136`) - Sort databases on homepage by argument order - thanks, Tobias Kunze (:issue:`585`) -- Display metadata footer on custom SQL queries - thanks, Tobias Kunze (`#589 <https://github.com/simonw/datasette/pull/589>`__) +- Display metadata footer on custom SQL queries - thanks, Tobias Kunze (:pr:`589`) - Use ``--platform=managed`` for ``publish cloudrun`` (:issue:`587`) - Fixed bug returning non-ASCII characters in CSV (:issue:`584`) - Fix for ``/foo`` v.s. ``/foo-bar`` bug (:issue:`601`) @@ -1885,7 +1885,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you ------------------- - Fixed implementation of CodeMirror on database page (:issue:`560`) -- Documentation typo fixes - thanks, Min ho Kim (`#561 <https://github.com/simonw/datasette/pull/561>`__) +- Documentation typo fixes - thanks, Min ho Kim (:pr:`561`) - Mechanism for detecting if a table has FTS enabled now works if the table name used alternative escaping mechanisms (:issue:`570`) - for compatibility with `a recent change to sqlite-utils <https://github.com/simonw/sqlite-utils/pull/57>`__. .. _v0_29_2: @@ -2030,7 +2030,7 @@ Datasette :ref:`facets` provide an intuitive way to quickly summarize and intera Facet by array (:issue:`359`) is only available if your SQLite installation provides the ``json1`` extension. Datasette will automatically detect columns that contain JSON arrays of values and offer a faceting interface against those columns - useful for modelling things like tags without needing to break them out into a new table. See :ref:`facet_by_json_array` for more. -The new :ref:`plugin_register_facet_classes` plugin hook (`#445 <https://github.com/simonw/datasette/pull/445>`__) can be used to register additional custom facet classes. Each facet class should provide two methods: ``suggest()`` which suggests facet selections that might be appropriate for a provided SQL query, and ``facet_results()`` which executes a facet operation and returns results. Datasette's own faceting implementations have been refactored to use the same API as these plugins. +The new :ref:`plugin_register_facet_classes` plugin hook (:pr:`445`) can be used to register additional custom facet classes. Each facet class should provide two methods: ``suggest()`` which suggests facet selections that might be appropriate for a provided SQL query, and ``facet_results()`` which executes a facet operation and returns results. Datasette's own faceting implementations have been refactored to use the same API as these plugins. .. _v0_28_publish_cloudrun: @@ -2039,7 +2039,7 @@ datasette publish cloudrun `Google Cloud Run <https://cloud.google.com/run/>`__ is a brand new serverless hosting platform from Google, which allows you to build a Docker container which will run only when HTTP traffic is received and will shut down (and hence cost you nothing) the rest of the time. It's similar to Zeit's Now v1 Docker hosting platform which sadly is `no longer accepting signups <https://hyperion.alpha.spectrum.chat/zeit/now/cannot-create-now-v1-deployments~d206a0d4-5835-4af5-bb5c-a17f0171fb25?m=MTU0Njk2NzgwODM3OA==>`__ from new users. -The new ``datasette publish cloudrun`` command was contributed by Romain Primet (`#434 <https://github.com/simonw/datasette/pull/434>`__) and publishes selected databases to a new Datasette instance running on Google Cloud Run. +The new ``datasette publish cloudrun`` command was contributed by Romain Primet (:pr:`434`) and publishes selected databases to a new Datasette instance running on Google Cloud Run. See :ref:`publish_cloud_run` for full documentation. @@ -2048,7 +2048,7 @@ See :ref:`publish_cloud_run` for full documentation. register_output_renderer plugins ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Russ Garrett implemented a new Datasette plugin hook called :ref:`register_output_renderer <plugin_register_output_renderer>` (`#441 <https://github.com/simonw/datasette/pull/441>`__) which allows plugins to create additional output renderers in addition to Datasette's default ``.json`` and ``.csv``. +Russ Garrett implemented a new Datasette plugin hook called :ref:`register_output_renderer <plugin_register_output_renderer>` (:pr:`441`) which allows plugins to create additional output renderers in addition to Datasette's default ``.json`` and ``.csv``. Russ's in-development `datasette-geo <https://github.com/russss/datasette-geo>`__ plugin includes `an example <https://github.com/russss/datasette-geo/blob/d4cecc020848bbde91e9e17bf352f7c70bc3dccf/datasette_plugin_geo/geojson.py>`__ of this hook being used to output ``.geojson`` automatically converted from SpatiaLite. @@ -2057,7 +2057,7 @@ Russ's in-development `datasette-geo <https://github.com/russss/datasette-geo>`_ Medium changes ~~~~~~~~~~~~~~ -- Datasette now conforms to the `Black coding style <https://github.com/python/black>`__ (`#449 <https://github.com/simonw/datasette/pull/449>`__) - and has a unit test to enforce this in the future +- Datasette now conforms to the `Black coding style <https://github.com/python/black>`__ (:pr:`449`) - and has a unit test to enforce this in the future - New :ref:`json_api_table_arguments`: - ``?columnname__in=value1,value2,value3`` filter for executing SQL IN queries against a table, see :ref:`table_arguments` (:issue:`433`) - ``?columnname__date=yyyy-mm-dd`` filter which returns rows where the spoecified datetime column falls on the specified date (`583b22a <https://github.com/simonw/datasette/commit/583b22aa28e26c318de0189312350ab2688c90b1>`__) @@ -2078,17 +2078,17 @@ Small changes - We now show the size of the database file next to the download link (:issue:`172`) - New ``/-/databases`` introspection page shows currently connected databases (:issue:`470`) -- Binary data is no longer displayed on the table and row pages (`#442 <https://github.com/simonw/datasette/pull/442>`__ - thanks, Russ Garrett) +- Binary data is no longer displayed on the table and row pages (:pr:`442` - thanks, Russ Garrett) - New show/hide SQL links on custom query pages (:issue:`415`) -- The :ref:`extra_body_script <plugin_hook_extra_body_script>` plugin hook now accepts an optional ``view_name`` argument (`#443 <https://github.com/simonw/datasette/pull/443>`__ - thanks, Russ Garrett) -- Bumped Jinja2 dependency to 2.10.1 (`#426 <https://github.com/simonw/datasette/pull/426>`__) +- The :ref:`extra_body_script <plugin_hook_extra_body_script>` plugin hook now accepts an optional ``view_name`` argument (:pr:`443` - thanks, Russ Garrett) +- Bumped Jinja2 dependency to 2.10.1 (:pr:`426`) - All table filters are now documented, and documentation is enforced via unit tests (`2c19a27 <https://github.com/simonw/datasette/commit/2c19a27d15a913e5f3dd443f04067169a6f24634>`__) - New project guideline: master should stay shippable at all times! (`31f36e1 <https://github.com/simonw/datasette/commit/31f36e1b97ccc3f4387c80698d018a69798b6228>`__) - Fixed a bug where ``sqlite_timelimit()`` occasionally failed to clean up after itself (`bac4e01 <https://github.com/simonw/datasette/commit/bac4e01f40ae7bd19d1eab1fb9349452c18de8f5>`__) - We no longer load additional plugins when executing pytest (:issue:`438`) - Homepage now links to database views if there are less than five tables in a database (:issue:`373`) - The ``--cors`` option is now respected by error pages (:issue:`453`) -- ``datasette publish heroku`` now uses the ``--include-vcs-ignore`` option, which means it works under Travis CI (`#407 <https://github.com/simonw/datasette/pull/407>`__) +- ``datasette publish heroku`` now uses the ``--include-vcs-ignore`` option, which means it works under Travis CI (:pr:`407`) - ``datasette publish heroku`` now publishes using Python 3.6.8 (`666c374 <https://github.com/simonw/datasette/commit/666c37415a898949fae0437099d62a35b1e9c430>`__) - Renamed ``datasette publish now`` to ``datasette publish nowv1`` (:issue:`472`) - ``datasette publish nowv1`` now accepts multiple ``--alias`` parameters (`09ef305 <https://github.com/simonw/datasette/commit/09ef305c687399384fe38487c075e8669682deb4>`__) diff --git a/docs/conf.py b/docs/conf.py index 0879eeb9..5dd06b57 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,6 +51,7 @@ markdown_uri_doc_suffix = ".html" extlinks = { "issue": ("https://github.com/simonw/datasette/issues/%s", "#%s"), + "pr": ("https://github.com/simonw/datasette/pull/%s", "#%s"), } # Add any paths that contain templates here, relative to this directory. From 1f99d5dd20cb8d422a96bc028e5a6e51c696e162 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 15 Apr 2026 16:11:54 -0700 Subject: [PATCH 030/156] Release 1.0a27 Refs #1936, #2678, #2681, #2682, #2683, #2684, #2688, #2689 --- datasette/version.py | 2 +- docs/changelog.rst | 29 ++++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index add192f6..e2c80e50 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a26" +__version__ = "1.0a27" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index cff580da..9cd7a7d6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,12 +3,35 @@ ========= Changelog ========= -.. dev: -dev ---- +.. _v1_0_a27: +1.0a27 (2026-04-15) +------------------- + +CSRF protection no longer uses CSRF tokens +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Datasette's token-based CSRF protection has been replaced with a mechanism based on the ``Sec-Fetch-Site`` and ``Origin`` request headers, which are `supported by all modern browsers <https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Sec-Fetch-Site>`__. See `this article by Filippo Valsorda <https://words.filippo.io/csrf/>`__ for more details of this approach. This removes the need for CSRF tokens in forms and AJAX requests. (:pr:`2689`) + +``RenameTableEvent`` when a table is renamed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Renaming a table within Datasette will now fire a new :class:`~datasette.events.RenameTableEvent`, which plugins can use to react by updating ACL records or re-assigning comments or other associated records to the new table name. (:issue:`2681`) + +This event will not be fired if the table is renamed by SQL running in some other process. + +The ``datasette.track_event()`` method can now be called from within a write operation (using :ref:`database.execute_write() <database_execute_write>` and related methods) and the event will be fired after the write transaction has successfully committed. (:pr:`2682`) + +Other changes +~~~~~~~~~~~~~ + +- New :ref:`actor= parameter <internals_datasette_client_actor>` for ``datasette.client`` methods, allowing internal requests to be made as a specific actor. This is particularly useful for writing automated tests. (:pr:`2688`) +- New ``Database(is_temp_disk=True)`` option, used internally for the internal database. This helps resolve intermittent database locked errors caused by the internal database being in-memory as opposed to on-disk. (:issue:`2683`) (:pr:`2684`) +- The ``/<database>/<table>/-/upsert`` API (:ref:`docs <TableUpsertView>`) now rejects rows with ``null`` primary key values. (:issue:`1936`) +- Improved example in the API explorer for the ``/-/upsert`` endpoint (:ref:`docs <TableUpsertView>`). (:issue:`1936`) - The ``/<database>.json`` endpoint now includes an ``"ok": true`` key, for consistency with other JSON API responses. +- :ref:`call_with_supported_arguments() <internals_utils_call_with_supported_arguments>` is now documented as a supported public API. (:pr:`2678`) .. _v1_0_a26: From 2638200d26b07701108fa6275e35c7c011535e4c Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 15 Apr 2026 17:19:43 -0700 Subject: [PATCH 031/156] Link to datasette.io preview tool --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 635ca60e..5a109fda 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -296,7 +296,7 @@ Don't forget to create the release from the correct branch - usually ``main``, b While the release is running you can confirm that the correct commits made it into the release using the https://github.com/simonw/datasette/compare/0.64.6...0.64.7 URL. -Finally, post a news item about the release on `datasette.io <https://datasette.io/>`__ by editing the `news.yaml <https://github.com/simonw/datasette.io/blob/main/news.yaml>`__ file in that site's repository. +Finally, post a news item about the release on `datasette.io <https://datasette.io/>`__ by editing the `news.yaml <https://github.com/simonw/datasette.io/blob/main/news.yaml>`__ file in that site's repository. Use `this preview tool <https://tools.simonwillison.net/datasette-io-preview>`__ to preview the edits to the YAML. .. _contributing_alpha_beta: From ade0ef8a60bad2a3e659d1cf1581bfe1fa96e289 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 19:14:32 -0700 Subject: [PATCH 032/156] Restore compatibility with existing execute_write_fn() callbacks Closes #2691 --- datasette/database.py | 19 +++++++++++++------ tests/test_internals_database.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 8b824462..7364ff7f 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,6 +1,7 @@ import asyncio import atexit from collections import namedtuple +import inspect import os from pathlib import Path import janus @@ -263,15 +264,21 @@ class Database: def _wrap_fn_with_hooks(self, fn, request, transaction, track_event): from .plugins import pm - # Wrap fn so it receives track_event if its signature supports it + # Wrap fn so it receives track_event if its signature supports it. + # Historically fn was called positionally, so any single-parameter + # name (conn, connection, db, ...) worked. Preserve that by only + # switching to keyword dependency injection when the callback + # explicitly opts in by declaring a `track_event` parameter. original_fn = fn - def fn_with_track_event(conn): - return call_with_supported_arguments( - original_fn, conn=conn, track_event=track_event - ) + if "track_event" in inspect.signature(original_fn).parameters: - fn = fn_with_track_event + def fn_with_track_event(conn): + return call_with_supported_arguments( + original_fn, conn=conn, track_event=track_event + ) + + fn = fn_with_track_event wrappers = pm.hook.write_wrapper( datasette=self.ds, diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index e3d35f57..0d565d61 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -539,6 +539,37 @@ async def test_execute_write_fn_exception(db): await db.execute_write_fn(write_fn) +@pytest.mark.asyncio +@pytest.mark.parametrize("param_name", ["conn", "connection", "db", "c"]) +async def test_execute_write_fn_accepts_any_single_param_name(db, param_name): + # Plugins historically relied on the fact that the callback was invoked + # positionally, so any parameter name worked. Preserve that contract. + scope = {} + exec( + "def write_fn({0}):\n" + " return {0}.execute('select 1 + 1').fetchone()[0]".format(param_name), + scope, + ) + write_fn = scope["write_fn"] + result = await db.execute_write_fn(write_fn) + assert result == 2 + + +@pytest.mark.asyncio +async def test_execute_write_fn_with_track_event(db): + # When the callback declares track_event it still receives both args + # via dependency injection. + seen = [] + + def write_fn(conn, track_event): + seen.append(track_event) + return conn.execute("select 1 + 1").fetchone()[0] + + result = await db.execute_write_fn(write_fn) + assert result == 2 + assert len(seen) == 1 and callable(seen[0]) + + @pytest.mark.asyncio @pytest.mark.timeout(1) async def test_execute_write_fn_connection_exception(tmpdir, app_client): From dabf8e4199cd4598697e538c495cc66aa429a262 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:08:46 -0700 Subject: [PATCH 033/156] Database.close() shuts down write thread and raises DatasetteClosedError After this commit, Database.close() sends a sentinel to the write queue so the background write thread exits cleanly, closes cached read/write connections, and marks the instance closed. Subsequent calls to execute*() raise DatasetteClosedError. close() remains idempotent and one-way. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- datasette/database.py | 79 +++++++++++++++++++++++++++++++- docs/internals.rst | 6 ++- tests/test_internals_database.py | 56 ++++++++++++++++++++++ 3 files changed, 138 insertions(+), 3 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 7364ff7f..e3c4bfec 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -34,6 +34,13 @@ connections = threading.local() AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) +class DatasetteClosedError(RuntimeError): + """Raised when using a Datasette or Database instance after close().""" + + +_SHUTDOWN = object() + + class Database: # For table counts stop at this many rows: count_limit = 10000 @@ -76,6 +83,7 @@ class Database: self._cached_table_counts = None self._write_thread = None self._write_queue = None + self._closed = False # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None @@ -84,6 +92,12 @@ class Database: if not is_temp_disk: self.mode = mode + def _check_not_closed(self): + if self._closed: + raise DatasetteClosedError( + "Database {!r} has been closed".format(self.name) + ) + @property def cached_table_counts(self): if self._cached_table_counts is not None: @@ -149,9 +163,53 @@ class Database: return conn def close(self): - # Close all connections - useful to avoid running out of file handles in tests + """Release all resources held by this database. + + Idempotent. After close() further calls to execute()/execute_fn()/ + execute_write()/execute_write_fn() raise DatasetteClosedError. + """ + if self._closed: + return + self._closed = True + # Shut down the write thread, if any, via a sentinel. The thread + # drains any writes already queued before the sentinel and then + # closes its own write connection and returns. + write_thread = self._write_thread + if write_thread is not None and self._write_queue is not None: + self._write_queue.put(_SHUTDOWN) + write_thread.join(timeout=10) + if write_thread.is_alive(): + sys.stderr.write( + "Datasette: write thread for {!r} did not exit within 10s\n".format( + self.name + ) + ) + sys.stderr.flush() + # Close anything still tracked in _all_file_connections for connection in self._all_file_connections: - connection.close() + try: + connection.close() + except Exception: + pass + self._all_file_connections = [] + # Drop per-thread cached read connections we can reach + try: + delattr(connections, self._thread_local_id) + except AttributeError: + pass + # Close non-threaded-mode cached connections if still open + if self._read_connection is not None: + try: + self._read_connection.close() + except Exception: + pass + self._read_connection = None + if self._write_connection is not None: + try: + self._write_connection.close() + except Exception: + pass + self._write_connection = None if self.is_temp_disk: self._cleanup_temp_file() @@ -164,6 +222,8 @@ class Database: pass async def execute_write(self, sql, params=None, block=True, request=None): + self._check_not_closed() + def _inner(conn): return conn.execute(sql, params or []) @@ -172,6 +232,8 @@ class Database: return results async def execute_write_script(self, sql, block=True, request=None): + self._check_not_closed() + def _inner(conn): return conn.executescript(sql) @@ -182,6 +244,8 @@ class Database: return results async def execute_write_many(self, sql, params_seq, block=True, request=None): + self._check_not_closed() + def _inner(conn): count = 0 @@ -203,6 +267,7 @@ class Database: return results async def execute_isolated_fn(self, fn): + self._check_not_closed() # Open a new connection just for the duration of this function # blocking the write queue to avoid any writes occurring during it if self.ds.executor is None: @@ -223,6 +288,7 @@ class Database: return await self._send_to_write_thread(fn, isolated_connection=True) async def execute_write_fn(self, fn, block=True, transaction=True, request=None): + self._check_not_closed() pending_events = [] def track_event(event): @@ -334,6 +400,13 @@ class Database: conn_exception = e while True: task = self._write_queue.get() + if task is _SHUTDOWN: + if conn is not None: + try: + conn.close() + except Exception: + pass + return if conn_exception is not None: result = conn_exception else: @@ -366,6 +439,7 @@ class Database: task.reply_queue.sync_q.put(result) async def execute_fn(self, fn): + self._check_not_closed() if self.ds.executor is None: # non-threaded mode if self._read_connection is None: @@ -396,6 +470,7 @@ class Database: log_sql_errors=True, ): """Executes sql against db_name in a thread""" + self._check_not_closed() page_size = page_size or self.ds.page_size def sql_operation_in_thread(conn): diff --git a/docs/internals.rst b/docs/internals.rst index ba9d3131..53c20106 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1830,7 +1830,11 @@ The return value of the function will be returned by this method. Any exceptions db.close() ---------- -Closes all of the open connections to file-backed databases. This is mainly intended to be used by large test suites, to avoid hitting limits on the number of open files. +Release all resources held by this ``Database`` instance. This shuts down the background write thread (if one was started by a previous call to :ref:`database_execute_write_fn` or similar), closes the write connection, and closes any cached read connections. + +After ``db.close()`` has been called, any further call to :ref:`database_execute`, :ref:`database_execute_fn`, :ref:`database_execute_write`, :ref:`database_execute_write_fn`, :ref:`database_execute_write_many`, :ref:`database_execute_write_script` or :ref:`database_execute_isolated_fn` will raise a ``datasette.database.DatasetteClosedError`` exception. + +``close()`` is idempotent — calling it a second time is a no-op. It is one-way: a closed ``Database`` cannot be reopened. .. _internals_database_introspection: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 0d565d61..8ff74a83 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -4,6 +4,7 @@ Tests for the datasette.database.Database class from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues +from datasette.database import DatasetteClosedError from datasette.utils.sqlite import sqlite3, sqlite_version from datasette.utils import Column import pytest @@ -833,3 +834,58 @@ def test_repr_temp_disk(app_client): assert isinstance(db.size, int) assert isinstance(db.mtime_ns, int) db.close() + + +@pytest.mark.asyncio +async def test_database_close_shuts_down_write_thread(tmpdir): + path = str(tmpdir / "dbclose.db") + conn = sqlite3.connect(path) + conn.execute("create table t (id integer primary key)") + conn.close() + ds = Datasette([path]) + db = ds.get_database("dbclose") + # Trigger write thread creation + await db.execute_write("insert into t (id) values (1)") + assert db._write_thread is not None + assert db._write_thread.is_alive() + db.close() + # Wait briefly for the thread to exit — the sentinel should cause it to return. + db._write_thread.join(timeout=5) + assert not db._write_thread.is_alive() + ds._internal_database.close() + + +@pytest.mark.asyncio +async def test_database_close_raises_on_further_use(tmpdir): + path = str(tmpdir / "closed.db") + conn = sqlite3.connect(path) + conn.execute("create table t (id integer primary key)") + conn.close() + ds = Datasette([path]) + db = ds.get_database("closed") + await db.execute("select 1") + db.close() + with pytest.raises(DatasetteClosedError): + await db.execute("select 1") + with pytest.raises(DatasetteClosedError): + await db.execute_write("insert into t (id) values (1)") + with pytest.raises(DatasetteClosedError): + await db.execute_fn(lambda conn: conn.execute("select 1").fetchone()) + with pytest.raises(DatasetteClosedError): + await db.execute_write_fn(lambda conn: conn.execute("select 1")) + ds._internal_database.close() + + +@pytest.mark.asyncio +async def test_database_close_is_idempotent(tmpdir): + path = str(tmpdir / "idemp.db") + conn = sqlite3.connect(path) + conn.execute("create table t (id integer primary key)") + conn.close() + ds = Datasette([path]) + db = ds.get_database("idemp") + await db.execute_write("insert into t (id) values (1)") + db.close() + # Second call should be a no-op, not raise + db.close() + ds._internal_database.close() From 290f27158f1b53a9a90a36dbfb271ffbb6eef310 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:10:18 -0700 Subject: [PATCH 034/156] Datasette.close() closes databases, shuts down executor, unlinks temp file Datasette.close() iterates over every attached Database (including the internal database), calls Database.close() on each, then shuts down the ThreadPoolExecutor. Exceptions raised by one Database don't prevent the others from being closed; the first exception is re-raised afterwards. Idempotent. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- datasette/app.py | 28 +++++++++++++++ docs/internals.rst | 13 +++++++ tests/test_internals_datasette.py | 59 +++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 0f417ec9..367f38f9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -326,6 +326,7 @@ class Datasette: default_deny=False, ): self._startup_invoked = False + self._closed = False assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" @@ -834,6 +835,33 @@ class Datasette: new_databases.pop(name) self.databases = new_databases + def close(self): + """Release all resources held by this Datasette instance. + + Closes every attached Database (including the internal database), + shuts down the executor, and unlinks the temporary file used for + the internal database if one was created. Idempotent and one-way. + """ + if self._closed: + return + self._closed = True + first_exception = None + dbs = list(self.databases.values()) + [self._internal_database] + for db in dbs: + try: + db.close() + except Exception as e: + if first_exception is None: + first_exception = e + if self.executor is not None: + try: + self.executor.shutdown(wait=True, cancel_futures=True) + except Exception as e: + if first_exception is None: + first_exception = e + if first_exception is not None: + raise first_exception + def setting(self, key): return self._settings.get(key, None) diff --git a/docs/internals.rst b/docs/internals.rst index 53c20106..2710345b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1079,6 +1079,19 @@ The ``name`` and ``route`` parameters are optional and work the same way as they This removes a database that has been previously added. ``name=`` is the unique name of that database. +.. _datasette_close: + +.close() +-------- + +Release all resources held by this ``Datasette`` instance. This calls :ref:`database_close` on every attached database (including the internal database), shuts down the thread pool executor used to run SQL queries, and unlinks the temporary file used to back the internal database if one was created. + +``close()`` is synchronous, idempotent and one-way: after a call to ``close()`` any attempt to use the Datasette instance to execute SQL will raise a ``datasette.database.DatasetteClosedError`` exception. A closed ``Datasette`` cannot be reopened — callers that need a fresh instance should construct a new one. + +If a call to ``Database.close()`` on one of the attached databases raises an exception, ``Datasette.close()`` will continue trying to close the remaining databases and will re-raise the first exception after every database has been processed. + +When Datasette is being served over ASGI the ``close()`` method is wired up to the lifespan shutdown event, so resources are released cleanly on ``SIGTERM`` / ``SIGINT``. + .. _datasette_track_event: await .track_event(event) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index ec0180a7..5f773658 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -3,8 +3,10 @@ Tests for the datasette.app.Datasette class """ import dataclasses +import os from datasette import Context from datasette.app import Datasette, Database, ResourcesSQL +from datasette.database import DatasetteClosedError from datasette.resources import DatabaseResource from itsdangerous import BadSignature import pytest @@ -213,3 +215,60 @@ async def test_allowed_resources_sql(datasette): assert isinstance(result, ResourcesSQL) assert "all_rules AS" in result.sql assert result.params["action"] == "view-table" + + +@pytest.mark.asyncio +async def test_datasette_close_closes_all_databases_and_executor(): + ds = Datasette(memory=True) + await ds.invoke_startup() + # Confirm internal DB has write machinery running + assert ds._internal_database._write_thread is not None + assert ds._internal_database._write_thread.is_alive() + temp_path = ds._internal_database.path + assert os.path.exists(temp_path) + executor = ds.executor + ds.close() + # Executor is shut down + assert executor._shutdown + # All attached Database instances are closed + for db in ds.databases.values(): + assert db._closed + assert ds._internal_database._closed + # Temp internal DB file is unlinked + assert not os.path.exists(temp_path) + + +@pytest.mark.asyncio +async def test_datasette_close_is_idempotent(): + ds = Datasette(memory=True) + await ds.invoke_startup() + ds.close() + # Second call should be a no-op + ds.close() + + +@pytest.mark.asyncio +async def test_datasette_close_raises_on_use(): + ds = Datasette(memory=True) + await ds.invoke_startup() + ds.close() + with pytest.raises(DatasetteClosedError): + await ds.get_internal_database().execute("select 1") + + +@pytest.mark.asyncio +async def test_datasette_close_continues_past_db_error(): + # If one Database raises during close(), the others still get closed. + ds = Datasette(memory=True) + await ds.invoke_startup() + + class Boom(Database): + def close(self): + raise RuntimeError("boom") + + bad = ds.add_database(Boom(ds, is_memory=True), name="bad") + good = ds.add_database(Database(ds, is_memory=True), name="good") + with pytest.raises(RuntimeError, match="boom"): + ds.close() + assert good._closed + assert ds._internal_database._closed From d72dd3537850988fb24cd53d50c690dc7acb4332 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:11:02 -0700 Subject: [PATCH 035/156] Wire Datasette.close into ASGI lifespan shutdown AsgiLifespan now receives an on_shutdown callback that invokes Datasette.close(), so resources are released cleanly when the ASGI server delivers a lifespan.shutdown message (SIGTERM / SIGINT for uvicorn). Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- datasette/app.py | 5 ++++- tests/test_internals_datasette.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index 367f38f9..358081ef 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2338,10 +2338,13 @@ class Datasette: if not database.is_mutable: await database.table_counts(limit=60 * 60 * 1000) + async def _close_on_shutdown(): + self.close() + asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan(asgi) + asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 5f773658..11463eda 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -256,6 +256,29 @@ async def test_datasette_close_raises_on_use(): await ds.get_internal_database().execute("select 1") +@pytest.mark.asyncio +async def test_asgi_lifespan_shutdown_closes_datasette(): + ds = Datasette(memory=True) + app = ds.app() + # Drive an ASGI lifespan: startup, then shutdown. + messages_sent = [] + inbox = [ + {"type": "lifespan.startup"}, + {"type": "lifespan.shutdown"}, + ] + + async def receive(): + return inbox.pop(0) + + async def send(message): + messages_sent.append(message) + + await app({"type": "lifespan"}, receive, send) + assert {"type": "lifespan.startup.complete"} in messages_sent + assert {"type": "lifespan.shutdown.complete"} in messages_sent + assert ds._closed + + @pytest.mark.asyncio async def test_datasette_close_continues_past_db_error(): # If one Database raises during close(), the others still get closed. From 34cc320eabb09d7d62f8a6045b868c746adfe9d2 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:15:50 -0700 Subject: [PATCH 036/156] Pytest auto-close plugin for Datasette instances Installs a pytest11 entry point so that every Datasette() constructed inside a pytest_runtest_call phase is auto-closed at the end of the test. Fixture-scoped instances are untouched. Opt out via the datasette_autoclose = false ini option. This gives large test suites a safety net against FD exhaustion and leaked write threads from the now-default temp-disk internal database without requiring every existing test to be rewritten. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- datasette/_pytest_plugin.py | 78 ++++++++++++++++++++++ docs/testing_plugins.rst | 16 +++++ pyproject.toml | 3 + tests/test_pytest_autoclose_plugin.py | 93 +++++++++++++++++++++++++++ 4 files changed, 190 insertions(+) create mode 100644 datasette/_pytest_plugin.py create mode 100644 tests/test_pytest_autoclose_plugin.py diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py new file mode 100644 index 00000000..3f6c0d96 --- /dev/null +++ b/datasette/_pytest_plugin.py @@ -0,0 +1,78 @@ +""" +Pytest plugin that automatically closes any Datasette instances constructed +inside a test body. Fixture-scoped instances survive. + +Registered as a pytest11 entry point in pyproject.toml so that downstream +projects using Datasette get the same FD-safety net for their own tests. + +Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the +equivalent ini file). +""" + +from __future__ import annotations + +import contextvars +import weakref + +import pytest + +from datasette.app import Datasette + +_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( + "datasette_active_instances", default=None +) + +_original_init = Datasette.__init__ + + +def _tracking_init(self, *args, **kwargs): + _original_init(self, *args, **kwargs) + instances = _active_instances.get() + if instances is not None: + instances.append(weakref.ref(self)) + + +Datasette.__init__ = _tracking_init + + +def pytest_addoption(parser): + parser.addini( + "datasette_autoclose", + help=( + "Automatically close Datasette instances created inside test " + "bodies (default: true)." + ), + default="true", + ) + + +def _enabled(config) -> bool: + value = config.getini("datasette_autoclose") + if isinstance(value, bool): + return value + return str(value).strip().lower() not in ("false", "0", "no", "off") + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_call(item): + if not _enabled(item.config): + yield + return + refs: list[weakref.ref] = [] + token = _active_instances.set(refs) + try: + yield + finally: + _active_instances.reset(token) + for ref in reversed(refs): + ds = ref() + if ds is None: + continue + try: + ds.close() + except Exception as e: + item.warn( + pytest.PytestUnraisableExceptionWarning( + f"Error closing Datasette instance: {e!r}" + ) + ) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 1b10c132..070ab6cf 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -82,6 +82,22 @@ This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepar If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request. +.. _testing_plugins_autoclose: + +Automatic cleanup of Datasette instances +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Installing Datasette also installs a small pytest plugin that automatically calls :ref:`datasette_close` on any ``Datasette()`` instance constructed inside the body of a test function. This helps prevent large test suites from running out of file descriptors or leaking background threads from the hundreds of instances they may build up across a session. + +Instances created inside a pytest fixture are **not** closed by this plugin — pytest fixtures often create a single ``Datasette`` that is shared across many tests, and closing it automatically would break those tests. If you need a per-test instance and want to share it between multiple tests, create it inside a fixture rather than at the top level of a test function. + +If you need to opt out of this behavior, add the following to your ``pytest.ini`` (or equivalent): + +.. code-block:: ini + + [pytest] + datasette_autoclose = false + .. _testing_datasette_client: Using datasette.client in tests diff --git a/pyproject.toml b/pyproject.toml index a0ee050c..4a4ed75e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,9 @@ CI = "https://github.com/simonw/datasette/actions?query=workflow%3ATest" [project.scripts] datasette = "datasette.cli:cli" +[project.entry-points.pytest11] +datasette = "datasette._pytest_plugin" + [dependency-groups] dev = [ "pytest>=9", diff --git a/tests/test_pytest_autoclose_plugin.py b/tests/test_pytest_autoclose_plugin.py new file mode 100644 index 00000000..78154ef5 --- /dev/null +++ b/tests/test_pytest_autoclose_plugin.py @@ -0,0 +1,93 @@ +""" +Tests for datasette._pytest_plugin — the pytest plugin that auto-closes +Datasette instances constructed inside test bodies. + +These tests drive a real pytest session in a subprocess so the plugin +operates exactly as it would for a downstream consumer. +""" + +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parent.parent + + +def _run_pytest(tmp_path: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-m", "pytest", "-v", str(tmp_path)], + cwd=str(tmp_path), + capture_output=True, + text=True, + ) + + +def test_auto_close_of_instances_made_in_test_body(tmp_path): + # Two ordered tests: + # test_a makes a Datasette() and stashes a hard reference + # test_b asserts that the hard-reffed instance was closed by the plugin + (tmp_path / "test_sample.py").write_text(textwrap.dedent(""" + from datasette.app import Datasette + + _stash = {} + + def test_a(): + ds = Datasette(memory=True) + _stash["ds"] = ds + assert ds._closed is False + + def test_b(): + assert _stash["ds"]._closed is True + """)) + result = _run_pytest(tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_fixture_scoped_instance_is_not_closed(tmp_path): + # A module-scoped fixture instance must survive across tests in the module. + (tmp_path / "test_fixture.py").write_text(textwrap.dedent(""" + import pytest + from datasette.app import Datasette + + @pytest.fixture(scope="module") + def ds(): + return Datasette(memory=True) + + def test_first(ds): + assert ds._closed is False + + def test_second(ds): + # Still alive because the plugin only tracks instances + # constructed during pytest_runtest_call, not during fixture + # setup. + assert ds._closed is False + """)) + result = _run_pytest(tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_opt_out_via_ini(tmp_path): + # datasette_autoclose = false should leave instances untouched. + (tmp_path / "pytest.ini").write_text(textwrap.dedent(""" + [pytest] + datasette_autoclose = false + """).strip()) + (tmp_path / "test_optout.py").write_text(textwrap.dedent(""" + from datasette.app import Datasette + + _stash = {} + + def test_a(): + ds = Datasette(memory=True) + _stash["ds"] = ds + + def test_b(): + # Opt-out: plugin must not have closed it. + assert _stash["ds"]._closed is False + _stash["ds"].close() + """)) + result = _run_pytest(tmp_path) + assert result.returncode == 0, result.stdout + result.stderr From c0153386ef20126a289da96204718570d571b4b2 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:18:05 -0700 Subject: [PATCH 037/156] FD-leak regression test for Datasette.close() Creates and disposes 50 Datasette instances in a loop and asserts that the number of open file descriptors and live threads does not grow, exercising the full close() path end to end. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- pyproject.toml | 1 + tests/test_fd_leak.py | 56 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tests/test_fd_leak.py diff --git a/pyproject.toml b/pyproject.toml index 4a4ed75e..e6007afd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ dev = [ "myst-parser", "sphinx-markdown-builder", "ruamel.yaml", + "psutil>=5.9", ] [project.optional-dependencies] diff --git a/tests/test_fd_leak.py b/tests/test_fd_leak.py new file mode 100644 index 00000000..926722a1 --- /dev/null +++ b/tests/test_fd_leak.py @@ -0,0 +1,56 @@ +""" +Regression test for https://github.com/simonw/datasette/issues/2692 — +confirm that creating and closing Datasette instances in a loop does not +leak open file descriptors. + +Each Datasette() with is_temp_disk internal DB opens a temp file and a +write thread with its own SQLite connection. Without Datasette.close() +nothing unwinds this state, and a large pytest run exhausts the process +FD limit. +""" + +import asyncio +import threading + +import pytest + +try: + import psutil +except ImportError: # pragma: no cover + psutil = None + +from datasette.app import Datasette + + +def _count_open_files(): + return len(psutil.Process().open_files()) + + +def _count_threads(): + return threading.active_count() + + +@pytest.mark.skipif(psutil is None, reason="psutil not installed") +def test_close_releases_file_descriptors(): + # Warm-up so Python/library caches don't skew the baseline + ds = Datasette(memory=True) + asyncio.run(ds.invoke_startup()) + ds.close() + + baseline_fds = _count_open_files() + baseline_threads = _count_threads() + + for _ in range(50): + ds = Datasette(memory=True) + asyncio.run(ds.invoke_startup()) + ds.close() + + after_fds = _count_open_files() + after_threads = _count_threads() + + assert ( + after_fds - baseline_fds <= 2 + ), f"Leaked FDs: baseline={baseline_fds}, after=50 iterations={after_fds}" + assert ( + after_threads - baseline_threads <= 2 + ), f"Leaked threads: baseline={baseline_threads}, after={after_threads}" From d23b32c3e57469f0c0de149aee8594205dfdb319 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:25:58 -0700 Subject: [PATCH 038/156] Call ds.close() in more places in tests Refs #2692 --- tests/test_api_write.py | 7 +------ tests/test_column_types.py | 10 ++-------- tests/test_docs_plugins.py | 1 + tests/test_write_wrapper.py | 1 + 4 files changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 9ba08848..64f91701 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -22,12 +22,7 @@ def ds_write(tmp_path_factory): ds = Datasette([db_path], immutables=[db_path_immutable]) ds.root_enabled = True yield ds - # Close both setup connections plus any Datasette-managed connections. - db1.close() - db2.close() - for database in ds.databases.values(): - if not database.is_memory: - database.close() + ds.close() def write_token(ds, actor_id="root", permissions=None): diff --git a/tests/test_column_types.py b/tests/test_column_types.py index 6e89acb9..d77f2cf5 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -52,10 +52,7 @@ def ds_ct(tmp_path_factory): ) ds.root_enabled = True yield ds - db.close() - for database in ds.databases.values(): - if not database.is_memory: - database.close() + ds.close() @pytest.fixture @@ -95,10 +92,7 @@ def ds_ct_editor_permission(tmp_path_factory): ) ds.root_enabled = True yield ds - db.close() - for database in ds.databases.values(): - if not database.is_memory: - database.close() + ds.close() def write_token(ds, actor_id="root", permissions=None): diff --git a/tests/test_docs_plugins.py b/tests/test_docs_plugins.py index c51858d3..613160ac 100644 --- a/tests/test_docs_plugins.py +++ b/tests/test_docs_plugins.py @@ -23,6 +23,7 @@ async def datasette_with_plugin(): yield datasette finally: datasette.pm.unregister(name="undo") + datasette.close() # -- end datasette_with_plugin_fixture -- diff --git a/tests/test_write_wrapper.py b/tests/test_write_wrapper.py index c2ceb344..48c964b4 100644 --- a/tests/test_write_wrapper.py +++ b/tests/test_write_wrapper.py @@ -505,6 +505,7 @@ def ds_with_event_tracking(tmp_path): ds.track_event = recording_track_event yield ds + ds.close() @pytest.mark.asyncio From df96e12737454077c707f469506be0ee96091965 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:32:19 -0700 Subject: [PATCH 039/156] Auto-close Datasette instances from function-scoped fixtures too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plugin now tracks instances across the full test protocol (setup, call, teardown) and closes all of them at the end — including ones created inside function-scoped pytest fixtures. Session-, module-, class- and package-scoped fixtures are still exempted by subtracting any instances their setup adds from the tracking list. This makes downstream projects like datasette-alerts work at low FD limits without every fixture needing an explicit ds.close() call. Refs #2692 See https://github.com/simonw/datasette/issues/2692#issuecomment-4265072230 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- datasette/_pytest_plugin.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 3f6c0d96..5fb6b473 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -1,6 +1,9 @@ """ Pytest plugin that automatically closes any Datasette instances constructed -inside a test body. Fixture-scoped instances survive. +during a pytest test — both in the test body and in function-scoped +fixtures. Instances constructed by session-, module-, class- or package- +scoped fixtures are left alone, because other tests in the session will +still want to use them. Registered as a pytest11 entry point in pyproject.toml so that downstream projects using Datasette get the same FD-safety net for their own tests. @@ -40,7 +43,7 @@ def pytest_addoption(parser): "datasette_autoclose", help=( "Automatically close Datasette instances created inside test " - "bodies (default: true)." + "bodies and function-scoped fixtures (default: true)." ), default="true", ) @@ -54,7 +57,8 @@ def _enabled(config) -> bool: @pytest.hookimpl(hookwrapper=True) -def pytest_runtest_call(item): +def pytest_runtest_protocol(item, nextitem): + """Track Datasette instances across setup, call and teardown; close at end.""" if not _enabled(item.config): yield return @@ -76,3 +80,29 @@ def pytest_runtest_call(item): f"Error closing Datasette instance: {e!r}" ) ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_setup(fixturedef, request): + """Exempt instances created by non-function-scoped fixtures. + + Session-, module-, class- and package-scoped fixtures produce Datasette + instances that must survive beyond the current test — other tests in + the session will still use them. When such a fixture creates one or + more Datasette instances during its setup, we snapshot the tracking + list before the fixture runs and subtract off any instances that were + added during its setup, so they don't get closed at test teardown. + """ + refs = _active_instances.get() + if refs is None: + yield + return + before_ids = {id(ref) for ref in refs} + yield + if fixturedef.scope != "function": + new_refs = [ref for ref in refs if id(ref) not in before_ids] + for new_ref in new_refs: + try: + refs.remove(new_ref) + except ValueError: + pass From ede942a32e65191ccf554d481987f2d42f4a9a92 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:34:48 -0700 Subject: [PATCH 040/156] Fix ruff lints in close-related tests Drop unused `bad = ...` assignment and unused `import pytest`. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- tests/test_internals_datasette.py | 2 +- tests/test_pytest_autoclose_plugin.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 11463eda..d58c9a29 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -289,7 +289,7 @@ async def test_datasette_close_continues_past_db_error(): def close(self): raise RuntimeError("boom") - bad = ds.add_database(Boom(ds, is_memory=True), name="bad") + ds.add_database(Boom(ds, is_memory=True), name="bad") good = ds.add_database(Database(ds, is_memory=True), name="good") with pytest.raises(RuntimeError, match="boom"): ds.close() diff --git a/tests/test_pytest_autoclose_plugin.py b/tests/test_pytest_autoclose_plugin.py index 78154ef5..3af1aace 100644 --- a/tests/test_pytest_autoclose_plugin.py +++ b/tests/test_pytest_autoclose_plugin.py @@ -11,8 +11,6 @@ import sys import textwrap from pathlib import Path -import pytest - REPO_ROOT = Path(__file__).parent.parent From 03eeeb9d92e3821611931d9fa259811d95b646e8 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:38:08 -0700 Subject: [PATCH 041/156] Docs: auto-close plugin now handles function-scoped fixtures Describe the updated scoping rule: instances from test bodies and function-scoped fixtures are closed automatically; session-, module-, class- and package-scoped fixtures are exempt. Refs #2692 --- docs/testing_plugins.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 070ab6cf..b82a6e0c 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -87,9 +87,18 @@ If you are using ``await datasette.client.get()`` and similar methods then you d Automatic cleanup of Datasette instances ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Installing Datasette also installs a small pytest plugin that automatically calls :ref:`datasette_close` on any ``Datasette()`` instance constructed inside the body of a test function. This helps prevent large test suites from running out of file descriptors or leaking background threads from the hundreds of instances they may build up across a session. +Installing Datasette also installs a small pytest plugin that automatically calls :ref:`datasette_close` on any ``Datasette()`` instance constructed during a test. This helps prevent large test suites from running out of file descriptors or leaking background threads from the hundreds of instances they may build up across a session. -Instances created inside a pytest fixture are **not** closed by this plugin — pytest fixtures often create a single ``Datasette`` that is shared across many tests, and closing it automatically would break those tests. If you need a per-test instance and want to share it between multiple tests, create it inside a fixture rather than at the top level of a test function. +The plugin closes: + +- Instances created in the body of a test function. +- Instances created inside **function-scoped** pytest fixtures (the default scope — ``@pytest.fixture`` with no ``scope=`` argument, or ``scope="function"``). + +The plugin deliberately does **not** close: + +- Instances created inside higher-scoped fixtures (``scope="session"``, ``"module"``, ``"class"`` or ``"package"``). Those fixtures are typically designed to produce a single ``Datasette`` that is shared across many tests, and closing it automatically would break the tests that run after the first. + +In practice this means downstream projects rarely need to call ``ds.close()`` themselves — function-scoped fixtures and inline test code are both covered automatically, while long-lived shared fixtures keep working as before. If you need to opt out of this behavior, add the following to your ``pytest.ini`` (or equivalent): From c9a7dc9be21f586e86ae09e3e55376c5f9a5fd25 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:40:51 -0700 Subject: [PATCH 042/156] Declare ds_client as session-scoped so auto-close plugin spares it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ds_client already caches a single Datasette for the whole session via a module-level _ds_client global, so the declared fixture scope should match. With function scope the auto-close plugin correctly closes it after the first test that uses it, which then breaks every subsequent test that reuses the cached (now-closed) instance — as seen in the CI coverage job, which runs serially rather than under pytest-xdist. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3a3203fd..171a5433 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -53,7 +53,7 @@ def bare_ds(): return Datasette(memory=True) -@pytest_asyncio.fixture +@pytest_asyncio.fixture(scope="session") async def ds_client(): from datasette.app import Datasette from datasette.database import Database From b3001c1e5a5d5d5b2a04daf4a7445023f24806d5 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:41:58 -0700 Subject: [PATCH 043/156] Drop redundant _ds_client global now that ds_client is session-scoped Session-scoped fixtures are cached per worker by pytest itself, so the manual _ds_client module global is no longer needed. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- tests/conftest.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 171a5433..5f1cc587 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,9 +28,6 @@ UNDOCUMENTED_PERMISSIONS = { "view_document", } -_ds_client = None - - def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): start = time.time() while time.time() - start < timeout: @@ -60,10 +57,6 @@ async def ds_client(): from .fixtures import CONFIG, METADATA, PLUGINS_DIR import secrets - global _ds_client - if _ds_client is not None: - return _ds_client - ds = Datasette( metadata=METADATA, config=CONFIG, @@ -95,8 +88,7 @@ async def ds_client(): await db.execute_write_fn(prepare) await ds.invoke_startup() - _ds_client = ds.client - return _ds_client + return ds.client def pytest_report_header(config): From 630e557cdb7cce7ac05b4b3f8067990211e5477c Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:44:21 -0700 Subject: [PATCH 044/156] Ran black --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 5f1cc587..4ea89458 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,7 @@ UNDOCUMENTED_PERMISSIONS = { "view_document", } + def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): start = time.time() while time.time() - start < timeout: From a6031c98476487ec2aa5830bea75df9e6615262d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 16 Apr 2026 20:59:21 -0700 Subject: [PATCH 045/156] Release 1.0a28 Refs #2691, #2692, #2693 --- datasette/version.py | 2 +- docs/changelog.rst | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index e2c80e50..cf908bb2 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a27" +__version__ = "1.0a28" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9cd7a7d6..7c1da152 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v1_0_a28: + +1.0a28 (2026-04-16) +------------------- + +- Fixed a compatibility bug introduced in 1.0a27 where ``execute_write_fn()`` callbacks with a parameter name other than ``conn`` were seeing errors. (:issue:`2691`) +- The :ref:`database.close() <database_close>` method now also shuts down the write connection for that database. +- New :ref:`datasette.close() <datasette_close>` method for closing down all databases and resources associated with a Datasette instance. This is called automatically when the server shuts down. (:pr:`2693`) +- Datasette now includes a pytest plugin which automatically calls ``datasette.close()`` on temporary instances created in function-scoped fixtures and during tests. See :ref:`testing_plugins_autoclose` for details. This helps avoid running out of file descriptors in plugin test suites that were written before the ``Database(is_temp_disk=True)`` feature introduced in Datasette 1.0a27. (:issue:`2692`) + .. _v1_0_a27: 1.0a27 (2026-04-15) From b15ce18ddc463a537a52879381ad929d1867143d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 17 Apr 2026 08:44:43 -0700 Subject: [PATCH 046/156] TokenRestrictions.abbreviated(datasette) utility method for creating _r dicts (#2696) Closes #2695 Refs https://github.com/simonw/datasette-auth-tokens/pull/42 --- datasette/tokens.py | 59 ++++++++++++++++++++++--------------- docs/internals.rst | 24 +++++++++++++++ tests/test_token_handler.py | 37 +++++++++++++++++++++++ 3 files changed, 97 insertions(+), 23 deletions(-) diff --git a/datasette/tokens.py b/datasette/tokens.py index 5a12d8e0..38a55529 100644 --- a/datasette/tokens.py +++ b/datasette/tokens.py @@ -52,6 +52,38 @@ class TokenRestrictions: self.resource.setdefault(database, {}).setdefault(resource, []).append(action) return self + def abbreviated(self, datasette: "Datasette") -> Optional[dict]: + """ + Return the abbreviated ``_r`` dictionary shape for this set of + restrictions, using action abbreviations registered with ``datasette``. + Returns ``None`` if no restrictions are set. + """ + if not (self.all or self.database or self.resource): + return None + + def abbreviate_action(action): + action_obj = datasette.actions.get(action) + if not action_obj: + return action + return action_obj.abbr or action + + result: dict = {} + if self.all: + result["a"] = [abbreviate_action(a) for a in self.all] + if self.database: + result["d"] = { + database: [abbreviate_action(a) for a in actions] + for database, actions in self.database.items() + } + if self.resource: + result["r"] = {} + for database, resources in self.resource.items(): + for resource, actions in resources.items(): + result["r"].setdefault(database, {})[resource] = [ + abbreviate_action(a) for a in actions + ] + return result + class TokenHandler: """ @@ -104,31 +136,12 @@ class SignedTokenHandler(TokenHandler): token = {"a": actor_id, "t": int(time.time())} - def abbreviate_action(action): - action_obj = datasette.actions.get(action) - if not action_obj: - return action - return action_obj.abbr or action - if expires_after: token["d"] = expires_after - if restrictions and ( - restrictions.all or restrictions.database or restrictions.resource - ): - token["_r"] = {} - if restrictions.all: - token["_r"]["a"] = [abbreviate_action(a) for a in restrictions.all] - if restrictions.database: - token["_r"]["d"] = {} - for database, actions in restrictions.database.items(): - token["_r"]["d"][database] = [abbreviate_action(a) for a in actions] - if restrictions.resource: - token["_r"]["r"] = {} - for database, resources in restrictions.resource.items(): - for resource, actions in resources.items(): - token["_r"]["r"].setdefault(database, {})[resource] = [ - abbreviate_action(a) for a in actions - ] + if restrictions is not None: + abbreviated = restrictions.abbreviated(datasette) + if abbreviated is not None: + token["_r"] = abbreviated return "dstok_{}".format(datasette.sign(token, namespace="token")) async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: diff --git a/docs/internals.rst b/docs/internals.rst index 2710345b..e0123a7b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -729,6 +729,30 @@ The builder methods are: Each method returns the ``TokenRestrictions`` instance so calls can be chained. +``TokenRestrictions`` also provides an ``abbreviated(datasette)`` method which returns the restrictions as a dictionary using the compact format described in :ref:`authentication_cli_create_token_restrict`, with action names replaced by their registered abbreviations. It returns the inner dictionary only - the ``"_r"`` wrapping key shown in that section is not included. Returns ``None`` if no restrictions are set. This is useful when writing a custom :ref:`plugin_hook_register_token_handler` that needs to embed restrictions in a token payload. + +For example, the following restrictions: + +.. code-block:: python + + restrictions = ( + TokenRestrictions() + .allow_all("view-instance") + .allow_database("docs", "view-query") + .allow_resource("docs", "attachments", "insert-row") + ) + restrictions.abbreviated(datasette) + +Returns this dictionary, using the abbreviations registered for each action: + +.. code-block:: python + + { + "a": ["vi"], + "d": {"docs": ["vq"]}, + "r": {"docs": {"attachments": ["ir"]}}, + } + The following example creates a token that can access ``view-instance`` and ``view-table`` across everything, can additionally use ``view-query`` for anything in the ``docs`` database and is allowed to execute ``insert-row`` and ``update-row`` in the ``attachments`` table in that database: .. code-block:: python diff --git a/tests/test_token_handler.py b/tests/test_token_handler.py index 83f09046..5c87f577 100644 --- a/tests/test_token_handler.py +++ b/tests/test_token_handler.py @@ -291,6 +291,43 @@ async def test_expires_after_round_trip(datasette): assert "token_expires" in actor +@pytest.mark.asyncio +@pytest.mark.parametrize( + "build_restrictions,expected", + [ + (lambda r: r, None), + (lambda r: r.allow_all("view-instance"), {"a": ["vi"]}), + ( + lambda r: r.allow_database("docs", "view-query"), + {"d": {"docs": ["vq"]}}, + ), + ( + lambda r: r.allow_resource("docs", "attachments", "insert-row"), + {"r": {"docs": {"attachments": ["ir"]}}}, + ), + ( + lambda r: r.allow_all("view-instance") + .allow_database("docs", "view-query") + .allow_resource("docs", "attachments", "insert-row"), + { + "a": ["vi"], + "d": {"docs": ["vq"]}, + "r": {"docs": {"attachments": ["ir"]}}, + }, + ), + ( + lambda r: r.allow_all("not-a-real-action"), + {"a": ["not-a-real-action"]}, + ), + ], + ids=["empty", "all", "database", "resource", "combined", "unknown_action"], +) +async def test_token_restrictions_abbreviated(datasette, build_restrictions, expected): + await datasette.invoke_startup() + restrictions = build_restrictions(TokenRestrictions()) + assert restrictions.abbreviated(datasette) == expected + + @pytest.mark.asyncio async def test_signed_tokens_disabled(): """create_token and verify_token should fail/skip when signed tokens are disabled.""" From 0dc7bb19d9a95df9d9c6bd00e943d407fc11f49e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 22 Apr 2026 22:22:47 -0700 Subject: [PATCH 047/156] Table headers and column options visible for 0 rows Closes #2701 --- datasette/templates/_table.html | 5 +++-- datasette/templates/table.html | 2 -- tests/test_html.py | 2 +- tests/test_table_html.py | 26 ++++++++++++++++++++++++-- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index ba34b60f..f47a325f 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -1,6 +1,6 @@ <!-- above-table-panel is a hook node for plugins to attach to . Displays even if no data available --> <div class="above-table-panel"> </div> -{% if display_rows %} +{% if display_columns %} <div class="table-wrapper"> <table class="rows-and-columns"> <thead> @@ -31,6 +31,7 @@ </tbody> </table> </div> -{% else %} +{% endif %} +{% if not display_rows %} <p class="zero-results">0 records</p> {% endif %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 2919d306..c841e1be 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -141,7 +141,6 @@ {% if all_columns %} <column-chooser></column-chooser> <button class="choose-columns-mobile small-screen-only" onclick="openColumnChooser()">Choose columns</button> -{% if display_rows %} <button type="button" class="column-actions-mobile small-screen-only"> <svg aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> <circle cx="12" cy="12" r="3"></circle> @@ -149,7 +148,6 @@ </svg> <span>Column actions</span> </button> -{% endif %} <script> window._columnChooserData = {{ {"allColumns": all_columns, "selectedColumns": display_columns|map(attribute='name')|list, "primaryKeys": primary_keys}|tojson }}; </script> diff --git a/tests/test_html.py b/tests/test_html.py index e38898da..7425692d 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -805,7 +805,7 @@ async def test_blob_download_invalid_messages(ds_client, path, expected_message) async def test_zero_results(ds_client, path): response = await ds_client.get(path) soup = Soup(response.text, "html.parser") - assert 0 == len(soup.select("table")) + assert 0 == len(soup.select("table tbody tr")) assert 1 == len(soup.select("p.zero-results")) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index d8dde593..86b9a4eb 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -752,8 +752,11 @@ async def test_column_chooser_present(ds_client): @pytest.mark.asyncio -async def test_mobile_column_actions_present(ds_client): - response = await ds_client.get("/fixtures/facetable") +@pytest.mark.parametrize( + "path", ["/fixtures/facetable", "/fixtures/123_starts_with_digits"] +) +async def test_mobile_column_actions_present(ds_client, path): + response = await ds_client.get(path) assert response.status_code == 200 soup = Soup(response.text, "html.parser") button = soup.select_one("button.column-actions-mobile.small-screen-only") @@ -764,6 +767,25 @@ async def test_mobile_column_actions_present(ds_client): "mobile-column-actions.js" in (script.get("src") or "") for script in soup.find_all("script") ) + # mobile-column-actions.js builds its dialog from <th data-column> elements, + # so the thead must render even when the table has no rows. + ths = soup.select("table.rows-and-columns thead th[data-column]") + assert len(ths) >= 1 + + +@pytest.mark.asyncio +async def test_zero_row_table_renders_thead(ds_client): + response = await ds_client.get("/fixtures/123_starts_with_digits") + assert response.status_code == 200 + soup = Soup(response.text, "html.parser") + table = soup.select_one("table.rows-and-columns") + assert table is not None + column_names = [ + th.get("data-column") for th in table.select("thead th[data-column]") + ] + assert "content" in column_names + assert table.select_one("tbody tr") is None + assert soup.select_one("p.zero-results") is not None @pytest.mark.asyncio From aa84fe008d7c6263bd8712adcfe9be53a9f207ea Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 5 May 2026 16:05:12 -0700 Subject: [PATCH 048/156] Fix for column actions on Mobile Safari, closes #2708 --- datasette/static/app.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 26717c43..1ce84bc8 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -818,7 +818,8 @@ dialog.mobile-column-actions-dialog::backdrop { } .mobile-column-actions-dialog .list-wrap { - flex: 1; + flex: 1 1 auto; + min-height: 0; overflow-y: auto; overflow-x: hidden; position: relative; From 345f910043bebd4fb829c1f8c248a17e38856191 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 12 May 2026 16:31:36 -0700 Subject: [PATCH 049/156] Fix for Database.close()/Datasette.close() order (#2710) Closes: - #2709 The key behavior change: after close() starts, no new execute work can be submitted, but already-running execute work is allowed to finish before SQLite connections are closed. --- datasette/database.py | 26 +++++++++++++--- datasette/version.py | 2 +- tests/test_internals_datasette.py | 49 +++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index e3c4bfec..657adfa5 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -84,6 +84,8 @@ class Database: self._write_thread = None self._write_queue = None self._closed = False + self._pending_execute_futures = set() + self._pending_execute_futures_lock = threading.Lock() # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None @@ -98,6 +100,10 @@ class Database: "Database {!r} has been closed".format(self.name) ) + def _remove_pending_execute_future(self, future): + with self._pending_execute_futures_lock: + self._pending_execute_futures.discard(future) + @property def cached_table_counts(self): if self._cached_table_counts is not None: @@ -170,7 +176,11 @@ class Database: """ if self._closed: return - self._closed = True + with self._pending_execute_futures_lock: + if self._closed: + return + self._closed = True + pending_execute_futures = tuple(self._pending_execute_futures) # Shut down the write thread, if any, via a sentinel. The thread # drains any writes already queued before the sentinel and then # closes its own write connection and returns. @@ -185,6 +195,11 @@ class Database: ) ) sys.stderr.flush() + for future in pending_execute_futures: + try: + future.result() + except Exception: + pass # Close anything still tracked in _all_file_connections for connection in self._all_file_connections: try: @@ -456,9 +471,12 @@ class Database: setattr(connections, self._thread_local_id, conn) return fn(conn) - return await asyncio.get_event_loop().run_in_executor( - self.ds.executor, in_thread - ) + with self._pending_execute_futures_lock: + self._check_not_closed() + future = self.ds.executor.submit(in_thread) + self._pending_execute_futures.add(future) + future.add_done_callback(self._remove_pending_execute_future) + return await asyncio.wrap_future(future) async def execute( self, diff --git a/datasette/version.py b/datasette/version.py index cf908bb2..898d388c 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a28" +__version__ = "1.0a28.post1" __version_info__ = tuple(__version__.split(".")) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index d58c9a29..3f867eb0 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -2,8 +2,11 @@ Tests for the datasette.app.Datasette class """ +import asyncio import dataclasses import os +import sqlite3 +import time from datasette import Context from datasette.app import Datasette, Database, ResourcesSQL from datasette.database import DatasetteClosedError @@ -256,6 +259,52 @@ async def test_datasette_close_raises_on_use(): await ds.get_internal_database().execute("select 1") +async def _datasette_with_sleeping_execute(tmp_path, sleep_ms=200): + db_path = tmp_path / "data.db" + internal_path = tmp_path / "internal.db" + sqlite3.connect(db_path).close() + ds = Datasette([str(db_path)], internal=str(internal_path)) + loop = asyncio.get_running_loop() + sql_started = asyncio.Event() + original_prepare_connection = ds._prepare_connection + + def prepare_connection(conn, name): + original_prepare_connection(conn, name) + + def sleep_ms(ms): + loop.call_soon_threadsafe(sql_started.set) + time.sleep(ms / 1000) + return ms + + conn.create_function("sleep_ms", 1, sleep_ms) + + ds._prepare_connection = prepare_connection + task = asyncio.create_task( + ds.get_database().execute( + f"select sleep_ms({sleep_ms})", custom_time_limit=1000 + ) + ) + await asyncio.wait_for(sql_started.wait(), timeout=5) + return ds, task + + +@pytest.mark.asyncio +async def test_datasette_close_waits_for_in_flight_execute(tmp_path): + ds, task = await _datasette_with_sleeping_execute(tmp_path) + ds.close() + results = await task + assert [tuple(row) for row in results.rows] == [(200,)] + + +@pytest.mark.asyncio +async def test_datasette_close_waits_for_cancelled_in_flight_execute(tmp_path): + ds, task = await _datasette_with_sleeping_execute(tmp_path) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + ds.close() + + @pytest.mark.asyncio async def test_asgi_lifespan_shutdown_closes_datasette(): ds = Datasette(memory=True) From db16003865dee862d63895dbc156461e8b89372b Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 12 May 2026 16:39:06 -0700 Subject: [PATCH 050/156] Release 1.0a29 Refs #2695, #2701, #2708, #2709 --- datasette/version.py | 2 +- docs/changelog.rst | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 898d388c..e661e76d 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a28.post1" +__version__ = "1.0a29" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7c1da152..dd9273ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v1_0_a29: + +1.0a29 (2026-05-12) +------------------- + +- New ``TokenRestrictions.abbreviated(datasette)`` :ref:`utility method <TokenRestrictions>` for creating ``"_r"`` dictionaries. (:issue:`2695`) +- Table headers and column options are now visible even if a table contains zero rows. (:issue:`2701`) +- Fixed bug with display of column actions dialog on Mobile Safari. (:issue:`2708`) +- Fixed bug where tests could crash with a segfault due to a race condition between ``Datasette.close()`` and ``Datasette.close()``. (:issue:`2709`) +- .. _v1_0_a28: 1.0a28 (2026-04-16) From 036aa6aa2ef71350d5e99cd12620f9bcd70d7a19 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 12 May 2026 16:39:46 -0700 Subject: [PATCH 051/156] Removed a rogue hyphen --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index dd9273ca..4f26066c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,7 @@ Changelog - Table headers and column options are now visible even if a table contains zero rows. (:issue:`2701`) - Fixed bug with display of column actions dialog on Mobile Safari. (:issue:`2708`) - Fixed bug where tests could crash with a segfault due to a race condition between ``Datasette.close()`` and ``Datasette.close()``. (:issue:`2709`) -- + .. _v1_0_a28: 1.0a28 (2026-04-16) From 46d90a0b887bfa23f986a28499eb5f85cd7eed04 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Tue, 12 May 2026 16:46:56 -0700 Subject: [PATCH 052/156] Bump to actions/checkout@v6 --- .github/workflows/publish.yml | 8 ++++---- .github/workflows/test.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2e8cea9c..87300593 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -35,7 +35,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: @@ -56,7 +56,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: @@ -92,7 +92,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a0f5477b..a1b2e9d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: From 3110faa0bab8cdeb4e4e042e87fefa434f64162f Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 16 May 2026 11:45:43 -0700 Subject: [PATCH 053/156] Replace Janus queue with asyncio.Future Closes #1752 AI generated patch explanation: https://gisthost.github.io/?e2b8d9c7666e988b5c003ff5e5ef3098 --- datasette/database.py | 117 +++++++++++++++++++------------ docs/changelog.rst | 8 +++ pyproject.toml | 1 - tests/test_internals_database.py | 34 +++++++++ tests/test_write_wrapper.py | 27 ++++++- 5 files changed, 140 insertions(+), 47 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 657adfa5..66d50ffa 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -4,7 +4,6 @@ from collections import namedtuple import inspect import os from pathlib import Path -import janus import queue import sqlite_utils import sys @@ -330,13 +329,16 @@ class Database: else: # For non-blocking writes, spawn a background task to # dispatch events after the write thread completes - task_id, reply_queue = result + task_id, reply_future = result async def _dispatch_events_after_write(): - write_result = await reply_queue.async_q.get() - if not isinstance(write_result, Exception): - for event in pending_events: - await self.ds.track_event(event) + try: + await reply_future + except Exception: + # if the write failed, don't emit success events + return + for event in pending_events: + await self.ds.track_event(event) asyncio.ensure_future(_dispatch_events_after_write()) result = task_id @@ -390,18 +392,15 @@ class Database: ) self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") - reply_queue = janus.Queue() + loop = asyncio.get_running_loop() + reply_future = loop.create_future() self._write_queue.put( - WriteTask(fn, task_id, reply_queue, isolated_connection, transaction) + WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) ) if block: - result = await reply_queue.async_q.get() - if isinstance(result, Exception): - raise result - else: - return result + return await reply_future else: - return task_id, reply_queue + return task_id, reply_future def _execute_writes(self): # Infinite looping thread that protects the single write connection @@ -422,36 +421,37 @@ class Database: except Exception: pass return + exception = None + result = None if conn_exception is not None: - result = conn_exception + exception = conn_exception + elif task.isolated_connection: + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + exception = e + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass else: - if task.isolated_connection: - isolated_connection = self.connect(write=True) - try: - result = task.fn(isolated_connection) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - result = e - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass - else: - try: - if task.transaction: - with conn: - result = task.fn(conn) - else: + try: + if task.transaction: + with conn: result = task.fn(conn) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - result = e - task.reply_queue.sync_q.put(result) + else: + result = task.fn(conn) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + exception = e + _deliver_write_result(task, result, exception) async def execute_fn(self, fn): self._check_not_closed() @@ -892,16 +892,45 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: - __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection", "transaction") + __slots__ = ( + "fn", + "task_id", + "loop", + "reply_future", + "isolated_connection", + "transaction", + ) - def __init__(self, fn, task_id, reply_queue, isolated_connection, transaction): + def __init__( + self, fn, task_id, loop, reply_future, isolated_connection, transaction + ): self.fn = fn self.task_id = task_id - self.reply_queue = reply_queue + self.loop = loop + self.reply_future = reply_future self.isolated_connection = isolated_connection self.transaction = transaction +def _deliver_write_result(task, result, exception): + # Called from the write thread. Delivers the result back to the + # awaiting coroutine on its event loop via call_soon_threadsafe. + def _set(): + if task.reply_future.done(): + # Awaiter was cancelled; nothing to do. + return + if exception is not None: + task.reply_future.set_exception(exception) + else: + task.reply_future.set_result(result) + + try: + task.loop.call_soon_threadsafe(_set) + except RuntimeError: + # Event loop has been closed; the awaiter is gone. + pass + + class QueryInterrupted(Exception): def __init__(self, e, sql, params): self.e = e diff --git a/docs/changelog.rst b/docs/changelog.rst index 4f26066c..5b637797 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,14 @@ Changelog ========= +.. _unreleased: + +Unreleased +---------- + +- Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) + + .. _v1_0_a29: 1.0a29 (2026-05-12) diff --git a/pyproject.toml b/pyproject.toml index e6007afd..c50c720a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,6 @@ dependencies = [ "pluggy>=1.0", "uvicorn>=0.11", "aiofiles>=0.4", - "janus>=0.6.2", "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 8ff74a83..75ae8d39 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -2,9 +2,12 @@ Tests for the datasette.database.Database class """ +import asyncio +from types import SimpleNamespace from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues from datasette.database import DatasetteClosedError +from datasette.database import _deliver_write_result from datasette.utils.sqlite import sqlite3, sqlite_version from datasette.utils import Column import pytest @@ -590,6 +593,37 @@ async def test_execute_write_fn_connection_exception(tmpdir, app_client): app_client.ds.remove_database("immutable-db") +@pytest.mark.asyncio +async def test_deliver_write_result_leaves_done_future_alone(): + loop = asyncio.get_running_loop() + reply_future = loop.create_future() + reply_future.set_result("original") + task = SimpleNamespace(loop=loop, reply_future=reply_future) + + # The write thread can finish after the caller has stopped waiting for the + # result. Delivery should notice that the future is already resolved and + # leave the caller's outcome alone instead of raising InvalidStateError. + _deliver_write_result(task, "replacement", None) + await asyncio.sleep(0) + + assert reply_future.result() == "original" + + +@pytest.mark.asyncio +async def test_deliver_write_result_ignores_closed_loop(): + closed_loop = asyncio.new_event_loop() + closed_loop.close() + reply_future = asyncio.get_running_loop().create_future() + task = SimpleNamespace(loop=closed_loop, reply_future=reply_future) + + # If the event loop that submitted the write has gone away, the write + # thread should drop the result rather than crash while reporting back to + # that closed loop. + _deliver_write_result(task, "result", None) + + assert not reply_future.done() + + def table_exists(conn, name): return bool( conn.execute( diff --git a/tests/test_write_wrapper.py b/tests/test_write_wrapper.py index 48c964b4..88ce5520 100644 --- a/tests/test_write_wrapper.py +++ b/tests/test_write_wrapper.py @@ -2,6 +2,7 @@ Tests for the write_wrapper plugin hook. """ +import asyncio from dataclasses import dataclass from datasette.app import Datasette from datasette.events import Event @@ -633,8 +634,6 @@ async def test_track_event_with_block_false(ds_with_event_tracking): assert task_id is not None # Give the background task time to complete - import asyncio - for _ in range(50): if ds._tracked_events: break @@ -644,6 +643,30 @@ async def test_track_event_with_block_false(ds_with_event_tracking): assert ds._tracked_events[0].message == "non-blocking" +@pytest.mark.asyncio +async def test_track_event_with_block_false_discarded_on_exception( + ds_with_event_tracking, +): + """Events queued by a non-blocking write are discarded if the write fails.""" + ds = ds_with_event_tracking + db = ds.get_database("test") + + def my_write(conn, track_event): + track_event(DummyEvent(actor=None, message="should not fire")) + raise ValueError("deliberate error") + + task_id = await db.execute_write_fn(my_write, block=False) + assert task_id is not None + + # A following blocking write proves the failed non-blocking task has + # completed; one more loop turn lets its event-dispatch task observe the + # exception and exit. + await db.execute_write_fn(lambda conn: conn.execute("select 1")) + await asyncio.sleep(0) + + assert ds._tracked_events == [] + + # --- Tests for RenameTableEvent detection --- From 10a1caac53be3d4b8344500f676c49bf82dd5384 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 16 May 2026 16:38:49 -0700 Subject: [PATCH 054/156] Upgrade a whole lot of GitHum Actions references --- .github/workflows/deploy-branch-preview.yml | 2 +- .github/workflows/deploy-latest.yml | 2 +- .github/workflows/prettier.yml | 4 ++-- .github/workflows/push_docker_tag.yml | 2 +- .github/workflows/spellcheck.yml | 2 +- .github/workflows/stable-docs.yml | 2 +- .github/workflows/test-coverage.yml | 2 +- .github/workflows/test-pyodide.yml | 4 ++-- .github/workflows/test-sqlite-support.yml | 2 +- .github/workflows/tmate-mac.yml | 2 +- .github/workflows/tmate.yml | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml index e56d9c27..4aa676da 100644 --- a/.github/workflows/deploy-branch-preview.yml +++ b/.github/workflows/deploy-branch-preview.yml @@ -12,7 +12,7 @@ jobs: deploy-branch-preview: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Set up Python 3.11 uses: actions/setup-python@v6 with: diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 7349a1ab..18c01fdc 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 77cce7d1..735e14e9 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -10,8 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v4 - - uses: actions/cache@v4 + uses: actions/checkout@v6 + - uses: actions/cache@v5 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index afe8d6b2..e622ef4c 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -13,7 +13,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index d42ae96b..9a808194 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,7 +9,7 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml index 3119d617..59b5fbc0 100644 --- a/.github/workflows/stable-docs.yml +++ b/.github/workflows/stable-docs.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 # We need all commits to find docs/ changes - name: Set up Git user diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 1b3d2f2c..c514048e 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index b490a9bf..5162c47a 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -12,7 +12,7 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python 3.10 uses: actions/setup-python@v6 with: @@ -20,7 +20,7 @@ jobs: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - name: Cache Playwright browsers - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-browsers diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index c81a3c0b..23fce459 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -25,7 +25,7 @@ jobs: #"3.23.1" # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index fcee0f21..a033cd92 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -10,6 +10,6 @@ jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 123f6c71..72af1eec 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -11,7 +11,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 env: From c1b30818633e5cbb43d32bf163f99b0aacc391b7 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 16 May 2026 16:44:28 -0700 Subject: [PATCH 055/156] Removed obsolete workflow --- .github/workflows/deploy-branch-preview.yml | 35 --------------------- 1 file changed, 35 deletions(-) delete mode 100644 .github/workflows/deploy-branch-preview.yml diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml deleted file mode 100644 index 4aa676da..00000000 --- a/.github/workflows/deploy-branch-preview.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Deploy a Datasette branch preview to Vercel - -on: - workflow_dispatch: - inputs: - branch: - description: "Branch to deploy" - required: true - type: string - -jobs: - deploy-branch-preview: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Set up Python 3.11 - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - name: Install dependencies - run: | - pip install datasette-publish-vercel - - name: Deploy the preview - env: - VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} - run: | - export BRANCH="${{ github.event.inputs.branch }}" - wget https://latest.datasette.io/fixtures.db - datasette publish vercel fixtures.db \ - --branch $BRANCH \ - --project "datasette-preview-$BRANCH" \ - --token $VERCEL_TOKEN \ - --scope datasette \ - --about "Preview of $BRANCH" \ - --about_url "https://github.com/simonw/datasette/tree/$BRANCH" From 40e78e09277ae547dc63172f6b2bad50b0f24b64 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 16 May 2026 16:48:10 -0700 Subject: [PATCH 056/156] Change pull_request_target to pull_request event --- .github/workflows/documentation-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index a54bd83a..b8fb8aaa 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -1,6 +1,6 @@ name: Read the Docs Pull Request Preview on: - pull_request_target: + pull_request: types: - opened From 7a914f8c656de2ffa3f662e49bc95b24dd36b854 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 20 May 2026 12:14:50 -0700 Subject: [PATCH 057/156] Clear stale tables/other resources when DB removed, closes #2723 --- datasette/app.py | 23 +++++++++++++++---- docs/changelog.rst | 2 +- tests/test_internal_db.py | 48 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 358081ef..218d40c6 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -618,11 +618,24 @@ class Datasette: stale_databases = set(current_schema_versions.keys()) - set( self.databases.keys() ) - for stale_db_name in stale_databases: - await internal_db.execute_write( - "DELETE FROM catalog_databases WHERE database_name = ?", - [stale_db_name], - ) + if stale_databases: + + def delete_stale_database_catalog(conn): + for stale_db_name in stale_databases: + for table in ( + "catalog_columns", + "catalog_foreign_keys", + "catalog_indexes", + "catalog_views", + "catalog_tables", + "catalog_databases", + ): + conn.execute( + "DELETE FROM {} WHERE database_name = ?".format(table), + [stale_db_name], + ) + + await internal_db.execute_write_fn(delete_stale_database_catalog) for database_name, db in self.databases.items(): schema_version = (await db.execute("PRAGMA schema_version")).first()[0] # Compare schema versions to see if we should skip it diff --git a/docs/changelog.rst b/docs/changelog.rst index 5b637797..eb408287 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ Unreleased ---------- - Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) - +- Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) .. _v1_0_a29: diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index 7a0d1630..ec013b43 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -139,3 +139,51 @@ async def test_stale_catalog_entry_database_fix(tmp_path): f"Index page should return 200, not {response.status_code}. " "This fails due to stale catalog entries causing KeyError." ) + + +@pytest.mark.asyncio +async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path): + from datasette.app import Datasette + + import sqlite3 + + internal_db_path = str(tmp_path / "internal.db") + alpha_db_path = str(tmp_path / "alpha.db") + bravo_db_path = str(tmp_path / "bravo.db") + + for db_path, table_name in ( + (alpha_db_path, "alpha_table"), + (bravo_db_path, "bravo_table"), + (bravo_db_path, "bravo_table_2"), + ): + conn = sqlite3.connect(db_path) + conn.execute(f"CREATE TABLE {table_name} (id INTEGER PRIMARY KEY)") + conn.close() + + ds1 = Datasette(files=[alpha_db_path, bravo_db_path], internal=internal_db_path) + await ds1.invoke_startup() + + catalog_tables = await ds1.get_internal_database().execute(""" + SELECT database_name, table_name + FROM catalog_tables + ORDER BY database_name, table_name + """) + assert [tuple(row) for row in catalog_tables.rows] == [ + ("alpha", "alpha_table"), + ("bravo", "bravo_table"), + ("bravo", "bravo_table_2"), + ] + + ds1.close() + + ds2 = Datasette(files=[alpha_db_path], internal=internal_db_path) + await ds2.invoke_startup() + + catalog_tables = await ds2.get_internal_database().execute(""" + SELECT database_name, table_name + FROM catalog_tables + ORDER BY database_name, table_name + """) + assert [tuple(row) for row in catalog_tables.rows] == [("alpha", "alpha_table")] + + ds2.close() From 5d6de0154d18d0ed07e7ac7cf02cd3c37324ddbc Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 20 May 2026 12:18:01 -0700 Subject: [PATCH 058/156] Bump Black to black==26.3.1 Refs https://github.com/advisories/GHSA-3936-cmfr-pm3m --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c50c720a..38085476 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ dev = [ "pytest-xdist>=2.2.1", "pytest-asyncio>=1.2.0", "beautifulsoup4>=4.8.1", - "black==26.1.0", + "black==26.3.1", "blacken-docs==1.20.0", "pytest-timeout>=1.4.2", "trustme>=0.7", From bbbc1cd59620c7c53a2eff6138ef338c8901ba6e Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 20 May 2026 12:33:33 -0700 Subject: [PATCH 059/156] Remove height: 100% to fix Safari bug, closes #2724 --- datasette/static/navigation-search.js | 1 - docs/changelog.rst | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index 95e7dfc5..d2c300e2 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -54,7 +54,6 @@ class NavigationSearch extends HTMLElement { .search-container { display: flex; flex-direction: column; - height: 100%; } .search-input-wrapper { diff --git a/docs/changelog.rst b/docs/changelog.rst index eb408287..56c49ea3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased - Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) - Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) +- Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`) .. _v1_0_a29: From 54b272baf61cb014e6d262fea1b01dc84981c1d0 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 20 May 2026 12:39:54 -0700 Subject: [PATCH 060/156] Remove existing stale catalog_ tables, refs #2723 Now if there are any existing stale records in internal.db those will be removed as well. --- datasette/app.py | 30 ++++++++++++++++---------- tests/test_internal_db.py | 45 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 218d40c6..b1f9b2f7 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -614,22 +614,30 @@ class Datasette: "select database_name, schema_version from catalog_databases" ) } - # Delete stale entries for databases that are no longer attached - stale_databases = set(current_schema_versions.keys()) - set( - self.databases.keys() + catalog_table_names = ( + "catalog_columns", + "catalog_foreign_keys", + "catalog_indexes", + "catalog_views", + "catalog_tables", + "catalog_databases", ) + # Delete stale entries for databases that are no longer attached + catalog_database_names = set(current_schema_versions.keys()) + for table in catalog_table_names[:-1]: + catalog_database_names.update( + row["database_name"] + for row in await internal_db.execute( + "select distinct database_name from {}".format(table) + ) + if row["database_name"] is not None + ) + stale_databases = catalog_database_names - set(self.databases.keys()) if stale_databases: def delete_stale_database_catalog(conn): for stale_db_name in stale_databases: - for table in ( - "catalog_columns", - "catalog_foreign_keys", - "catalog_indexes", - "catalog_views", - "catalog_tables", - "catalog_databases", - ): + for table in catalog_table_names: conn.execute( "DELETE FROM {} WHERE database_name = ?".format(table), [stale_db_name], diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index ec013b43..dcf14126 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -187,3 +187,48 @@ async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path assert [tuple(row) for row in catalog_tables.rows] == [("alpha", "alpha_table")] ds2.close() + + +@pytest.mark.asyncio +async def test_orphan_stale_catalog_child_entries_removed(tmp_path): + from datasette.app import Datasette + + import sqlite3 + + internal_db_path = str(tmp_path / "internal.db") + alpha_db_path = str(tmp_path / "alpha.db") + + conn = sqlite3.connect(alpha_db_path) + conn.execute("CREATE TABLE alpha_table (id INTEGER PRIMARY KEY)") + conn.close() + + ds1 = Datasette(files=[alpha_db_path], internal=internal_db_path) + await ds1.invoke_startup() + ds1.close() + + # Simulate the state left behind by old cleanup code: the parent database + # row was deleted, but child catalog rows survived because foreign key + # enforcement is not enabled for these internal catalog writes. + conn = sqlite3.connect(internal_db_path) + conn.execute("DELETE FROM catalog_databases WHERE database_name = 'fixtures'") + conn.execute(""" + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) + VALUES ('fixtures', 'stale_table', 1, 'CREATE TABLE stale_table (id INTEGER)') + """) + conn.commit() + conn.close() + + ds2 = Datasette(files=[alpha_db_path], internal=internal_db_path) + await ds2.invoke_startup() + + catalog_tables = await ds2.get_internal_database().execute(""" + SELECT database_name, table_name + FROM catalog_tables + ORDER BY database_name, table_name + """) + assert [tuple(row) for row in catalog_tables.rows] == [("alpha", "alpha_table")] + + response = await ds2.client.get("/-/tables.json") + assert response.status_code == 200 + + ds2.close() From d3330695fa42ad1cdb2f2b1b80470e95bea8ed12 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Wed, 20 May 2026 13:23:05 -0700 Subject: [PATCH 061/156] Always show 'Jump to...' menu item, closes #2725 --- datasette/static/app.css | 26 ++++++++++++++++++++++++++ datasette/static/navigation-search.js | 13 ++++++++++++- datasette/templates/base.html | 7 +++---- docs/changelog.rst | 1 + tests/test_html.py | 18 +++++++++++++++--- 5 files changed, 57 insertions(+), 8 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 1ce84bc8..c21d0dc4 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -362,6 +362,32 @@ form.nav-menu-logout { .nav-menu-inner a { display: block; } +.nav-menu-inner button.button-as-link { + display: block; + width: 100%; + text-align: left; + font: inherit; +} +.nav-menu-inner .keyboard-shortcut { + float: right; + box-sizing: border-box; + min-width: 1.4em; + margin-left: 0.75rem; + padding: 0 0.35em; + border: 1px solid rgba(255,255,244,0.6); + border-radius: 3px; + background: rgba(255,255,244,0.12); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.85em; + line-height: 1.35; + text-align: center; + text-decoration: none; +} +@media (max-width: 640px) { + .nav-menu-inner .keyboard-shortcut { + display: none; + } +} /* Table/database actions menu */ .page-action-menu { diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index d2c300e2..09d58898 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -199,6 +199,15 @@ class NavigationSearch extends HTMLElement { } }); + document.addEventListener("click", (e) => { + const trigger = e.target.closest("[data-navigation-search-open]"); + if (trigger) { + e.preventDefault(); + trigger.closest("details")?.removeAttribute("open"); + this.openMenu(); + } + }); + // Input event input.addEventListener("input", (e) => { this.handleSearch(e.target.value); @@ -390,7 +399,9 @@ class NavigationSearch extends HTMLElement { const dialog = this.shadowRoot.querySelector("dialog"); const input = this.shadowRoot.querySelector(".search-input"); - dialog.showModal(); + if (!dialog.open) { + dialog.showModal(); + } input.value = ""; input.focus(); diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 21f8c693..b4fecf70 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -20,7 +20,7 @@ <body class="{% block body_class %}{% endblock %}"> <div class="not-footer"> <header class="hd"><nav>{% block nav %}{% block crumbs %}{{ crumbs.nav(request=request) }}{% endblock %} - {% set links = menu_links() %}{% if links or show_logout %} + {% set links = menu_links() %} <details class="nav-menu details-menu"> <summary><svg aria-labelledby="nav-menu-svg-title" role="img" fill="currentColor" stroke="currentColor" xmlns="http://www.w3.org/2000/svg" @@ -29,19 +29,18 @@ <path fill-rule="evenodd" d="M1 2.75A.75.75 0 011.75 2h12.5a.75.75 0 110 1.5H1.75A.75.75 0 011 2.75zm0 5A.75.75 0 011.75 7h12.5a.75.75 0 110 1.5H1.75A.75.75 0 011 7.75zM1.75 12a.75.75 0 100 1.5h12.5a.75.75 0 100-1.5H1.75z"></path> </svg></summary> <div class="nav-menu-inner"> - {% if links %} <ul> + <li><button type="button" class="button-as-link" data-navigation-search-open>Jump to... <kbd class="keyboard-shortcut" aria-hidden="true" title="Keyboard shortcut: press / to open Jump to">/</kbd></button></li> {% for link in links %} <li><a href="{{ link.href }}">{{ link.label }}</a></li> {% endfor %} </ul> - {% endif %} {% if show_logout %} <form class="nav-menu-logout" action="{{ urls.logout() }}" method="post"> <button class="button-as-link">Log out</button> </form>{% endif %} </div> - </details>{% endif %} + </details> {% if actor %} <div class="actor"> <strong>{{ display_actor(actor) }}</strong> diff --git a/docs/changelog.rst b/docs/changelog.rst index 56c49ea3..f2ba23ec 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ Unreleased - Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) - Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) - Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`) +- New "Jump to..." menu item, always visible, for triggering the previously undocumented ``/`` menu. (:issue:`2725`) .. _v1_0_a29: diff --git a/tests/test_html.py b/tests/test_html.py index 7425692d..64b4075a 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1008,10 +1008,22 @@ async def test_navigation_menu_links( kwargs["actor"] = {"id": actor_id} html = (await ds_client.get("/", **kwargs)).text soup = Soup(html, "html.parser") - details = soup.find("nav").find("details") + details = soup.find("nav").find("details", {"class": "nav-menu"}) + assert details is not None + search_button = details.find("button", {"data-navigation-search-open": True}) + assert search_button is not None + assert search_button.text.strip() == "Jump to... /" + assert search_button.find("kbd", {"class": "keyboard-shortcut"}).text == "/" + assert search_button.find("kbd")["aria-hidden"] == "true" + assert ( + search_button.find("kbd")["title"] + == "Keyboard shortcut: press / to open Jump to" + ) + assert details.find("li").find("button") == search_button if not actor_id: - # Should not show a menu - assert details is None + # The app menu is always visible, but anonymous users do not see logout + # or debug links. + assert details.find("form") is None return # They are logged in: should show a menu assert details is not None From fae847ac1017bb76044fd3aad703d5b9725c12ed Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 21 May 2026 15:02:14 -0700 Subject: [PATCH 062/156] Prototype of new /-/jump menu plus plugin hook --- datasette/app.py | 47 +++- datasette/handle_exception.py | 3 + datasette/hookspecs.py | 10 + datasette/jump.py | 33 +++ datasette/static/navigation-search.js | 240 ++++++++++++++++++--- datasette/templates/base.html | 11 +- datasette/views/special.py | 259 ++++++++++++++++++----- docs/introspection.rst | 51 +++-- tests/test_allowed_resources.py | 20 +- tests/test_html.py | 7 + tests/test_internal_db.py | 2 +- tests/test_internals_datasette_client.py | 8 +- tests/test_jump.py | 224 ++++++++++++++++++++ tests/test_navigation_search_js.py | 199 +++++++++++++++++ tests/test_search_tables.py | 21 +- 15 files changed, 1005 insertions(+), 130 deletions(-) create mode 100644 datasette/jump.py create mode 100644 tests/test_jump.py create mode 100644 tests/test_navigation_search_js.py diff --git a/datasette/app.py b/datasette/app.py index b1f9b2f7..c9605af3 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -58,7 +58,7 @@ from .views.special import ( AllowedResourcesView, PermissionRulesView, PermissionCheckView, - TablesView, + JumpView, InstanceSchemaView, DatabaseSchemaView, TableSchemaView, @@ -1219,13 +1219,24 @@ class Datasette: return db_plugin_config + def static_hash(self, filename): + if not hasattr(self, "_static_hashes"): + self._static_hashes = {} + path = os.path.join(str(app_root), "datasette/static", filename) + signature = (os.path.getmtime(path), os.path.getsize(path)) + cached = self._static_hashes.get(filename) + if cached and cached["signature"] == signature: + return cached["hash"] + with open(path) as fp: + static_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[:6] + self._static_hashes[filename] = { + "signature": signature, + "hash": static_hash, + } + return static_hash + def app_css_hash(self): - if not hasattr(self, "_app_css_hash"): - with open(os.path.join(str(app_root), "datasette/static/app.css")) as fp: - self._app_css_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[ - :6 - ] - return self._app_css_hash + return self.static_hash("app.css") async def get_canned_queries(self, database_name, actor): queries = {} @@ -2028,6 +2039,22 @@ class Datasette: links.extend(extra_links) return links + async def jump_start(): + html_bits = [] + for hook in pm.hook.jump_start( + datasette=self, + actor=request.actor if request else None, + request=request or None, + ): + extra_html = await await_me_maybe(hook) + if not extra_html: + continue + if isinstance(extra_html, (list, tuple)): + html_bits.extend(extra_html) + else: + html_bits.append(extra_html) + return Markup("").join(Markup(html) for html in html_bits) + template_context = { **context, **{ @@ -2036,11 +2063,13 @@ class Datasette: "urls": self.urls, "actor": request.actor if request else None, "menu_links": menu_links, + "jump_start": jump_start, "display_actor": display_actor, "show_logout": request is not None and "ds_actor" in request.cookies and request.actor, "app_css_hash": self.app_css_hash(), + "navigation_search_js_hash": self.static_hash("navigation-search.js"), "zip": zip, "body_scripts": body_scripts, "format_bytes": format_bytes, @@ -2222,8 +2251,8 @@ class Datasette: r"/-/api$", ) add_route( - TablesView.as_view(self), - r"/-/tables(\.(?P<format>json))?$", + JumpView.as_view(self), + r"/-/jump(\.(?P<format>json))?$", ) add_route( InstanceSchemaView.as_view(self), diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index 96398a4c..25ec26d9 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -67,6 +67,9 @@ def handle_exception(datasette, request, exception): info, urls=datasette.urls, app_css_hash=datasette.app_css_hash(), + navigation_search_js_hash=datasette.static_hash( + "navigation-search.js" + ), menu_links=lambda: [], ) ), diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 27e20bd4..a4d8143b 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -157,6 +157,16 @@ def menu_links(datasette, actor, request): """Links for the navigation menu""" +@hookspec +def jump_items_sql(datasette, actor, request): + """SQL fragments for extra items in the jump menu, optionally with display_name""" + + +@hookspec +def jump_start(datasette, actor, request): + """HTML to display in the jump menu before the user types""" + + @hookspec def row_actions(datasette, actor, request, database, table, row): """Links for the row actions menu""" diff --git a/datasette/jump.py b/datasette/jump.py new file mode 100644 index 00000000..96e18547 --- /dev/null +++ b/datasette/jump.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + + +@dataclass +class JumpSQL: + sql: str + params: dict[str, Any] | None = None + has_display_name: bool = False + + +_PARAM_RE = re.compile(r"(?<!:):([A-Za-z_][A-Za-z0-9_]*)") + + +def namespace_sql_params(sql: str, params: dict[str, Any], prefix: str): + """Rename named SQL parameters so UNION fragments cannot collide.""" + if not params: + return sql, {} + + renamed = {key: f"{prefix}_{key}" for key in params} + + def replace(match): + key = match.group(1) + if key not in renamed: + return match.group(0) + return f":{renamed[key]}" + + return _PARAM_RE.sub(replace, sql), { + renamed[key]: value for key, value in params.items() + } diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index 09d58898..49cc172c 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -100,16 +100,65 @@ class NavigationSearch extends HTMLElement { background-color: #dbeafe; } + .jump-start-content { + border-bottom: 1px solid #e5e7eb; + margin-bottom: 0.5rem; + padding: 0.5rem 0.5rem 1rem; + } + + .jump-start-content:empty { + display: none; + } + .result-name { font-weight: 500; color: #111827; } + .result-label { + font-size: 0.875rem; + color: #4b5563; + } + + .result-description { + color: #4b5563; + font-size: 0.75rem; + font-weight: 600; + text-transform: uppercase; + } + .result-url { font-size: 0.875rem; color: #6b7280; } + .results-heading { + color: #4b5563; + font-size: 0.75rem; + font-weight: 600; + letter-spacing: 0; + padding: 0.5rem 1rem 0.25rem; + text-transform: uppercase; + } + + .recent-actions { + padding: 0.25rem 1rem 0.75rem; + } + + .clear-recent { + background: transparent; + border: 0; + color: #2563eb; + cursor: pointer; + font: inherit; + font-size: 0.875rem; + padding: 0; + } + + .clear-recent:hover { + text-decoration: underline; + } + .no-results { padding: 2rem; text-align: center; @@ -168,8 +217,8 @@ class NavigationSearch extends HTMLElement { <input type="text" class="search-input" - placeholder="Search..." - aria-label="Search navigation" + placeholder="Jump to..." + aria-label="Jump to" autocomplete="off" spellcheck="false" > @@ -231,6 +280,13 @@ class NavigationSearch extends HTMLElement { // Click on result item resultsContainer.addEventListener("click", (e) => { + const clearRecent = e.target.closest("[data-clear-recent-items]"); + if (clearRecent) { + e.preventDefault(); + this.clearRecentItems(); + return; + } + const item = e.target.closest(".result-item"); if (item) { const index = parseInt(item.dataset.index); @@ -306,12 +362,13 @@ class NavigationSearch extends HTMLElement { filterLocalItems(query) { if (!query.trim()) { - this.matches = []; + this.matches = this.allItems || []; } else { const lowerQuery = query.toLowerCase(); this.matches = (this.allItems || []).filter( (item) => item.name.toLowerCase().includes(lowerQuery) || + (item.display_name || "").toLowerCase().includes(lowerQuery) || item.url.toLowerCase().includes(lowerQuery), ); } @@ -319,43 +376,165 @@ class NavigationSearch extends HTMLElement { this.renderResults(); } - renderResults() { - const container = this.shadowRoot.querySelector(".results-container"); - const input = this.shadowRoot.querySelector(".search-input"); + recentItemsStorageKey() { + return "datasette.navigationSearch.recentItems"; + } - if (this.matches.length === 0) { - const message = input.value.trim() - ? "No results found" - : "Start typing to search..."; - container.innerHTML = `<div class="no-results">${message}</div>`; + loadRecentItems() { + if (typeof localStorage === "undefined") { + return []; + } + + try { + const raw = localStorage.getItem(this.recentItemsStorageKey()); + if (!raw) { + return []; + } + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) { + return []; + } + return parsed + .filter((item) => item && item.name && item.url) + .map((item) => ({ + name: String(item.name), + display_name: item.display_name ? String(item.display_name) : "", + url: String(item.url), + type: item.type ? String(item.type) : "", + description: item.description ? String(item.description) : "", + })) + .slice(0, 5); + } catch (e) { + return []; + } + } + + saveRecentItem(match) { + if (typeof localStorage === "undefined" || !match || !match.name || !match.url) { return; } - container.innerHTML = this.matches - .map( - (match, index) => ` - <div - class="result-item ${ - index === this.selectedIndex ? "selected" : "" - }" + try { + const item = { + name: String(match.name), + display_name: match.display_name ? String(match.display_name) : "", + url: String(match.url), + type: match.type ? String(match.type) : "", + description: match.description ? String(match.description) : "", + }; + const recentItems = this.loadRecentItems().filter( + (recentItem) => recentItem.url !== item.url, + ); + localStorage.setItem( + this.recentItemsStorageKey(), + JSON.stringify([item, ...recentItems].slice(0, 5)), + ); + } catch (e) { + // localStorage may be unavailable, full, or disabled. + } + } + + clearRecentItems() { + if (typeof localStorage === "undefined") { + return; + } + + try { + localStorage.removeItem(this.recentItemsStorageKey()); + } catch (e) { + localStorage.setItem(this.recentItemsStorageKey(), "[]"); + } + this.renderResults(); + } + + startContentHtml() { + const template = this.querySelector("template[data-jump-start]"); + return template ? template.innerHTML.trim() : ""; + } + + resultItemHtml(match, index) { + const displayName = match.display_name || match.name; + const label = + match.display_name && match.display_name !== match.name + ? `<div class="result-label">${this.escapeHtml(match.name)}</div>` + : ""; + const description = match.description + ? `<div class="result-description">${this.escapeHtml( + match.description, + )}</div>` + : ""; + return ` + <div + class="result-item ${index === this.selectedIndex ? "selected" : ""}" data-index="${index}" role="option" aria-selected="${index === this.selectedIndex}" > <div> - <div class="result-name">${this.escapeHtml( - match.name, - )}</div> + ${description} + <div class="result-name">${this.escapeHtml(displayName)}</div> + ${label} <div class="result-url">${this.escapeHtml(match.url)}</div> </div> </div> - `, + `; + } + + renderResults() { + const container = this.shadowRoot.querySelector(".results-container"); + const input = this.shadowRoot.querySelector(".search-input"); + const showStartContent = !input.value.trim(); + const startContent = showStartContent ? this.startContentHtml() : ""; + const startBlock = startContent + ? `<div class="jump-start-content">${startContent}</div>` + : ""; + const recentItems = showStartContent ? this.loadRecentItems() : []; + const defaultMatches = showStartContent ? [] : this.matches; + const renderedMatches = [...recentItems, ...defaultMatches]; + this.renderedMatches = renderedMatches; + + if (renderedMatches.length) { + if ( + this.selectedIndex < 0 || + this.selectedIndex >= renderedMatches.length + ) { + this.selectedIndex = 0; + } + } else { + this.selectedIndex = -1; + } + + if (renderedMatches.length === 0) { + if (startBlock) { + container.innerHTML = startBlock; + } else if (showStartContent) { + container.innerHTML = ""; + } else { + const message = input.value.trim() + ? "No results found" + : "Start typing to search..."; + container.innerHTML = `<div class="no-results">${message}</div>`; + } + return; + } + + const recentHtml = recentItems.length + ? `<div class="results-heading">Recent</div>${recentItems + .map((match, index) => this.resultItemHtml(match, index)) + .join("")}<div class="recent-actions"><button type="button" class="clear-recent" data-clear-recent-items>Clear recent</button></div>` + : ""; + const defaultHtml = defaultMatches + .map((match, index) => + this.resultItemHtml(match, recentItems.length + index), ) .join(""); + container.innerHTML = startBlock + recentHtml + defaultHtml; // Scroll selected item into view if (this.selectedIndex >= 0) { - const selectedItem = container.children[this.selectedIndex]; + const selectedItem = container.querySelector( + `.result-item[data-index="${this.selectedIndex}"]`, + ); if (selectedItem) { selectedItem.scrollIntoView({ block: "nearest" }); } @@ -363,22 +542,27 @@ class NavigationSearch extends HTMLElement { } moveSelection(direction) { + const matches = this.renderedMatches || this.matches; const newIndex = this.selectedIndex + direction; - if (newIndex >= 0 && newIndex < this.matches.length) { + if (newIndex >= 0 && newIndex < matches.length) { this.selectedIndex = newIndex; this.renderResults(); } } selectCurrentItem() { - if (this.selectedIndex >= 0 && this.selectedIndex < this.matches.length) { + const matches = this.renderedMatches || this.matches; + if (this.selectedIndex >= 0 && this.selectedIndex < matches.length) { this.selectItem(this.selectedIndex); } } selectItem(index) { - const match = this.matches[index]; + const matches = this.renderedMatches || this.matches; + const match = matches[index]; if (match) { + this.saveRecentItem(match); + // Dispatch custom event this.dispatchEvent( new CustomEvent("select", { @@ -405,7 +589,7 @@ class NavigationSearch extends HTMLElement { input.value = ""; input.focus(); - // Reset state - start with no items shown + // Reset state, then populate the default jump list. this.matches = []; this.selectedIndex = -1; this.renderResults(); @@ -418,7 +602,7 @@ class NavigationSearch extends HTMLElement { escapeHtml(text) { const div = document.createElement("div"); - div.textContent = text; + div.textContent = text == null ? "" : text; return div.innerHTML; } } diff --git a/datasette/templates/base.html b/datasette/templates/base.html index b4fecf70..6a55bd1f 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -70,7 +70,14 @@ {% endfor %} {% if select_templates %}<!-- Templates considered: {{ select_templates|join(", ") }} -->{% endif %} -<script src="{{ urls.static('navigation-search.js') }}" defer></script> -<navigation-search url="/-/tables"></navigation-search> +<script src="{{ urls.static('navigation-search.js') }}{% if navigation_search_js_hash is defined %}?{{ navigation_search_js_hash }}{% endif %}" defer></script> +{% if jump_start is defined %} +{% set jump_start_html = jump_start() %} +{% else %} +{% set jump_start_html = "" %} +{% endif %} +<navigation-search url="/-/jump">{% if jump_start_html %} + <template data-jump-start>{{ jump_start_html }}</template> +{% endif %}</navigation-search> </body> </html> diff --git a/datasette/views/special.py b/datasette/views/special.py index b28e9257..990714cf 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -1,11 +1,14 @@ import json import logging +from datasette.jump import JumpSQL, namespace_sql_params +from datasette.plugins import pm from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent from datasette.resources import DatabaseResource, TableResource from datasette.utils.asgi import Response, Forbidden from datasette.utils import ( actor_matches_allow, add_cors_headers, + await_me_maybe, tilde_encode, tilde_decode, ) @@ -910,75 +913,231 @@ class ApiExplorerView(BaseView): ) -class TablesView(BaseView): +class JumpView(BaseView): """ - Simple endpoint that uses the new allowed_resources() API. - Returns JSON list of all tables the actor can view. - - Supports ?q=foo+bar to filter tables matching .*foo.*bar.* pattern, - ordered by shortest name first. + Endpoint for the jump menu. Returns JSON navigation items the actor can use. """ - name = "tables" + name = "jump" has_json_alternate = False - async def get(self, request): - # Get search query parameter - q = request.args.get("q", "").strip() + async def _query_display_names_sql(self, request): + selects = [] + params = {} + for database_name in self.ds.databases.keys(): + queries = await self.ds.get_canned_queries(database_name, request.actor) + for query_name, query in queries.items(): + display_name = query.get("title") if isinstance(query, dict) else None + if not display_name: + continue + index = len(selects) + params[f"display_database_{index}"] = database_name + params[f"display_query_{index}"] = query_name + params[f"display_name_{index}"] = str(display_name) + selects.append(f""" + SELECT + :display_database_{index} AS database_name, + :display_query_{index} AS query_name, + :display_name_{index} AS display_name + """) + if not selects: + return ( + "SELECT NULL AS database_name, NULL AS query_name, NULL AS display_name WHERE 0", + {}, + ) + return " UNION ALL ".join(selects), params - # Get SQL for allowed resources using the permission system - permission_sql, params = await self.ds.allowed_resources_sql( + async def _core_fragments(self, request): + database_sql, database_params = await self.ds.allowed_resources_sql( + action="view-database", actor=request.actor + ) + table_sql, table_params = await self.ds.allowed_resources_sql( action="view-table", actor=request.actor ) + query_sql, query_params = await self.ds.allowed_resources_sql( + action="view-query", actor=request.actor + ) + query_display_names_sql, query_display_names_params = ( + await self._query_display_names_sql(request) + ) + return [ + JumpSQL( + sql=f""" + WITH allowed_databases AS ( + {database_sql} + ) + SELECT + 'database' AS type, + parent AS label, + 'Database' AS description, + NULL AS url, + parent AS database_name, + NULL AS resource_name, + parent AS search_text, + 10 AS sort_key, + 'datasette' AS source, + NULL AS display_name + FROM allowed_databases + """, + params=database_params, + has_display_name=True, + ), + JumpSQL( + sql=f""" + WITH allowed_tables AS ( + {table_sql} + ) + SELECT + CASE WHEN catalog_views.view_name IS NULL THEN 'table' ELSE 'view' END AS type, + allowed_tables.parent || ': ' || allowed_tables.child AS label, + CASE WHEN catalog_views.view_name IS NULL THEN 'Table' ELSE 'View' END AS description, + NULL AS url, + allowed_tables.parent AS database_name, + allowed_tables.child AS resource_name, + allowed_tables.parent || ' ' || allowed_tables.child AS search_text, + CASE WHEN catalog_views.view_name IS NULL THEN 20 ELSE 25 END AS sort_key, + 'datasette' AS source, + NULL AS display_name + FROM allowed_tables + LEFT JOIN catalog_views + ON catalog_views.database_name = allowed_tables.parent + AND catalog_views.view_name = allowed_tables.child + """, + params=table_params, + has_display_name=True, + ), + JumpSQL( + sql=f""" + WITH allowed_queries AS ( + {query_sql} + ), + query_display_names AS ( + {query_display_names_sql} + ) + SELECT + 'query' AS type, + allowed_queries.parent || ': ' || allowed_queries.child AS label, + 'Canned query' AS description, + NULL AS url, + allowed_queries.parent AS database_name, + allowed_queries.child AS resource_name, + allowed_queries.parent || ' ' || allowed_queries.child || ' ' || COALESCE(query_display_names.display_name, '') AS search_text, + 30 AS sort_key, + 'datasette' AS source, + query_display_names.display_name AS display_name + FROM allowed_queries + LEFT JOIN query_display_names + ON query_display_names.database_name = allowed_queries.parent + AND query_display_names.query_name = allowed_queries.child + """, + params={**query_params, **query_display_names_params}, + has_display_name=True, + ), + ] - # Build query based on whether we have a search query - if q: - # Build SQL LIKE pattern from search terms - # Split search terms by whitespace and build pattern: %term1%term2%term3% - terms = q.split() - pattern = "%" + "%".join(terms) + "%" + async def _plugin_fragments(self, request): + fragments = [] + for hook in pm.hook.jump_items_sql( + datasette=self.ds, + actor=request.actor, + request=request, + ): + value = await await_me_maybe(hook) + if value is None: + continue + if isinstance(value, JumpSQL): + fragments.append(value) + elif isinstance(value, (list, tuple)): + for fragment in value: + if fragment is not None: + assert isinstance( + fragment, JumpSQL + ), "jump_items_sql must return JumpSQL instances" + fragments.append(fragment) + else: + raise TypeError("jump_items_sql must return JumpSQL instances") + return fragments - # Build query with CTE to filter by search pattern - sql = f""" - WITH allowed_tables AS ( - {permission_sql} + def _url_for_row(self, row): + if row["url"]: + return row["url"] + if row["type"] == "database": + return self.ds.urls.database(row["database_name"]) + if row["type"] in ("table", "view"): + return self.ds.urls.table(row["database_name"], row["resource_name"]) + if row["type"] == "query": + return self.ds.urls.query(row["database_name"], row["resource_name"]) + return "" + + async def get(self, request): + q = request.args.get("q", "").strip() + terms = q.split() + pattern = "%" + "%".join(terms) + "%" if terms else "%" + fragments = await self._core_fragments(request) + fragments.extend(await self._plugin_fragments(request)) + + union_parts = [] + all_params = {"q": q, "pattern": pattern} + for index, fragment in enumerate(fragments): + fragment_sql, fragment_params = namespace_sql_params( + fragment.sql, + fragment.params or {}, + f"jump_{index}", ) - SELECT parent, child - FROM allowed_tables - WHERE child LIKE :pattern COLLATE NOCASE - ORDER BY length(child), child - """ - all_params = {**params, "pattern": pattern} - else: - # No search query - return all tables, ordered by name - # Fetch 101 to detect if we need to truncate - sql = f""" - WITH allowed_tables AS ( - {permission_sql} - ) - SELECT parent, child - FROM allowed_tables - ORDER BY parent, child - LIMIT 101 - """ - all_params = params + union_parts.append(f""" + SELECT + type, + label, + description, + url, + database_name, + resource_name, + search_text, + sort_key, + source, + {"display_name" if fragment.has_display_name else "NULL AS display_name"} + FROM ( + {fragment_sql} + ) + """) + all_params.update(fragment_params) - # Execute against internal database + sql = f""" + WITH jump_items AS ( + {" UNION ALL ".join(union_parts)} + ) + SELECT * + FROM jump_items + WHERE :q = '' + OR search_text LIKE :pattern COLLATE NOCASE + ORDER BY + CASE + WHEN lower(COALESCE(display_name, label)) = lower(:q) THEN 0 + WHEN lower(COALESCE(display_name, label)) LIKE lower(:q || '%') THEN 1 + ELSE 2 + END, + sort_key, + length(COALESCE(display_name, label)), + label + LIMIT 101 + """ result = await self.ds.get_internal_database().execute(sql, all_params) - - # Build response with truncation rows = list(result.rows) truncated = len(rows) > 100 if truncated: rows = rows[:100] - matches = [ - { - "name": f"{row['parent']}: {row['child']}", - "url": self.ds.urls.table(row["parent"], row["child"]), + matches = [] + for row in rows: + match = { + "name": row["label"], + "url": self._url_for_row(row), + "type": row["type"], + "description": row["description"], } - for row in rows - ] + if row["display_name"]: + match["display_name"] = row["display_name"] + matches.append(match) return Response.json({"matches": matches, "truncated": truncated}) diff --git a/docs/introspection.rst b/docs/introspection.rst index 19c6bffb..9f0358ac 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -144,46 +144,65 @@ Shows currently attached databases. `Databases example <https://latest.datasette } ] -.. _TablesView: +.. _JumpView: -/-/tables ---------- +/-/jump +------- -Returns a JSON list of all tables that the current actor has permission to view. This endpoint uses the resource-based permission system and respects database and table-level access controls. +Returns a JSON list of items that the current actor has permission to view for Datasette's jump menu. By default this includes visible databases, tables, views and canned queries, and plugins can contribute additional items. -The endpoint supports a ``?q=`` query parameter for filtering tables by name using case-insensitive regex matching. +The endpoint supports a ``?q=`` query parameter for filtering items by name. +Canned queries with a configured ``title`` also include a ``display_name`` in +their results, and can be found by searching for that title. Plugins can provide +the same extra field from ``jump_items_sql`` by returning a ``display_name`` +column and setting ``JumpSQL(..., has_display_name=True)``. -`Tables example <https://latest.datasette.io/-/tables>`_: +`Jump example <https://latest.datasette.io/-/jump>`_: .. code-block:: json { "matches": [ { - "name": "fixtures/facetable", - "url": "/fixtures/facetable" + "name": "fixtures", + "url": "/fixtures", + "type": "database", + "description": "Database" }, { - "name": "fixtures/searchable", - "url": "/fixtures/searchable" + "name": "fixtures: facetable", + "url": "/fixtures/facetable", + "type": "table", + "description": "Table" + }, + { + "name": "fixtures: recent_releases", + "display_name": "Recent Datasette releases", + "url": "/fixtures/recent_releases", + "type": "query", + "description": "Canned query" } - ] + ], + "truncated": false } -Search example with ``?q=facet`` returns only tables matching ``.*facet.*``: +Search example with ``?q=facet`` returns only items matching ``.*facet.*``: .. code-block:: json { "matches": [ { - "name": "fixtures/facetable", - "url": "/fixtures/facetable" + "name": "fixtures: facetable", + "url": "/fixtures/facetable", + "type": "table", + "description": "Table" } - ] + ], + "truncated": false } -When multiple search terms are provided (e.g., ``?q=user+profile``), tables must match the pattern ``.*user.*profile.*``. Results are ordered by shortest table name first. +When multiple search terms are provided (e.g., ``?q=user+profile``), items must match the pattern ``.*user.*profile.*``. Results are ordered by relevance, then by item type and shortest display name. .. _JsonDataView_threads: diff --git a/tests/test_allowed_resources.py b/tests/test_allowed_resources.py index 08adbe48..8048ae2c 100644 --- a/tests/test_allowed_resources.py +++ b/tests/test_allowed_resources.py @@ -52,7 +52,7 @@ async def test_ds(): @pytest.mark.asyncio async def test_tables_endpoint_global_access(test_ds): - """Test /-/tables with global access permissions""" + """Test allowed_resources() with global access permissions""" def rules_callback(datasette, actor, action): if actor and actor.get("id") == "alice": @@ -91,7 +91,7 @@ async def test_tables_endpoint_global_access(test_ds): @pytest.mark.asyncio async def test_tables_endpoint_database_restriction(test_ds): - """Test /-/tables with database-level restriction""" + """Test allowed_resources() with database-level restriction""" def rules_callback(datasette, actor, action): if actor and actor.get("role") == "analyst": @@ -133,7 +133,7 @@ async def test_tables_endpoint_database_restriction(test_ds): @pytest.mark.asyncio async def test_tables_endpoint_table_exception(test_ds): - """Test /-/tables with table-level exception (deny database, allow specific table)""" + """Test allowed_resources() with table-level exception (deny database, allow specific table)""" def rules_callback(datasette, actor, action): if actor and actor.get("id") == "carol": @@ -217,7 +217,7 @@ async def test_tables_endpoint_deny_overrides_allow(test_ds): @pytest.mark.asyncio async def test_tables_endpoint_no_permissions(): - """Test /-/tables when user has no custom permissions (only defaults)""" + """Test allowed_resources() when user has no custom permissions (only defaults)""" ds = Datasette() await ds.invoke_startup() @@ -241,7 +241,7 @@ async def test_tables_endpoint_no_permissions(): @pytest.mark.asyncio async def test_tables_endpoint_specific_table_only(test_ds): - """Test /-/tables when only specific tables are allowed (no parent/global rules)""" + """Test allowed_resources() when only specific tables are allowed (no parent/global rules)""" def rules_callback(datasette, actor, action): if actor and actor.get("id") == "dave": @@ -283,7 +283,7 @@ async def test_tables_endpoint_specific_table_only(test_ds): @pytest.mark.asyncio async def test_tables_endpoint_empty_result(test_ds): - """Test /-/tables when all tables are explicitly denied""" + """Test allowed_resources() when all tables are explicitly denied""" def rules_callback(datasette, actor, action): if actor and actor.get("id") == "blocked": @@ -314,7 +314,7 @@ async def test_tables_endpoint_empty_result(test_ds): @pytest.mark.asyncio async def test_tables_endpoint_no_query_returns_all(): - """Test /-/tables without query parameter returns all tables""" + """Test allowed_resources() without query parameter returns all tables""" ds = Datasette() await ds.invoke_startup() @@ -338,7 +338,7 @@ async def test_tables_endpoint_no_query_returns_all(): @pytest.mark.asyncio async def test_tables_endpoint_truncation(): - """Test /-/tables truncates at 100 tables and sets truncated flag""" + """Test allowed_resources() truncates at 100 tables and sets truncated flag""" ds = Datasette() await ds.invoke_startup() @@ -359,7 +359,7 @@ async def test_tables_endpoint_truncation(): @pytest.mark.asyncio async def test_tables_endpoint_search_single_term(): - """Test /-/tables?q=user to filter tables matching 'user'""" + """Test allowed_resources()?q=user to filter tables matching 'user'""" ds = Datasette() await ds.invoke_startup() @@ -396,7 +396,7 @@ async def test_tables_endpoint_search_single_term(): @pytest.mark.asyncio async def test_tables_endpoint_search_multiple_terms(): - """Test /-/tables?q=user+profile to filter tables matching .*user.*profile.*""" + """Test allowed_resources()?q=user+profile to filter tables matching .*user.*profile.*""" ds = Datasette() await ds.invoke_startup() diff --git a/tests/test_html.py b/tests/test_html.py index 64b4075a..ac77f10f 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1019,6 +1019,13 @@ async def test_navigation_menu_links( search_button.find("kbd")["title"] == "Keyboard shortcut: press / to open Jump to" ) + navigation_search_script = soup.find( + "script", {"src": re.compile(r"navigation-search\.js")} + ) + assert navigation_search_script["src"] == ( + "/-/static/navigation-search.js?" + + ds_client.ds.static_hash("navigation-search.js") + ) assert details.find("li").find("button") == search_button if not actor_id: # The app menu is always visible, but anonymous users do not see logout diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index dcf14126..26d63a92 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -228,7 +228,7 @@ async def test_orphan_stale_catalog_child_entries_removed(tmp_path): """) assert [tuple(row) for row in catalog_tables.rows] == [("alpha", "alpha_table")] - response = await ds2.client.get("/-/tables.json") + response = await ds2.client.get("/-/jump.json") assert response.status_code == 200 ds2.close() diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py index ccac280b..543077a5 100644 --- a/tests/test_internals_datasette_client.py +++ b/tests/test_internals_datasette_client.py @@ -195,7 +195,7 @@ async def test_skip_permission_checks_with_admin_actor(datasette_with_permission @pytest.mark.asyncio async def test_skip_permission_checks_shows_denied_tables(): - """Test that skip_permission_checks=True shows tables from denied databases in /-/tables.json""" + """Test that skip_permission_checks=True shows tables from denied databases in /-/jump.json""" ds = Datasette( config={ "databases": { @@ -211,8 +211,8 @@ async def test_skip_permission_checks_shows_denied_tables(): await db.execute_write("INSERT INTO test_table (id, name) VALUES (1, 'Alice')") await ds._refresh_schemas() - # Without skip_permission_checks, tables from denied database should not appear in /-/tables.json - response = await ds.client.get("/-/tables.json") + # Without skip_permission_checks, tables from denied database should not appear in /-/jump.json + response = await ds.client.get("/-/jump.json") assert response.status_code == 200 data = response.json() table_names = [match["name"] for match in data["matches"]] @@ -221,7 +221,7 @@ async def test_skip_permission_checks_shows_denied_tables(): assert len(fixtures_tables) == 0 # With skip_permission_checks=True, tables from denied database SHOULD appear - response = await ds.client.get("/-/tables.json", skip_permission_checks=True) + response = await ds.client.get("/-/jump.json", skip_permission_checks=True) assert response.status_code == 200 data = response.json() table_names = [match["name"] for match in data["matches"]] diff --git a/tests/test_jump.py b/tests/test_jump.py new file mode 100644 index 00000000..55325b95 --- /dev/null +++ b/tests/test_jump.py @@ -0,0 +1,224 @@ +import pytest +import pytest_asyncio +from markupsafe import Markup + +from datasette import hookimpl +from datasette.app import Datasette +from datasette.plugins import pm + + +@pytest_asyncio.fixture +async def ds_for_jump(): + ds = Datasette( + config={ + "databases": { + "content": { + "allow": {"id": "*"}, + "tables": { + "articles": {"allow": {"id": "editor"}}, + "comments": {"allow": True}, + }, + "queries": { + "recent_comments": { + "sql": "select * from comments", + "allow": {"id": "*"}, + "title": "Recent comments", + }, + "release_notes": { + "sql": "select 1", + "allow": {"id": "*"}, + "title": "Recent Datasette releases", + }, + "editor_report": { + "sql": "select * from articles", + "allow": {"id": "editor"}, + }, + }, + }, + "private": { + "allow": False, + "queries": { + "private_report": "select 1", + }, + }, + } + } + ) + await ds.invoke_startup() + + content_db = ds.add_memory_database("jump_test_content", name="content") + await content_db.execute_write( + "CREATE TABLE IF NOT EXISTS articles (id INTEGER PRIMARY KEY, title TEXT)" + ) + await content_db.execute_write( + "CREATE TABLE IF NOT EXISTS comments (id INTEGER PRIMARY KEY, body TEXT)" + ) + await content_db.execute_write( + "CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY, name TEXT)" + ) + await content_db.execute_write( + "CREATE VIEW IF NOT EXISTS comment_summary AS SELECT body FROM comments" + ) + + private_db = ds.add_memory_database("jump_test_private", name="private") + await private_db.execute_write( + "CREATE TABLE IF NOT EXISTS secrets (id INTEGER PRIMARY KEY, data TEXT)" + ) + + public_db = ds.add_memory_database("jump_test_public", name="public") + await public_db.execute_write( + "CREATE TABLE IF NOT EXISTS articles (id INTEGER PRIMARY KEY, content TEXT)" + ) + + await ds._refresh_schemas() + return ds + + +@pytest.mark.asyncio +async def test_jump_searches_tables_databases_views_and_canned_queries(ds_for_jump): + response = await ds_for_jump.client.get( + "/-/jump.json?q=content", actor={"id": "user"} + ) + assert response.status_code == 200 + data = response.json() + + matches_by_type_and_name = { + (match["type"], match["name"]): match for match in data["matches"] + } + assert ("database", "content") in matches_by_type_and_name + assert ("table", "content: comments") in matches_by_type_and_name + assert ("view", "content: comment_summary") in matches_by_type_and_name + assert ("query", "content: recent_comments") in matches_by_type_and_name + assert matches_by_type_and_name[("database", "content")]["url"] == "/content" + assert ( + matches_by_type_and_name[("query", "content: recent_comments")]["display_name"] + == "Recent comments" + ) + assert ( + matches_by_type_and_name[("query", "content: recent_comments")]["url"] + == "/content/recent_comments" + ) + + +@pytest.mark.asyncio +async def test_jump_searches_and_displays_canned_query_titles(ds_for_jump): + response = await ds_for_jump.client.get( + "/-/jump.json?q=datasette", actor={"id": "user"} + ) + assert response.status_code == 200 + data = response.json() + + assert data["matches"] == [ + { + "name": "content: release_notes", + "display_name": "Recent Datasette releases", + "url": "/content/release_notes", + "type": "query", + "description": "Canned query", + } + ] + + +@pytest.mark.asyncio +async def test_jump_respects_resource_permissions(ds_for_jump): + regular = await ds_for_jump.client.get( + "/-/jump.json?q=articles", actor={"id": "regular"} + ) + editor = await ds_for_jump.client.get( + "/-/jump.json?q=articles", actor={"id": "editor"} + ) + private = await ds_for_jump.client.get( + "/-/jump.json?q=secrets", actor={"id": "editor"} + ) + + assert {match["name"] for match in regular.json()["matches"]} == { + "public: articles" + } + assert {match["name"] for match in editor.json()["matches"]} == { + "content: articles", + "public: articles", + } + assert private.json()["matches"] == [] + + +@pytest.mark.asyncio +async def test_jump_uses_plugin_sql_with_namespaced_parameters(ds_for_jump): + from datasette.jump import JumpSQL + + class JumpPlugin: + @hookimpl + def jump_items_sql(self, datasette, actor, request): + return JumpSQL( + sql=""" + SELECT + 'plugin' AS type, + 'plugin-dashboard: ' || :actor_id AS label, + 'Plugin supplied item' AS description, + '/-/plugin-dashboard' AS url, + NULL AS database_name, + NULL AS resource_name, + 'plugin dashboard ' || :actor_id AS search_text, + 80 AS sort_key, + 'test-plugin' AS source, + 'Plugin dashboard for ' || :actor_id AS display_name + """, + params={"actor_id": actor["id"] if actor else "anonymous"}, + has_display_name=True, + ) + + plugin = JumpPlugin() + pm.register(plugin, name="test-jump-plugin") + try: + response = await ds_for_jump.client.get( + "/-/jump.json?q=dashboard", actor={"id": "alice"} + ) + finally: + pm.unregister(name="test-jump-plugin") + + assert response.status_code == 200 + plugin_matches = [ + match for match in response.json()["matches"] if match["type"] == "plugin" + ] + assert plugin_matches == [ + { + "name": "plugin-dashboard: alice", + "display_name": "Plugin dashboard for alice", + "url": "/-/plugin-dashboard", + "type": "plugin", + "description": "Plugin supplied item", + } + ] + + +@pytest.mark.asyncio +async def test_jump_start_hook_renders_empty_state_template(ds_for_jump): + class JumpStartPlugin: + @hookimpl + def jump_start(self, datasette, actor, request): + if not actor: + return None + return Markup( + '<section class="agent-jump-start">' + "<h3>Agent chat</h3>" + '<a href="/-/agent/new">Start a new agent chat</a>' + "</section>" + ) + + plugin = JumpStartPlugin() + pm.register(plugin, name="test-jump-start-plugin") + try: + anonymous = await ds_for_jump.client.get("/") + authenticated = await ds_for_jump.client.get("/", actor={"id": "alice"}) + finally: + pm.unregister(name="test-jump-start-plugin") + + assert 'url="/-/jump"' in authenticated.text + assert "<template data-jump-start>" not in anonymous.text + assert "<template data-jump-start>" in authenticated.text + assert "Start a new agent chat" in authenticated.text + + +@pytest.mark.asyncio +async def test_tables_endpoint_removed(ds_for_jump): + response = await ds_for_jump.client.get("/-/tables.json") + assert response.status_code == 404 diff --git a/tests/test_navigation_search_js.py b/tests/test_navigation_search_js.py new file mode 100644 index 00000000..3096d9a0 --- /dev/null +++ b/tests/test_navigation_search_js.py @@ -0,0 +1,199 @@ +import json +import subprocess +import textwrap + + +def test_navigation_search_tracks_and_renders_recent_items(): + script = textwrap.dedent(""" + const fs = require("fs"); + const vm = require("vm"); + + class FakeElement { + constructor() { + this.innerHTML = ""; + this.value = ""; + this.dataset = {}; + this.open = false; + } + addEventListener() {} + close() { this.open = false; } + focus() {} + querySelector() { + return { scrollIntoView() {} }; + } + showModal() { this.open = true; } + } + + class FakeShadowRoot { + constructor() { + this.innerHTML = ""; + this.dialog = new FakeElement(); + this.input = new FakeElement(); + this.results = new FakeElement(); + } + querySelector(selector) { + if (selector == "dialog") return this.dialog; + if (selector == ".search-input") return this.input; + if (selector == ".results-container") return this.results; + return new FakeElement(); + } + } + + global.HTMLElement = class { + constructor() { + this.attributes = {}; + } + attachShadow() { + this.shadowRoot = new FakeShadowRoot(); + return this.shadowRoot; + } + dispatchEvent() {} + getAttribute(name) { + return this.attributes[name] || null; + } + querySelector() { + return null; + } + setAttribute(name, value) { + this.attributes[name] = value; + } + }; + global.CustomEvent = class { + constructor(name, options) { + this.name = name; + this.options = options; + } + }; + global.customElements = { + registry: new Map(), + define(name, cls) { + this.registry.set(name, cls); + }, + }; + global.document = { + addEventListener() {}, + activeElement: null, + createElement() { + return { + set textContent(value) { + this.innerHTML = String(value) + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, """); + }, + }; + }, + }; + global.localStorage = { + store: {}, + getItem(key) { + return Object.prototype.hasOwnProperty.call(this.store, key) + ? this.store[key] + : null; + }, + setItem(key, value) { + this.store[key] = String(value); + }, + removeItem(key) { + delete this.store[key]; + }, + }; + global.window = { location: { href: "" } }; + + vm.runInThisContext( + fs.readFileSync("datasette/static/navigation-search.js", "utf8"), + { filename: "navigation-search.js" } + ); + + const Component = customElements.registry.get("navigation-search"); + const element = new Component(); + const items = Array.from({ length: 6 }, (_, index) => ({ + name: `Item ${index + 1}`, + url: `/item-${index + 1}`, + type: "table", + description: "Table", + })); + items[5].name = "content: recent_datasette_releases"; + items[5].display_name = "Recent Datasette releases"; + + for (const item of items) { + element.matches = [item]; + element.renderedMatches = [item]; + element.selectedIndex = 0; + element.selectCurrentItem(); + } + + const stored = JSON.parse( + Object.values(localStorage.store).find((value) => value.includes("/item-6")) + ); + if (stored.length !== 5) { + throw new Error(`Expected 5 recent items, got ${stored.length}`); + } + if (stored[0].url !== "/item-6" || stored[4].url !== "/item-2") { + throw new Error(`Unexpected recent order: ${JSON.stringify(stored)}`); + } + if (stored[0].display_name !== "Recent Datasette releases") { + throw new Error(`Missing display_name in recent item: ${JSON.stringify(stored[0])}`); + } + + element.matches = [ + items[5], + items[4], + { + name: "Other", + url: "/other", + type: "database", + description: "Database", + }, + ]; + element.shadowRoot.input.value = ""; + element.renderResults(); + + const html = element.shadowRoot.results.innerHTML; + if (!html.includes("Recent")) { + throw new Error(`Missing Recent heading: ${html}`); + } + if (!html.includes("Recent Datasette releases") || !html.includes("Item 5")) { + throw new Error(`Missing recent items: ${html}`); + } + if (!html.includes("content: recent_datasette_releases")) { + throw new Error(`Missing canonical item name for display_name item: ${html}`); + } + if (!html.includes("Item 4") || !html.includes("Item 2")) { + throw new Error(`Expected all stored recent items in empty state: ${html}`); + } + if (html.includes("Other")) { + throw new Error(`Rendered non-recent item in empty state: ${html}`); + } + if (!html.includes("Clear recent")) { + throw new Error(`Missing Clear recent control: ${html}`); + } + + element.clearRecentItems(); + if (localStorage.getItem(element.recentItemsStorageKey()) !== null) { + throw new Error("Expected recent items to be cleared"); + } + element.renderResults(); + if (element.shadowRoot.results.innerHTML.includes("Clear recent")) { + throw new Error("Clear recent should disappear after clearing"); + } + + process.stdout.write(JSON.stringify(stored)); + """) + result = subprocess.run( + ["node", "-e", script], + cwd=".", + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert [item["url"] for item in json.loads(result.stdout)] == [ + "/item-6", + "/item-5", + "/item-4", + "/item-3", + "/item-2", + ] + assert json.loads(result.stdout)[0]["display_name"] == "Recent Datasette releases" diff --git a/tests/test_search_tables.py b/tests/test_search_tables.py index b901c0b3..ce774327 100644 --- a/tests/test_search_tables.py +++ b/tests/test_search_tables.py @@ -33,7 +33,7 @@ async def ds_with_tables(): await ds.invoke_startup() # Add content database with some tables - content_db = ds.add_memory_database("content") + content_db = ds.add_memory_database("search_tables_content", name="content") await content_db.execute_write( "CREATE TABLE IF NOT EXISTS articles (id INTEGER PRIMARY KEY, title TEXT)" ) @@ -45,27 +45,28 @@ async def ds_with_tables(): ) # Add private database with a table - private_db = ds.add_memory_database("private") + private_db = ds.add_memory_database("search_tables_private", name="private") await private_db.execute_write( "CREATE TABLE IF NOT EXISTS secrets (id INTEGER PRIMARY KEY, data TEXT)" ) # Add another public database - public_db = ds.add_memory_database("public") + public_db = ds.add_memory_database("search_tables_public", name="public") await public_db.execute_write( "CREATE TABLE IF NOT EXISTS articles (id INTEGER PRIMARY KEY, content TEXT)" ) + await ds._refresh_schemas() return ds -# /-/tables.json tests +# /-/jump.json table search tests @pytest.mark.asyncio async def test_tables_basic_search(ds_with_tables): """Test basic table search functionality.""" # Search for "articles" - should find it in both content and public databases # but only return public.articles for anonymous user (content.articles requires auth) - response = await ds_with_tables.client.get("/-/tables.json?q=articles") + response = await ds_with_tables.client.get("/-/jump.json?q=articles") assert response.status_code == 200 data = response.json() @@ -85,7 +86,7 @@ async def test_tables_search_with_auth(ds_with_tables): """Test that authenticated users see more tables.""" # Editor user should see content.articles response = await ds_with_tables.client.get( - "/-/tables.json?q=articles", + "/-/jump.json?q=articles", actor={"id": "editor"}, ) assert response.status_code == 200 @@ -103,7 +104,7 @@ async def test_tables_search_partial_match(ds_with_tables): """Test that search matches partial table names.""" # Search for "com" should match "comments" response = await ds_with_tables.client.get( - "/-/tables.json?q=com", + "/-/jump.json?q=com", actor={"id": "user"}, ) assert response.status_code == 200 @@ -119,7 +120,7 @@ async def test_tables_search_respects_database_permissions(ds_with_tables): # Search for "secrets" which is in the private database # Even authenticated users shouldn't see it because database is denied response = await ds_with_tables.client.get( - "/-/tables.json?q=secrets", + "/-/jump.json?q=secrets", actor={"id": "user"}, ) assert response.status_code == 200 @@ -134,7 +135,7 @@ async def test_tables_search_respects_table_permissions(ds_with_tables): """Test that tables with specific permissions are filtered correctly.""" # Regular authenticated user searching for "users" response = await ds_with_tables.client.get( - "/-/tables.json?q=users", + "/-/jump.json?q=users", actor={"id": "regular"}, ) assert response.status_code == 200 @@ -149,7 +150,7 @@ async def test_tables_search_respects_table_permissions(ds_with_tables): async def test_tables_search_response_structure(ds_with_tables): """Test that response has correct structure.""" response = await ds_with_tables.client.get( - "/-/tables.json?q=users", + "/-/jump.json?q=users", actor={"id": "user"}, ) assert response.status_code == 200 From 1000d50220faef339ee688aae9150e83a205be3a Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 21 May 2026 23:05:37 -0700 Subject: [PATCH 063/156] datasette.fixtures module, closes #2733 https://gist.github.com/simonw/613be79094d491dd08f45e05f4f70691 --- datasette/fixtures.py | 415 ++++++++++++++++++++++++++++++++++++ datasette/utils/__init__.py | 12 +- docs/changelog.rst | 1 + docs/testing_plugins.rst | 42 ++++ tests/conftest.py | 9 +- tests/fixtures.py | 401 ++-------------------------------- tests/test_docs.py | 18 +- tests/test_fixtures.py | 49 +++++ tests/test_plugins.py | 2 +- tests/test_table_api.py | 3 +- 10 files changed, 545 insertions(+), 407 deletions(-) create mode 100644 datasette/fixtures.py create mode 100644 tests/test_fixtures.py diff --git a/datasette/fixtures.py b/datasette/fixtures.py new file mode 100644 index 00000000..7c85e16a --- /dev/null +++ b/datasette/fixtures.py @@ -0,0 +1,415 @@ +from datasette.utils.sqlite import sqlite3 +from datasette.utils import documented +import itertools +import random +import string + +__all__ = [ + "EXTRA_DATABASE_SQL", + "TABLES", + "TABLE_PARAMETERIZED_SQL", + "generate_compound_rows", + "generate_sortable_rows", + "populate_extra_database", + "populate_fixture_database", + "write_extra_database", + "write_fixture_database", +] + + +def generate_compound_rows(num): + """Generate rows for the compound_three_primary_keys fixture table.""" + for a, b, c in itertools.islice( + itertools.product(string.ascii_lowercase, repeat=3), num + ): + yield a, b, c, f"{a}-{b}-{c}" + + +def generate_sortable_rows(num): + """Generate rows for the sortable fixture table.""" + rand = random.Random(42) + for a, b in itertools.islice( + itertools.product(string.ascii_lowercase, repeat=2), num + ): + yield { + "pk1": a, + "pk2": b, + "content": f"{a}-{b}", + "sortable": rand.randint(-100, 100), + "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]), + "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]), + "text": rand.choice(["$null", "$blah"]), + } + + +TABLES = ( + """ +CREATE TABLE simple_primary_key ( + id integer primary key, + content text +); + +CREATE TABLE primary_key_multiple_columns ( + id varchar(30) primary key, + content text, + content2 text +); + +CREATE TABLE primary_key_multiple_columns_explicit_label ( + id varchar(30) primary key, + content text, + content2 text +); + +CREATE TABLE compound_primary_key ( + pk1 varchar(30), + pk2 varchar(30), + content text, + PRIMARY KEY (pk1, pk2) +); + +INSERT INTO compound_primary_key VALUES ('a', 'b', 'c'); +INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c'); +INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO'); + +CREATE TABLE compound_three_primary_keys ( + pk1 varchar(30), + pk2 varchar(30), + pk3 varchar(30), + content text, + PRIMARY KEY (pk1, pk2, pk3) +); +CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); + +CREATE TABLE foreign_key_references ( + pk varchar(30) primary key, + foreign_key_with_label integer, + foreign_key_with_blank_label integer, + foreign_key_with_no_label varchar(30), + foreign_key_compound_pk1 varchar(30), + foreign_key_compound_pk2 varchar(30), + FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id), + FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id), + FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id) + FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2) +); + +CREATE TABLE sortable ( + pk1 varchar(30), + pk2 varchar(30), + content text, + sortable integer, + sortable_with_nulls real, + sortable_with_nulls_2 real, + text text, + PRIMARY KEY (pk1, pk2) +); + +CREATE TABLE no_primary_key ( + content text, + a text, + b text, + c text +); + +CREATE TABLE [123_starts_with_digits] ( + content text +); + +CREATE VIEW paginated_view AS + SELECT + content, + '- ' || content || ' -' AS content_extra + FROM no_primary_key; + +CREATE TABLE "Table With Space In Name" ( + pk varchar(30) primary key, + content text +); + +CREATE TABLE "table/with/slashes.csv" ( + pk varchar(30) primary key, + content text +); + +CREATE TABLE "complex_foreign_keys" ( + pk varchar(30) primary key, + f1 integer, + f2 integer, + f3 integer, + FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id), + FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id), + FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id) +); + +CREATE TABLE "custom_foreign_key_label" ( + pk varchar(30) primary key, + foreign_key_with_custom_label text, + FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id) +); + +CREATE TABLE tags ( + tag TEXT PRIMARY KEY +); + +CREATE TABLE searchable ( + pk integer primary key, + text1 text, + text2 text, + [name with . and spaces] text +); + +CREATE TABLE searchable_tags ( + searchable_id integer, + tag text, + PRIMARY KEY (searchable_id, tag), + FOREIGN KEY (searchable_id) REFERENCES searchable(pk), + FOREIGN KEY (tag) REFERENCES tags(tag) +); + +INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther'); +INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma'); + +INSERT INTO tags VALUES ("canine"); +INSERT INTO tags VALUES ("feline"); + +INSERT INTO searchable_tags (searchable_id, tag) VALUES + (1, "feline"), + (2, "canine") +; + +CREATE VIRTUAL TABLE "searchable_fts" + USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk"); +INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild'); + +CREATE TABLE [select] ( + [group] text, + [having] text, + [and] text, + [json] text +); +INSERT INTO [select] VALUES ('group', 'having', 'and', + '{"href": "http://example.com/", "label":"Example"}' +); + +CREATE TABLE infinity ( + value REAL +); +INSERT INTO infinity VALUES + (1e999), + (-1e999), + (1.5) +; + +CREATE TABLE facet_cities ( + id integer primary key, + name text +); +INSERT INTO facet_cities (id, name) VALUES + (1, 'San Francisco'), + (2, 'Los Angeles'), + (3, 'Detroit'), + (4, 'Memnonia') +; + +CREATE TABLE facetable ( + pk integer primary key, + created text, + planet_int integer, + on_earth integer, + state text, + _city_id integer, + _neighborhood text, + tags text, + complex_array text, + distinct_some_null, + n text, + FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id) +); +INSERT INTO facetable + (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n) +VALUES + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null), + ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null), + ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null), + ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null), + ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null) +; + +CREATE TABLE binary_data ( + data BLOB +); + +-- Many 2 Many demo: roadside attractions! + +CREATE TABLE roadside_attractions ( + pk integer primary key, + name text, + address text, + url text, + latitude real, + longitude real +); +INSERT INTO roadside_attractions VALUES ( + 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", + 37.0167, -122.0024 +); +INSERT INTO roadside_attractions VALUES ( + 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", + 37.3184, -121.9511 +); +INSERT INTO roadside_attractions VALUES ( + 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, + 37.5793, -122.3442 +); +INSERT INTO roadside_attractions VALUES ( + 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", + 37.0414, -122.0725 +); + +CREATE TABLE attraction_characteristic ( + pk integer primary key, + name text +); +INSERT INTO attraction_characteristic VALUES ( + 1, "Museum" +); +INSERT INTO attraction_characteristic VALUES ( + 2, "Paranormal" +); + +CREATE TABLE roadside_attraction_characteristics ( + attraction_id INTEGER REFERENCES roadside_attractions(pk), + characteristic_id INTEGER REFERENCES attraction_characteristic(pk) +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 1, 2 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 2, 2 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 4, 2 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 3, 1 +); +INSERT INTO roadside_attraction_characteristics VALUES ( + 4, 1 +); + +INSERT INTO simple_primary_key VALUES (1, 'hello'); +INSERT INTO simple_primary_key VALUES (2, 'world'); +INSERT INTO simple_primary_key VALUES (3, ''); +INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO'); +INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC'); + +INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); +INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); + +INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b'); +INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null); + +INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); +INSERT INTO custom_foreign_key_label VALUES (1, 1); + +INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey'); + +CREATE VIEW simple_view AS + SELECT content, upper(content) AS upper_content FROM simple_primary_key; + +CREATE VIEW searchable_view AS + SELECT * from searchable; + +CREATE VIEW searchable_view_configured_by_metadata AS + SELECT * from searchable; + +""" + + "\n".join( + [ + 'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format( + i=i + 1 + ) + for i in range(201) + ] + ) + + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' + + "\n".join( + [ + 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( + a=a, b=b, c=c, content=content + ) + for a, b, c, content in generate_compound_rows(1001) + ] + ) + + "\n".join(["""INSERT INTO sortable VALUES ( + "{pk1}", "{pk2}", "{content}", {sortable}, + {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}"); + """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)]) +) + +TABLE_PARAMETERIZED_SQL = [ + ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]), + ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]), + ("insert into binary_data (data) values (null);", []), +] + +EXTRA_DATABASE_SQL = """ +CREATE TABLE searchable ( + pk integer primary key, + text1 text, + text2 text +); + +CREATE VIEW searchable_view AS SELECT * FROM searchable; + +INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog'); +INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel'); + +CREATE VIRTUAL TABLE "searchable_fts" + USING FTS3 (text1, text2, content="searchable"); +INSERT INTO "searchable_fts" (rowid, text1, text2) + SELECT rowid, text1, text2 FROM searchable; +""" + + +@documented(label="datasette_fixtures_populate_fixture_database") +def populate_fixture_database(conn): + """Populate a SQLite connection with Datasette's test fixture tables.""" + conn.executescript(TABLES) + for sql, params in TABLE_PARAMETERIZED_SQL: + with conn: + conn.execute(sql, params) + + +def populate_extra_database(conn): + """Populate a SQLite connection with the extra database used in tests.""" + conn.executescript(EXTRA_DATABASE_SQL) + + +def write_fixture_database(db_filename): + """Write Datasette's test fixture tables to a SQLite database file.""" + conn = sqlite3.connect(db_filename) + try: + populate_fixture_database(conn) + finally: + conn.close() + + +def write_extra_database(db_filename): + """Write the extra test database tables to a SQLite database file.""" + conn = sqlite3.connect(db_filename) + try: + populate_extra_database(conn) + finally: + conn.close() diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 1fea992e..9d189459 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -155,9 +155,15 @@ Column = namedtuple( functions_marked_as_documented = [] -def documented(fn): - functions_marked_as_documented.append(fn) - return fn +def documented(fn=None, *, label=None): + def decorate(fn): + fn._datasette_docs_label = label or "internals_utils_{}".format(fn.__name__) + functions_marked_as_documented.append(fn) + return fn + + if fn is None: + return decorate + return decorate(fn) @documented diff --git a/docs/changelog.rst b/docs/changelog.rst index f2ba23ec..51188461 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Unreleased - Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) - Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`) - New "Jump to..." menu item, always visible, for triggering the previously undocumented ``/`` menu. (:issue:`2725`) +- New documented :ref:`datasette.fixtures.populate_fixture_database(conn) <datasette_fixtures_populate_fixture_database>` helper for creating the fixture database tables used by Datasette's own tests, intended for plugin test suites. .. _v1_0_a29: diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index b82a6e0c..15891963 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -82,6 +82,48 @@ This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepar If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request. +.. _testing_plugins_datasette_fixtures_database: + +Using Datasette's fixtures database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Datasette's own test suite uses a SQLite database with tables that exercise features such as compound primary keys, foreign keys, sortable columns, facets, full-text search and binary data. + +You can use those same tables in your plugin tests using the ``populate_fixture_database(conn)`` helper in ``datasette.fixtures``: + +Be aware that future Datasette releases may change details of these tables, so try not to rely on their exact structure in your own tests. + +.. _datasette_fixtures_populate_fixture_database: + +``populate_fixture_database(conn)`` + Populates an existing SQLite connection with the fixture tables. + +For an in-memory test database: + +.. code-block:: python + + from datasette.app import Datasette + from datasette.fixtures import populate_fixture_database + import pytest + import pytest_asyncio + + + @pytest_asyncio.fixture + async def datasette(): + datasette = Datasette() + db = datasette.add_memory_database("fixtures") + await db.execute_write_fn(populate_fixture_database) + await datasette.invoke_startup() + return datasette + + + @pytest.mark.asyncio + async def test_facetable(datasette): + response = await datasette.client.get( + "/fixtures/facetable.json?_shape=array" + ) + assert response.status_code == 200 + .. _testing_plugins_autoclose: Automatic cleanup of Datasette instances diff --git a/tests/conftest.py b/tests/conftest.py index 4ea89458..b9b3c35e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -72,7 +72,7 @@ async def ds_client(): "num_sql_threads": 1, }, ) - from .fixtures import TABLES, TABLE_PARAMETERIZED_SQL + from datasette.fixtures import populate_fixture_database # Use a unique memory_name to avoid collisions between different # Datasette instances in the same process, but use "fixtures" for routing @@ -82,10 +82,7 @@ async def ds_client(): def prepare(conn): if not conn.execute("select count(*) from sqlite_master").fetchone()[0]: - conn.executescript(TABLES) - for sql, params in TABLE_PARAMETERIZED_SQL: - with conn: - conn.execute(sql, params) + populate_fixture_database(conn) await db.execute_write_fn(prepare) await ds.invoke_startup() @@ -264,8 +261,6 @@ from .fixtures import ( # noqa: E402, F401 app_client_with_cors, app_client_with_dot, app_client_with_trace, - generate_compound_rows, - generate_sortable_rows, make_app_client, TEMP_PLUGIN_SECRET_FILE, ) diff --git a/tests/fixtures.py b/tests/fixtures.py index f61ec0c7..71884294 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,15 +1,16 @@ from datasette.app import Datasette -from datasette.utils.sqlite import sqlite3 +from datasette.fixtures import ( + EXTRA_DATABASE_SQL, + write_extra_database, + write_fixture_database, +) from datasette.utils.testing import TestClient import click import contextlib -import itertools import json import os import pathlib import pytest -import random -import string import tempfile import textwrap @@ -128,19 +129,18 @@ def make_app_client( else: files = [filepath] immutables = [] - conn = sqlite3.connect(filepath) - conn.executescript(TABLES) - for sql, params in TABLE_PARAMETERIZED_SQL: - with conn: - conn.execute(sql, params) - # Close the connection to avoid "too many open files" errors - conn.close() + write_fixture_database(filepath) if extra_databases is not None: for extra_filename, extra_sql in extra_databases.items(): extra_filepath = os.path.join(tmpdir, extra_filename) - c2 = sqlite3.connect(extra_filepath) - c2.executescript(extra_sql) - c2.close() + if extra_sql == EXTRA_DATABASE_SQL: + write_extra_database(extra_filepath) + else: + from datasette.utils.sqlite import sqlite3 + + c2 = sqlite3.connect(extra_filepath) + c2.executescript(extra_sql) + c2.close() # Insert at start to help test /-/databases ordering: files.insert(0, extra_filepath) os.chdir(os.path.dirname(filepath)) @@ -279,29 +279,6 @@ def app_client_immutable_and_inspect_file(): yield client -def generate_compound_rows(num): - for a, b, c in itertools.islice( - itertools.product(string.ascii_lowercase, repeat=3), num - ): - yield a, b, c, f"{a}-{b}-{c}" - - -def generate_sortable_rows(num): - rand = random.Random(42) - for a, b in itertools.islice( - itertools.product(string.ascii_lowercase, repeat=2), num - ): - yield { - "pk1": a, - "pk2": b, - "content": f"{a}-{b}", - "sortable": rand.randint(-100, 100), - "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]), - "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]), - "text": rand.choice(["$null", "$blah"]), - } - - CONFIG = { "plugins": { "name-of-plugin": {"depth": "root"}, @@ -399,345 +376,6 @@ METADATA = { }, } -TABLES = ( - """ -CREATE TABLE simple_primary_key ( - id integer primary key, - content text -); - -CREATE TABLE primary_key_multiple_columns ( - id varchar(30) primary key, - content text, - content2 text -); - -CREATE TABLE primary_key_multiple_columns_explicit_label ( - id varchar(30) primary key, - content text, - content2 text -); - -CREATE TABLE compound_primary_key ( - pk1 varchar(30), - pk2 varchar(30), - content text, - PRIMARY KEY (pk1, pk2) -); - -INSERT INTO compound_primary_key VALUES ('a', 'b', 'c'); -INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c'); -INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO'); - -CREATE TABLE compound_three_primary_keys ( - pk1 varchar(30), - pk2 varchar(30), - pk3 varchar(30), - content text, - PRIMARY KEY (pk1, pk2, pk3) -); -CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content); - -CREATE TABLE foreign_key_references ( - pk varchar(30) primary key, - foreign_key_with_label integer, - foreign_key_with_blank_label integer, - foreign_key_with_no_label varchar(30), - foreign_key_compound_pk1 varchar(30), - foreign_key_compound_pk2 varchar(30), - FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id), - FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id), - FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id) - FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2) -); - -CREATE TABLE sortable ( - pk1 varchar(30), - pk2 varchar(30), - content text, - sortable integer, - sortable_with_nulls real, - sortable_with_nulls_2 real, - text text, - PRIMARY KEY (pk1, pk2) -); - -CREATE TABLE no_primary_key ( - content text, - a text, - b text, - c text -); - -CREATE TABLE [123_starts_with_digits] ( - content text -); - -CREATE VIEW paginated_view AS - SELECT - content, - '- ' || content || ' -' AS content_extra - FROM no_primary_key; - -CREATE TABLE "Table With Space In Name" ( - pk varchar(30) primary key, - content text -); - -CREATE TABLE "table/with/slashes.csv" ( - pk varchar(30) primary key, - content text -); - -CREATE TABLE "complex_foreign_keys" ( - pk varchar(30) primary key, - f1 integer, - f2 integer, - f3 integer, - FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id), - FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id), - FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id) -); - -CREATE TABLE "custom_foreign_key_label" ( - pk varchar(30) primary key, - foreign_key_with_custom_label text, - FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id) -); - -CREATE TABLE tags ( - tag TEXT PRIMARY KEY -); - -CREATE TABLE searchable ( - pk integer primary key, - text1 text, - text2 text, - [name with . and spaces] text -); - -CREATE TABLE searchable_tags ( - searchable_id integer, - tag text, - PRIMARY KEY (searchable_id, tag), - FOREIGN KEY (searchable_id) REFERENCES searchable(pk), - FOREIGN KEY (tag) REFERENCES tags(tag) -); - -INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther'); -INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma'); - -INSERT INTO tags VALUES ("canine"); -INSERT INTO tags VALUES ("feline"); - -INSERT INTO searchable_tags (searchable_id, tag) VALUES - (1, "feline"), - (2, "canine") -; - -CREATE VIRTUAL TABLE "searchable_fts" - USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk"); -INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild'); - -CREATE TABLE [select] ( - [group] text, - [having] text, - [and] text, - [json] text -); -INSERT INTO [select] VALUES ('group', 'having', 'and', - '{"href": "http://example.com/", "label":"Example"}' -); - -CREATE TABLE infinity ( - value REAL -); -INSERT INTO infinity VALUES - (1e999), - (-1e999), - (1.5) -; - -CREATE TABLE facet_cities ( - id integer primary key, - name text -); -INSERT INTO facet_cities (id, name) VALUES - (1, 'San Francisco'), - (2, 'Los Angeles'), - (3, 'Detroit'), - (4, 'Memnonia') -; - -CREATE TABLE facetable ( - pk integer primary key, - created text, - planet_int integer, - on_earth integer, - state text, - _city_id integer, - _neighborhood text, - tags text, - complex_array text, - distinct_some_null, - n text, - FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id) -); -INSERT INTO facetable - (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n) -VALUES - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null), - ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null), - ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null), - ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null), - ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null) -; - -CREATE TABLE binary_data ( - data BLOB -); - --- Many 2 Many demo: roadside attractions! - -CREATE TABLE roadside_attractions ( - pk integer primary key, - name text, - address text, - url text, - latitude real, - longitude real -); -INSERT INTO roadside_attractions VALUES ( - 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", - 37.0167, -122.0024 -); -INSERT INTO roadside_attractions VALUES ( - 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", - 37.3184, -121.9511 -); -INSERT INTO roadside_attractions VALUES ( - 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, - 37.5793, -122.3442 -); -INSERT INTO roadside_attractions VALUES ( - 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", - 37.0414, -122.0725 -); - -CREATE TABLE attraction_characteristic ( - pk integer primary key, - name text -); -INSERT INTO attraction_characteristic VALUES ( - 1, "Museum" -); -INSERT INTO attraction_characteristic VALUES ( - 2, "Paranormal" -); - -CREATE TABLE roadside_attraction_characteristics ( - attraction_id INTEGER REFERENCES roadside_attractions(pk), - characteristic_id INTEGER REFERENCES attraction_characteristic(pk) -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 1, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 2, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 4, 2 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 3, 1 -); -INSERT INTO roadside_attraction_characteristics VALUES ( - 4, 1 -); - -INSERT INTO simple_primary_key VALUES (1, 'hello'); -INSERT INTO simple_primary_key VALUES (2, 'world'); -INSERT INTO simple_primary_key VALUES (3, ''); -INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO'); -INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC'); - -INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world'); -INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2'); - -INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b'); -INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null); - -INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1); -INSERT INTO custom_foreign_key_label VALUES (1, 1); - -INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey'); - -CREATE VIEW simple_view AS - SELECT content, upper(content) AS upper_content FROM simple_primary_key; - -CREATE VIEW searchable_view AS - SELECT * from searchable; - -CREATE VIEW searchable_view_configured_by_metadata AS - SELECT * from searchable; - -""" - + "\n".join( - [ - 'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format( - i=i + 1 - ) - for i in range(201) - ] - ) - + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n' - + "\n".join( - [ - 'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format( - a=a, b=b, c=c, content=content - ) - for a, b, c, content in generate_compound_rows(1001) - ] - ) - + "\n".join(["""INSERT INTO sortable VALUES ( - "{pk1}", "{pk2}", "{content}", {sortable}, - {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}"); - """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)]) -) -TABLE_PARAMETERIZED_SQL = [ - ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]), - ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]), - ("insert into binary_data (data) values (null);", []), -] - -EXTRA_DATABASE_SQL = """ -CREATE TABLE searchable ( - pk integer primary key, - text1 text, - text2 text -); - -CREATE VIEW searchable_view AS SELECT * FROM searchable; - -INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog'); -INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel'); - -CREATE VIRTUAL TABLE "searchable_fts" - USING FTS3 (text1, text2, content="searchable"); -INSERT INTO "searchable_fts" (rowid, text1, text2) - SELECT rowid, text1, text2 FROM searchable; -""" - def assert_permissions_checked(datasette, actions): # actions is a list of "action" or (action, resource) tuples @@ -819,12 +457,7 @@ def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename ) else: pathlib.Path(db_filename).unlink() - conn = sqlite3.connect(db_filename) - conn.executescript(TABLES) - for sql, params in TABLE_PARAMETERIZED_SQL: - with conn: - conn.execute(sql, params) - conn.close() + write_fixture_database(db_filename) print(f"Test tables written to {db_filename}") if metadata: with open(metadata, "w") as fp: @@ -851,9 +484,7 @@ def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename ) else: pathlib.Path(extra_db_filename).unlink() - conn = sqlite3.connect(extra_db_filename) - conn.executescript(EXTRA_DATABASE_SQL) - conn.close() + write_extra_database(extra_db_filename) print(f"Test tables written to {extra_db_filename}") diff --git a/tests/test_docs.py b/tests/test_docs.py index b94a6f23..396ba1a2 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -3,6 +3,7 @@ Tests to ensure certain things are documented. """ from datasette import app, utils +import datasette.fixtures # noqa: F401 from datasette.app import Datasette from datasette.filters import Filters from pathlib import Path @@ -95,20 +96,17 @@ def test_table_filters_are_documented(documented_table_filters, subtests): @pytest.fixture(scope="session") -def documented_fns(): - internals_rst = (docs_path / "internals.rst").read_text() - # Any line that starts .. _internals_utils_X - lines = internals_rst.split("\n") - prefix = ".. _internals_utils_" - return { - line.split(prefix)[1].split(":")[0] for line in lines if line.startswith(prefix) - } +def documented_labels(): + labels = set() + for filename in docs_path.glob("*.rst"): + labels.update(get_labels(filename.name)) + return labels -def test_functions_marked_with_documented_are_documented(documented_fns, subtests): +def test_functions_marked_with_documented_are_documented(documented_labels, subtests): for fn in utils.functions_marked_as_documented: with subtests.test(fn=fn.__name__): - assert fn.__name__ in documented_fns + assert fn._datasette_docs_label in documented_labels def test_rst_heading_underlines_match_title_length(): diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py new file mode 100644 index 00000000..45f9854b --- /dev/null +++ b/tests/test_fixtures.py @@ -0,0 +1,49 @@ +from datasette.fixtures import ( + populate_extra_database, + populate_fixture_database, + write_extra_database, + write_fixture_database, +) +from datasette.utils.sqlite import sqlite3 + + +def count(conn, table): + return conn.execute(f"select count(*) from [{table}]").fetchone()[0] + + +def test_populate_fixture_database(): + conn = sqlite3.connect(":memory:") + try: + populate_fixture_database(conn) + assert count(conn, "facetable") == 15 + assert count(conn, "compound_three_primary_keys") == 1001 + assert count(conn, "binary_data") == 3 + finally: + conn.close() + + +def test_write_fixture_database(tmp_path): + db_path = tmp_path / "fixtures.db" + write_fixture_database(db_path) + conn = sqlite3.connect(db_path) + try: + assert count(conn, "sortable") == 201 + finally: + conn.close() + + +def test_extra_database_helpers(tmp_path): + conn = sqlite3.connect(":memory:") + try: + populate_extra_database(conn) + assert count(conn, "searchable") == 2 + finally: + conn.close() + + db_path = tmp_path / "extra.db" + write_extra_database(db_path) + conn = sqlite3.connect(db_path) + try: + assert count(conn, "searchable") == 2 + finally: + conn.close() diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 7ebd57f3..d87f577a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,7 +1,6 @@ from bs4 import BeautifulSoup as Soup from .fixtures import ( make_app_client, - TABLES, TEMP_PLUGIN_SECRET_FILE, PLUGINS_DIR, TestClient as _TestClient, @@ -9,6 +8,7 @@ from .fixtures import ( from click.testing import CliRunner from datasette.app import Datasette from datasette import cli, hookimpl +from datasette.fixtures import TABLES from datasette.filters import FilterArguments from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.permissions import PermissionSQL, Action diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 51e40ad1..ceeb646d 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1,6 +1,7 @@ from datasette.utils import detect_json1 from datasette.utils.sqlite import sqlite_version -from .fixtures import generate_compound_rows, generate_sortable_rows, make_app_client +from datasette.fixtures import generate_compound_rows, generate_sortable_rows +from .fixtures import make_app_client import json import pytest import urllib From 6057c76165d9dd3d2166824b12665814d07785de Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Thu, 21 May 2026 23:28:35 -0700 Subject: [PATCH 064/156] Initial docs for jump_items_sql and jump_start hooks Refs #2731 --- docs/plugin_hooks.rst | 114 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 54dde20c..041f3d9d 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1881,6 +1881,120 @@ Using :ref:`internals_datasette_urls` here ensures that links in the menu will t Examples: `datasette-search-all <https://datasette.io/plugins/datasette-search-all>`_, `datasette-graphql <https://datasette.io/plugins/datasette-graphql>`_ +.. _plugin_hook_jump_items_sql: + +jump_items_sql(datasette, actor, request) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor <authentication_actor>`. + +``request`` - :ref:`internals_request` or None + The current HTTP request. This can be ``None`` if the request object is not available. + +This hook allows plugins to add extra results to Datasette's ``/`` jump menu, which is powered by the ``/-/jump`` JSON endpoint. + +Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be combined with Datasette's own databases, tables, views and canned query results. + +The SQL query must return these columns: + +``type`` + A short type string for the result, for example ``"app"`` or ``"dashboard"``. + +``label`` + The stable name for the result. This is returned as ``name`` in the JSON API and is used for sorting. + +``description`` + A short description shown above the item in the jump menu. + +``url`` + The URL to navigate to when the item is selected. + +``database_name`` + The database name for Datasette resources, or ``NULL`` for custom plugin results. + +``resource_name`` + The table, view or query name for Datasette resources, or ``NULL`` for custom plugin results. + +``search_text`` + Text that should be searched by the ``?q=`` parameter. + +``sort_key`` + A numeric value used to order results. + +``source`` + A string identifying the plugin that supplied the result. + +If the SQL query also returns a ``display_name`` column, set ``has_display_name=True`` on the ``JumpSQL`` object. Datasette will return that value as ``display_name`` in the JSON API, and the jump menu will show it as the primary readable label with ``name`` shown underneath. + +This example adds a "Plugin dashboard" result for signed-in users: + +.. code-block:: python + + from datasette import hookimpl + from datasette.jump import JumpSQL + + + @hookimpl + def jump_items_sql(actor): + if not actor: + return None + return JumpSQL( + sql=""" + SELECT + 'dashboard' AS type, + 'plugin-dashboard' AS label, + 'Dashboard' AS description, + '/-/plugin-dashboard' AS url, + NULL AS database_name, + NULL AS resource_name, + 'plugin dashboard' AS search_text, + 80 AS sort_key, + 'my-plugin' AS source, + 'Plugin dashboard' AS display_name + """, + has_display_name=True, + ) + +Use ``params=`` to pass SQL parameters. Datasette will automatically namespace those parameters before combining SQL fragments from different plugins. + +.. _plugin_hook_jump_start: + +jump_start(datasette, actor, request) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +``actor`` - dictionary or None + The currently authenticated :ref:`actor <authentication_actor>`. + +``request`` - :ref:`internals_request` or None + The current HTTP request. This can be ``None`` if the request object is not available. + +This hook allows plugins to add custom HTML to the default blank state of Datasette's ``/`` jump menu, before the user starts typing a search. + +The hook can return a string, a ``markupsafe.Markup`` object, or an awaitable function that returns either of those. Return ``None`` to add nothing. + +This example shows a link for starting a new chat if the user is signed in: + +.. code-block:: python + + from datasette import hookimpl + from markupsafe import Markup + + + @hookimpl + def jump_start(actor): + if not actor: + return None + return Markup( + '<p><a href="/-/agent/new">Start a new chat</a></p>' + ) + .. _plugin_actions: Action hooks From 8568320a23b492392646e326514e4edad83391d9 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 22 May 2026 20:48:42 -0700 Subject: [PATCH 065/156] Replace jump_start() hook with JavaScript makeJumpSections() hook --- datasette/app.py | 17 --- datasette/hookspecs.py | 7 +- datasette/jump.py | 1 - datasette/static/datasette-manager.js | 14 ++- datasette/static/navigation-search.js | 51 +++++++- datasette/templates/base.html | 9 +- datasette/views/special.py | 5 +- docs/changelog.rst | 2 +- docs/introspection.rst | 2 +- docs/javascript_plugins.rst | 42 +++++++ docs/plugin_hooks.rst | 40 +----- tests/test_jump.py | 30 ----- tests/test_navigation_search_js.py | 175 ++++++++++++++++++++++++++ tests/test_plugins.py | 5 + 14 files changed, 288 insertions(+), 112 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index c9605af3..088403e0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2039,22 +2039,6 @@ class Datasette: links.extend(extra_links) return links - async def jump_start(): - html_bits = [] - for hook in pm.hook.jump_start( - datasette=self, - actor=request.actor if request else None, - request=request or None, - ): - extra_html = await await_me_maybe(hook) - if not extra_html: - continue - if isinstance(extra_html, (list, tuple)): - html_bits.extend(extra_html) - else: - html_bits.append(extra_html) - return Markup("").join(Markup(html) for html in html_bits) - template_context = { **context, **{ @@ -2063,7 +2047,6 @@ class Datasette: "urls": self.urls, "actor": request.actor if request else None, "menu_links": menu_links, - "jump_start": jump_start, "display_actor": display_actor, "show_logout": request is not None and "ds_actor" in request.cookies diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index a4d8143b..cf95abcb 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -159,12 +159,7 @@ def menu_links(datasette, actor, request): @hookspec def jump_items_sql(datasette, actor, request): - """SQL fragments for extra items in the jump menu, optionally with display_name""" - - -@hookspec -def jump_start(datasette, actor, request): - """HTML to display in the jump menu before the user types""" + """SQL fragments for extra items in the jump menu""" @hookspec diff --git a/datasette/jump.py b/datasette/jump.py index 96e18547..6ec7ae13 100644 --- a/datasette/jump.py +++ b/datasette/jump.py @@ -9,7 +9,6 @@ from typing import Any class JumpSQL: sql: str params: dict[str, Any] | None = None - has_display_name: bool = False _PARAM_RE = re.compile(r"(?<!:):([A-Za-z_][A-Za-z0-9_]*)") diff --git a/datasette/static/datasette-manager.js b/datasette/static/datasette-manager.js index d2347ab3..e75f7aae 100644 --- a/datasette/static/datasette-manager.js +++ b/datasette/static/datasette-manager.js @@ -82,6 +82,19 @@ const datasetteManager = { return columnActions; }, + makeJumpSections: (context) => { + let jumpSections = []; + + datasetteManager.plugins.forEach((plugin) => { + if (plugin.makeJumpSections) { + const sections = plugin.makeJumpSections(context) || []; + jumpSections.push(...sections); + } + }); + + return jumpSections; + }, + /** * In MVP, each plugin can only have 1 instance. * In future, panels could be repeated. We omit that for now since so many plugins depend on @@ -192,7 +205,6 @@ const initializeDatasette = () => { // DATASETTE_EVENTS.INIT event to avoid the habit of reading from the window. window.__DATASETTE__ = datasetteManager; - console.debug("Datasette Manager Created!"); const initDatasetteEvent = new CustomEvent(DATASETTE_EVENTS.INIT, { detail: datasetteManager, diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index 49cc172c..9e24681b 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -447,9 +447,46 @@ class NavigationSearch extends HTMLElement { this.renderResults(); } - startContentHtml() { - const template = this.querySelector("template[data-jump-start]"); - return template ? template.innerHTML.trim() : ""; + jumpSections() { + const manager = window.__DATASETTE__; + if (!manager || typeof manager.makeJumpSections !== "function") { + return []; + } + const sections = manager.makeJumpSections({ + navigationSearch: this, + }); + return Array.isArray(sections) + ? sections.filter( + (section) => section && typeof section.render === "function", + ) + : []; + } + + jumpSectionsHtml(jumpSections) { + return jumpSections + .map((section, index) => { + const id = section.id + ? ` data-jump-section-id="${this.escapeHtml(section.id)}"` + : ""; + return `<div class="jump-start-content" data-jump-section-index="${index}"${id}></div>`; + }) + .join(""); + } + + renderJumpSections(container, jumpSections) { + jumpSections.forEach((section, index) => { + const node = container.querySelector( + `[data-jump-section-index="${index}"]`, + ); + if (!node) { + return; + } + section.render(node, { + navigationSearch: this, + container, + input: this.shadowRoot.querySelector(".search-input"), + }); + }); } resultItemHtml(match, index) { @@ -484,9 +521,9 @@ class NavigationSearch extends HTMLElement { const container = this.shadowRoot.querySelector(".results-container"); const input = this.shadowRoot.querySelector(".search-input"); const showStartContent = !input.value.trim(); - const startContent = showStartContent ? this.startContentHtml() : ""; - const startBlock = startContent - ? `<div class="jump-start-content">${startContent}</div>` + const jumpSections = showStartContent ? this.jumpSections() : []; + const startBlock = showStartContent + ? this.jumpSectionsHtml(jumpSections) : ""; const recentItems = showStartContent ? this.loadRecentItems() : []; const defaultMatches = showStartContent ? [] : this.matches; @@ -507,6 +544,7 @@ class NavigationSearch extends HTMLElement { if (renderedMatches.length === 0) { if (startBlock) { container.innerHTML = startBlock; + this.renderJumpSections(container, jumpSections); } else if (showStartContent) { container.innerHTML = ""; } else { @@ -529,6 +567,7 @@ class NavigationSearch extends HTMLElement { ) .join(""); container.innerHTML = startBlock + recentHtml + defaultHtml; + this.renderJumpSections(container, jumpSections); // Scroll selected item into view if (this.selectedIndex >= 0) { diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 6a55bd1f..02365e68 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -71,13 +71,6 @@ {% if select_templates %}<!-- Templates considered: {{ select_templates|join(", ") }} -->{% endif %} <script src="{{ urls.static('navigation-search.js') }}{% if navigation_search_js_hash is defined %}?{{ navigation_search_js_hash }}{% endif %}" defer></script> -{% if jump_start is defined %} -{% set jump_start_html = jump_start() %} -{% else %} -{% set jump_start_html = "" %} -{% endif %} -<navigation-search url="/-/jump">{% if jump_start_html %} - <template data-jump-start>{{ jump_start_html }}</template> -{% endif %}</navigation-search> +<navigation-search url="/-/jump"></navigation-search> </body> </html> diff --git a/datasette/views/special.py b/datasette/views/special.py index 990714cf..cb6524dc 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -980,7 +980,6 @@ class JumpView(BaseView): FROM allowed_databases """, params=database_params, - has_display_name=True, ), JumpSQL( sql=f""" @@ -1004,7 +1003,6 @@ class JumpView(BaseView): AND catalog_views.view_name = allowed_tables.child """, params=table_params, - has_display_name=True, ), JumpSQL( sql=f""" @@ -1031,7 +1029,6 @@ class JumpView(BaseView): AND query_display_names.query_name = allowed_queries.child """, params={**query_params, **query_display_names_params}, - has_display_name=True, ), ] @@ -1095,7 +1092,7 @@ class JumpView(BaseView): search_text, sort_key, source, - {"display_name" if fragment.has_display_name else "NULL AS display_name"} + display_name FROM ( {fragment_sql} ) diff --git a/docs/changelog.rst b/docs/changelog.rst index 51188461..5d52d04d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -274,7 +274,7 @@ Other changes ~~~~~~~~~~~~~ - The internal ``catalog_views`` table now tracks SQLite views alongside tables in the introspection database. (:issue:`2495`) -- Hitting the ``/`` brings up a search interface for navigating to tables that the current user can view. A new ``/-/tables`` endpoint supports this functionality. (:issue:`2523`) +- Hitting the ``/`` brings up a search interface for navigating to databases, tables, views, canned queries and plugin-provided items that the current user can view. A new ``/-/jump`` endpoint supports this functionality, and JavaScript plugins can add custom blank-state sections using ``makeJumpSections()``. (:issue:`2523`) - Datasette attempts to detect some configuration errors on startup. - Datasette now supports Python 3.14 and no longer tests against Python 3.9. diff --git a/docs/introspection.rst b/docs/introspection.rst index 9f0358ac..a5a9753a 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -155,7 +155,7 @@ The endpoint supports a ``?q=`` query parameter for filtering items by name. Canned queries with a configured ``title`` also include a ``display_name`` in their results, and can be found by searching for that title. Plugins can provide the same extra field from ``jump_items_sql`` by returning a ``display_name`` -column and setting ``JumpSQL(..., has_display_name=True)``. +column. `Jump example <https://latest.datasette.io/-/jump>`_: diff --git a/docs/javascript_plugins.rst b/docs/javascript_plugins.rst index e7ee6817..805938c0 100644 --- a/docs/javascript_plugins.rst +++ b/docs/javascript_plugins.rst @@ -58,6 +58,48 @@ JavaScript plugins are blocks of code that can be registered with Datasette usin The ``implementation`` object passed to this method should include a ``version`` key defining the plugin version, and one or more of the following named functions providing the implementation of the plugin: +.. _javascript_plugins_makeJumpSections: + +makeJumpSections() +~~~~~~~~~~~~~~~~~~ + +This method should return a JavaScript array of objects defining additional sections to be added to the blank state of the ``/`` jump menu, before the user starts typing a search. + +Each object should have the following: + +``id`` - string + A unique string ID for the section, for example ``agent-chat`` +``render(node, context)`` - function + A function that will be called with a DOM node to render the section into + +The ``context`` object has the following keys: + +``navigationSearch`` + The ``<navigation-search>`` custom element instance. + +This example shows how a plugin might add a button for starting a new chat: + +.. code-block:: javascript + + document.addEventListener('datasette_init', function(ev) { + ev.detail.registerPlugin('agent-plugin', { + version: 0.1, + makeJumpSections: () => { + return [ + { + id: 'agent-chat', + render: node => { + node.innerHTML = '<button type="button">Start a new chat</button>'; + node.querySelector('button').addEventListener('click', () => { + location.href = '/-/agent/new'; + }); + } + } + ]; + } + }); + }); + .. _javascript_plugins_makeAboveTablePanelConfigs: makeAboveTablePanelConfigs() diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 041f3d9d..c3ce3ed9 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1928,7 +1928,8 @@ The SQL query must return these columns: ``source`` A string identifying the plugin that supplied the result. -If the SQL query also returns a ``display_name`` column, set ``has_display_name=True`` on the ``JumpSQL`` object. Datasette will return that value as ``display_name`` in the JSON API, and the jump menu will show it as the primary readable label with ``name`` shown underneath. +``display_name`` + A human-readable label for the result, or ``NULL``. Datasette returns this as ``display_name`` in the JSON API, and the jump menu shows it as the primary readable label with ``name`` shown underneath. This example adds a "Plugin dashboard" result for signed-in users: @@ -1955,46 +1956,11 @@ This example adds a "Plugin dashboard" result for signed-in users: 80 AS sort_key, 'my-plugin' AS source, 'Plugin dashboard' AS display_name - """, - has_display_name=True, + """ ) Use ``params=`` to pass SQL parameters. Datasette will automatically namespace those parameters before combining SQL fragments from different plugins. -.. _plugin_hook_jump_start: - -jump_start(datasette, actor, request) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``datasette`` - :ref:`internals_datasette` - You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. - -``actor`` - dictionary or None - The currently authenticated :ref:`actor <authentication_actor>`. - -``request`` - :ref:`internals_request` or None - The current HTTP request. This can be ``None`` if the request object is not available. - -This hook allows plugins to add custom HTML to the default blank state of Datasette's ``/`` jump menu, before the user starts typing a search. - -The hook can return a string, a ``markupsafe.Markup`` object, or an awaitable function that returns either of those. Return ``None`` to add nothing. - -This example shows a link for starting a new chat if the user is signed in: - -.. code-block:: python - - from datasette import hookimpl - from markupsafe import Markup - - - @hookimpl - def jump_start(actor): - if not actor: - return None - return Markup( - '<p><a href="/-/agent/new">Start a new chat</a></p>' - ) - .. _plugin_actions: Action hooks diff --git a/tests/test_jump.py b/tests/test_jump.py index 55325b95..acffe4a9 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -1,6 +1,5 @@ import pytest import pytest_asyncio -from markupsafe import Markup from datasette import hookimpl from datasette.app import Datasette @@ -163,7 +162,6 @@ async def test_jump_uses_plugin_sql_with_namespaced_parameters(ds_for_jump): 'Plugin dashboard for ' || :actor_id AS display_name """, params={"actor_id": actor["id"] if actor else "anonymous"}, - has_display_name=True, ) plugin = JumpPlugin() @@ -190,34 +188,6 @@ async def test_jump_uses_plugin_sql_with_namespaced_parameters(ds_for_jump): ] -@pytest.mark.asyncio -async def test_jump_start_hook_renders_empty_state_template(ds_for_jump): - class JumpStartPlugin: - @hookimpl - def jump_start(self, datasette, actor, request): - if not actor: - return None - return Markup( - '<section class="agent-jump-start">' - "<h3>Agent chat</h3>" - '<a href="/-/agent/new">Start a new agent chat</a>' - "</section>" - ) - - plugin = JumpStartPlugin() - pm.register(plugin, name="test-jump-start-plugin") - try: - anonymous = await ds_for_jump.client.get("/") - authenticated = await ds_for_jump.client.get("/", actor={"id": "alice"}) - finally: - pm.unregister(name="test-jump-start-plugin") - - assert 'url="/-/jump"' in authenticated.text - assert "<template data-jump-start>" not in anonymous.text - assert "<template data-jump-start>" in authenticated.text - assert "Start a new agent chat" in authenticated.text - - @pytest.mark.asyncio async def test_tables_endpoint_removed(ds_for_jump): response = await ds_for_jump.client.get("/-/tables.json") diff --git a/tests/test_navigation_search_js.py b/tests/test_navigation_search_js.py index 3096d9a0..590985c9 100644 --- a/tests/test_navigation_search_js.py +++ b/tests/test_navigation_search_js.py @@ -197,3 +197,178 @@ def test_navigation_search_tracks_and_renders_recent_items(): "/item-2", ] assert json.loads(result.stdout)[0]["display_name"] == "Recent Datasette releases" + + +def test_navigation_search_renders_jump_sections_from_javascript_plugins(): + script = textwrap.dedent(""" + const fs = require("fs"); + const vm = require("vm"); + + const documentListeners = {}; + + class FakeElement { + constructor(tagName = "div", parent = null) { + this._innerHTML = ""; + this.value = ""; + this.dataset = {}; + this.open = false; + this.parent = parent; + this.tagName = tagName.toUpperCase(); + } + set textContent(value) { + this.innerHTML = String(value) + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, """); + } + get innerHTML() { + return this._innerHTML; + } + set innerHTML(value) { + this._innerHTML = String(value); + if (this.parent) { + this.parent._innerHTML += this._innerHTML; + } + } + addEventListener() {} + appendChild(child) { + this._innerHTML += child.innerHTML || ""; + return child; + } + close() { this.open = false; } + focus() {} + querySelector(selector) { + if (selector.startsWith("[data-jump-section-index=")) { + return new FakeElement("div", this); + } + return { scrollIntoView() {} }; + } + showModal() { this.open = true; } + } + + class FakeShadowRoot { + constructor() { + this.innerHTML = ""; + this.dialog = new FakeElement("dialog"); + this.input = new FakeElement("input"); + this.results = new FakeElement("div"); + } + querySelector(selector) { + if (selector == "dialog") return this.dialog; + if (selector == ".search-input") return this.input; + if (selector == ".results-container") return this.results; + return new FakeElement(); + } + } + + global.HTMLElement = class { + constructor() { + this.attributes = {}; + } + attachShadow() { + this.shadowRoot = new FakeShadowRoot(); + return this.shadowRoot; + } + dispatchEvent() {} + getAttribute(name) { + return this.attributes[name] || null; + } + querySelector() { + return null; + } + setAttribute(name, value) { + this.attributes[name] = value; + } + }; + global.CustomEvent = class { + constructor(name, options) { + this.name = name; + this.type = name; + this.detail = options ? options.detail : undefined; + } + }; + global.customElements = { + registry: new Map(), + define(name, cls) { + this.registry.set(name, cls); + }, + }; + global.document = { + addEventListener(name, callback) { + documentListeners[name] = documentListeners[name] || []; + documentListeners[name].push(callback); + }, + activeElement: null, + createElement(tagName) { + return new FakeElement(tagName); + }, + dispatchEvent(event) { + for (const callback of documentListeners[event.type] || []) { + callback(event); + } + }, + querySelectorAll() { + return []; + }, + }; + global.localStorage = { + getItem() { return null; }, + setItem() {}, + removeItem() {}, + }; + global.window = { datasetteVersion: "test", location: { href: "" } }; + + vm.runInThisContext( + fs.readFileSync("datasette/static/datasette-manager.js", "utf8"), + { filename: "datasette-manager.js" } + ); + for (const callback of documentListeners.DOMContentLoaded || []) { + callback(); + } + window.__DATASETTE__.registerPlugin("agent", { + version: "0.1", + makeJumpSections() { + return [ + { + id: "agent-chat", + render(node, context) { + if (!context.navigationSearch) { + throw new Error("Expected navigationSearch in render context"); + } + node.innerHTML = [ + '<section class="agent-jump-start">', + '<button>Start a new agent chat</button>', + '</section>', + ].join(''); + }, + }, + ]; + }, + }); + + vm.runInThisContext( + fs.readFileSync("datasette/static/navigation-search.js", "utf8"), + { filename: "navigation-search.js" } + ); + + const Component = customElements.registry.get("navigation-search"); + const element = new Component(); + element.shadowRoot.input.value = ""; + element.renderResults(); + + const html = element.shadowRoot.results.innerHTML; + if (!html.includes("Start a new agent chat")) { + throw new Error(`Missing jump section content: ${html}`); + } + process.stdout.write("ok"); + """) + result = subprocess.run( + ["node", "-e", script], + cwd=".", + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.endswith("ok") diff --git a/tests/test_plugins.py b/tests/test_plugins.py index d87f577a..c5b9aef0 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -43,6 +43,11 @@ def test_plugin_hooks_have_tests(plugin_hook): assert ok, f"Plugin hook is missing tests: {plugin_hook}" +def test_hook_jump_items_sql(): + # Detailed behavior is covered in tests/test_jump.py. + assert "jump_items_sql" in dir(pm.hook) + + @pytest.mark.asyncio async def test_hook_plugins_dir_plugin_prepare_connection(ds_client): response = await ds_client.get( From 0eb78dec9a90c334c9e887a698cdcacf64ec8326 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 22 May 2026 21:19:13 -0700 Subject: [PATCH 066/156] Ran Prettier npx prettier 'datasette/static/*[!.min|bundle].js' --write --- datasette/static/navigation-search.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index 9e24681b..34ed2fc3 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -410,7 +410,12 @@ class NavigationSearch extends HTMLElement { } saveRecentItem(match) { - if (typeof localStorage === "undefined" || !match || !match.name || !match.url) { + if ( + typeof localStorage === "undefined" || + !match || + !match.name || + !match.url + ) { return; } @@ -559,7 +564,9 @@ class NavigationSearch extends HTMLElement { const recentHtml = recentItems.length ? `<div class="results-heading">Recent</div>${recentItems .map((match, index) => this.resultItemHtml(match, index)) - .join("")}<div class="recent-actions"><button type="button" class="clear-recent" data-clear-recent-items>Clear recent</button></div>` + .join( + "", + )}<div class="recent-actions"><button type="button" class="clear-recent" data-clear-recent-items>Clear recent</button></div>` : ""; const defaultHtml = defaultMatches .map((match, index) => From d44cfc3a55ae5b1466696147994e11a9ed00bf9a Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 22 May 2026 21:22:10 -0700 Subject: [PATCH 067/156] Fix for failing JS test --- tests/test_navigation_search_js.py | 31 +++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/test_navigation_search_js.py b/tests/test_navigation_search_js.py index 590985c9..acb1b85e 100644 --- a/tests/test_navigation_search_js.py +++ b/tests/test_navigation_search_js.py @@ -1,12 +1,18 @@ import json +from pathlib import Path import subprocess import textwrap +REPO_ROOT = Path(__file__).resolve().parents[1] +STATIC_DIR = REPO_ROOT / "datasette" / "static" + + def test_navigation_search_tracks_and_renders_recent_items(): script = textwrap.dedent(""" const fs = require("fs"); const vm = require("vm"); + const navigationSearchJs = __NAVIGATION_SEARCH_JS__; class FakeElement { constructor() { @@ -102,7 +108,7 @@ def test_navigation_search_tracks_and_renders_recent_items(): global.window = { location: { href: "" } }; vm.runInThisContext( - fs.readFileSync("datasette/static/navigation-search.js", "utf8"), + fs.readFileSync(navigationSearchJs, "utf8"), { filename: "navigation-search.js" } ); @@ -180,10 +186,13 @@ def test_navigation_search_tracks_and_renders_recent_items(): } process.stdout.write(JSON.stringify(stored)); - """) + """).replace( + "__NAVIGATION_SEARCH_JS__", + json.dumps(str(STATIC_DIR / "navigation-search.js")), + ) result = subprocess.run( ["node", "-e", script], - cwd=".", + cwd=REPO_ROOT, text=True, capture_output=True, check=False, @@ -203,6 +212,8 @@ def test_navigation_search_renders_jump_sections_from_javascript_plugins(): script = textwrap.dedent(""" const fs = require("fs"); const vm = require("vm"); + const datasetteManagerJs = __DATASETTE_MANAGER_JS__; + const navigationSearchJs = __NAVIGATION_SEARCH_JS__; const documentListeners = {}; @@ -320,7 +331,7 @@ def test_navigation_search_renders_jump_sections_from_javascript_plugins(): global.window = { datasetteVersion: "test", location: { href: "" } }; vm.runInThisContext( - fs.readFileSync("datasette/static/datasette-manager.js", "utf8"), + fs.readFileSync(datasetteManagerJs, "utf8"), { filename: "datasette-manager.js" } ); for (const callback of documentListeners.DOMContentLoaded || []) { @@ -348,7 +359,7 @@ def test_navigation_search_renders_jump_sections_from_javascript_plugins(): }); vm.runInThisContext( - fs.readFileSync("datasette/static/navigation-search.js", "utf8"), + fs.readFileSync(navigationSearchJs, "utf8"), { filename: "navigation-search.js" } ); @@ -362,10 +373,16 @@ def test_navigation_search_renders_jump_sections_from_javascript_plugins(): throw new Error(`Missing jump section content: ${html}`); } process.stdout.write("ok"); - """) + """).replace( + "__DATASETTE_MANAGER_JS__", + json.dumps(str(STATIC_DIR / "datasette-manager.js")), + ).replace( + "__NAVIGATION_SEARCH_JS__", + json.dumps(str(STATIC_DIR / "navigation-search.js")), + ) result = subprocess.run( ["node", "-e", script], - cwd=".", + cwd=REPO_ROOT, text=True, capture_output=True, check=False, From fba67250d1c4e7f6a7c47726ec998c79fb78ef95 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Fri, 22 May 2026 21:27:04 -0700 Subject: [PATCH 068/156] Ran Black --- tests/test_navigation_search_js.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/test_navigation_search_js.py b/tests/test_navigation_search_js.py index acb1b85e..b487357d 100644 --- a/tests/test_navigation_search_js.py +++ b/tests/test_navigation_search_js.py @@ -3,7 +3,6 @@ from pathlib import Path import subprocess import textwrap - REPO_ROOT = Path(__file__).resolve().parents[1] STATIC_DIR = REPO_ROOT / "datasette" / "static" @@ -209,7 +208,8 @@ def test_navigation_search_tracks_and_renders_recent_items(): def test_navigation_search_renders_jump_sections_from_javascript_plugins(): - script = textwrap.dedent(""" + script = ( + textwrap.dedent(""" const fs = require("fs"); const vm = require("vm"); const datasetteManagerJs = __DATASETTE_MANAGER_JS__; @@ -373,12 +373,15 @@ def test_navigation_search_renders_jump_sections_from_javascript_plugins(): throw new Error(`Missing jump section content: ${html}`); } process.stdout.write("ok"); - """).replace( - "__DATASETTE_MANAGER_JS__", - json.dumps(str(STATIC_DIR / "datasette-manager.js")), - ).replace( - "__NAVIGATION_SEARCH_JS__", - json.dumps(str(STATIC_DIR / "navigation-search.js")), + """) + .replace( + "__DATASETTE_MANAGER_JS__", + json.dumps(str(STATIC_DIR / "datasette-manager.js")), + ) + .replace( + "__NAVIGATION_SEARCH_JS__", + json.dumps(str(STATIC_DIR / "navigation-search.js")), + ) ) result = subprocess.run( ["node", "-e", script], From f46c245563ed7a50627dfcaf21824993d31025bf Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 08:58:51 -0700 Subject: [PATCH 069/156] blacken-docs --- docs/plugin_hooks.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index c3ce3ed9..b855d8b2 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1943,8 +1943,7 @@ This example adds a "Plugin dashboard" result for signed-in users: def jump_items_sql(actor): if not actor: return None - return JumpSQL( - sql=""" + return JumpSQL(sql=""" SELECT 'dashboard' AS type, 'plugin-dashboard' AS label, @@ -1956,8 +1955,7 @@ This example adds a "Plugin dashboard" result for signed-in users: 80 AS sort_key, 'my-plugin' AS source, 'Plugin dashboard' AS display_name - """ - ) + """) Use ``params=`` to pass SQL parameters. Datasette will automatically namespace those parameters before combining SQL fragments from different plugins. From 9e7419db8deb42c6236b6d6a142f6b225e010573 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 09:07:00 -0700 Subject: [PATCH 070/156] Remove navigation_search_js_hash mechanism Codex added this because CSS was not reloading in dev. --- datasette/app.py | 1 - datasette/handle_exception.py | 3 --- datasette/templates/base.html | 2 +- tests/test_html.py | 5 +---- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 088403e0..75f05d88 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2052,7 +2052,6 @@ class Datasette: and "ds_actor" in request.cookies and request.actor, "app_css_hash": self.app_css_hash(), - "navigation_search_js_hash": self.static_hash("navigation-search.js"), "zip": zip, "body_scripts": body_scripts, "format_bytes": format_bytes, diff --git a/datasette/handle_exception.py b/datasette/handle_exception.py index 25ec26d9..96398a4c 100644 --- a/datasette/handle_exception.py +++ b/datasette/handle_exception.py @@ -67,9 +67,6 @@ def handle_exception(datasette, request, exception): info, urls=datasette.urls, app_css_hash=datasette.app_css_hash(), - navigation_search_js_hash=datasette.static_hash( - "navigation-search.js" - ), menu_links=lambda: [], ) ), diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 02365e68..819715ba 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -70,7 +70,7 @@ {% endfor %} {% if select_templates %}<!-- Templates considered: {{ select_templates|join(", ") }} -->{% endif %} -<script src="{{ urls.static('navigation-search.js') }}{% if navigation_search_js_hash is defined %}?{{ navigation_search_js_hash }}{% endif %}" defer></script> +<script src="{{ urls.static('navigation-search.js') }}" defer></script> <navigation-search url="/-/jump"></navigation-search> </body> </html> diff --git a/tests/test_html.py b/tests/test_html.py index ac77f10f..4da321d2 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1022,10 +1022,7 @@ async def test_navigation_menu_links( navigation_search_script = soup.find( "script", {"src": re.compile(r"navigation-search\.js")} ) - assert navigation_search_script["src"] == ( - "/-/static/navigation-search.js?" - + ds_client.ds.static_hash("navigation-search.js") - ) + assert navigation_search_script["src"] == "/-/static/navigation-search.js" assert details.find("li").find("button") == search_button if not actor_id: # The app menu is always visible, but anonymous users do not see logout From 865f35ff10e03e8596b0267a9e49c05949fd4115 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 09:10:25 -0700 Subject: [PATCH 071/156] Move default Jump items to datasette.default_jump_items plugin --- datasette/default_jump_items.py | 119 ++++++++++++++++++++++++++++++++ datasette/plugins.py | 1 + datasette/views/special.py | 116 +------------------------------ 3 files changed, 122 insertions(+), 114 deletions(-) create mode 100644 datasette/default_jump_items.py diff --git a/datasette/default_jump_items.py b/datasette/default_jump_items.py new file mode 100644 index 00000000..74d8a87f --- /dev/null +++ b/datasette/default_jump_items.py @@ -0,0 +1,119 @@ +from datasette import hookimpl +from datasette.jump import JumpSQL + + +async def _query_display_names_sql(datasette, actor): + selects = [] + params = {} + for database_name in datasette.databases.keys(): + queries = await datasette.get_canned_queries(database_name, actor) + for query_name, query in queries.items(): + display_name = query.get("title") if isinstance(query, dict) else None + if not display_name: + continue + index = len(selects) + params[f"display_database_{index}"] = database_name + params[f"display_query_{index}"] = query_name + params[f"display_name_{index}"] = str(display_name) + selects.append(f""" + SELECT + :display_database_{index} AS database_name, + :display_query_{index} AS query_name, + :display_name_{index} AS display_name + """) + if not selects: + return ( + "SELECT NULL AS database_name, NULL AS query_name, NULL AS display_name WHERE 0", + {}, + ) + return " UNION ALL ".join(selects), params + + +@hookimpl +def jump_items_sql(datasette, actor, request): + async def inner(): + database_sql, database_params = await datasette.allowed_resources_sql( + action="view-database", actor=actor + ) + table_sql, table_params = await datasette.allowed_resources_sql( + action="view-table", actor=actor + ) + query_sql, query_params = await datasette.allowed_resources_sql( + action="view-query", actor=actor + ) + query_display_names_sql, query_display_names_params = ( + await _query_display_names_sql(datasette, actor) + ) + return [ + JumpSQL( + sql=f""" + WITH allowed_databases AS ( + {database_sql} + ) + SELECT + 'database' AS type, + parent AS label, + 'Database' AS description, + NULL AS url, + parent AS database_name, + NULL AS resource_name, + parent AS search_text, + 10 AS sort_key, + 'datasette' AS source, + NULL AS display_name + FROM allowed_databases + """, + params=database_params, + ), + JumpSQL( + sql=f""" + WITH allowed_tables AS ( + {table_sql} + ) + SELECT + CASE WHEN catalog_views.view_name IS NULL THEN 'table' ELSE 'view' END AS type, + allowed_tables.parent || ': ' || allowed_tables.child AS label, + CASE WHEN catalog_views.view_name IS NULL THEN 'Table' ELSE 'View' END AS description, + NULL AS url, + allowed_tables.parent AS database_name, + allowed_tables.child AS resource_name, + allowed_tables.parent || ' ' || allowed_tables.child AS search_text, + CASE WHEN catalog_views.view_name IS NULL THEN 20 ELSE 25 END AS sort_key, + 'datasette' AS source, + NULL AS display_name + FROM allowed_tables + LEFT JOIN catalog_views + ON catalog_views.database_name = allowed_tables.parent + AND catalog_views.view_name = allowed_tables.child + """, + params=table_params, + ), + JumpSQL( + sql=f""" + WITH allowed_queries AS ( + {query_sql} + ), + query_display_names AS ( + {query_display_names_sql} + ) + SELECT + 'query' AS type, + allowed_queries.parent || ': ' || allowed_queries.child AS label, + 'Canned query' AS description, + NULL AS url, + allowed_queries.parent AS database_name, + allowed_queries.child AS resource_name, + allowed_queries.parent || ' ' || allowed_queries.child || ' ' || COALESCE(query_display_names.display_name, '') AS search_text, + 30 AS sort_key, + 'datasette' AS source, + query_display_names.display_name AS display_name + FROM allowed_queries + LEFT JOIN query_display_names + ON query_display_names.database_name = allowed_queries.parent + AND query_display_names.query_name = allowed_queries.child + """, + params={**query_params, **query_display_names_params}, + ), + ] + + return inner diff --git a/datasette/plugins.py b/datasette/plugins.py index b01b386c..15041e59 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -29,6 +29,7 @@ DEFAULT_PLUGINS = ( "datasette.default_magic_parameters", "datasette.blob_renderer", "datasette.default_menu_links", + "datasette.default_jump_items", "datasette.handle_exception", "datasette.forbidden", "datasette.events", diff --git a/datasette/views/special.py b/datasette/views/special.py index cb6524dc..2022e4a7 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -921,118 +921,7 @@ class JumpView(BaseView): name = "jump" has_json_alternate = False - async def _query_display_names_sql(self, request): - selects = [] - params = {} - for database_name in self.ds.databases.keys(): - queries = await self.ds.get_canned_queries(database_name, request.actor) - for query_name, query in queries.items(): - display_name = query.get("title") if isinstance(query, dict) else None - if not display_name: - continue - index = len(selects) - params[f"display_database_{index}"] = database_name - params[f"display_query_{index}"] = query_name - params[f"display_name_{index}"] = str(display_name) - selects.append(f""" - SELECT - :display_database_{index} AS database_name, - :display_query_{index} AS query_name, - :display_name_{index} AS display_name - """) - if not selects: - return ( - "SELECT NULL AS database_name, NULL AS query_name, NULL AS display_name WHERE 0", - {}, - ) - return " UNION ALL ".join(selects), params - - async def _core_fragments(self, request): - database_sql, database_params = await self.ds.allowed_resources_sql( - action="view-database", actor=request.actor - ) - table_sql, table_params = await self.ds.allowed_resources_sql( - action="view-table", actor=request.actor - ) - query_sql, query_params = await self.ds.allowed_resources_sql( - action="view-query", actor=request.actor - ) - query_display_names_sql, query_display_names_params = ( - await self._query_display_names_sql(request) - ) - return [ - JumpSQL( - sql=f""" - WITH allowed_databases AS ( - {database_sql} - ) - SELECT - 'database' AS type, - parent AS label, - 'Database' AS description, - NULL AS url, - parent AS database_name, - NULL AS resource_name, - parent AS search_text, - 10 AS sort_key, - 'datasette' AS source, - NULL AS display_name - FROM allowed_databases - """, - params=database_params, - ), - JumpSQL( - sql=f""" - WITH allowed_tables AS ( - {table_sql} - ) - SELECT - CASE WHEN catalog_views.view_name IS NULL THEN 'table' ELSE 'view' END AS type, - allowed_tables.parent || ': ' || allowed_tables.child AS label, - CASE WHEN catalog_views.view_name IS NULL THEN 'Table' ELSE 'View' END AS description, - NULL AS url, - allowed_tables.parent AS database_name, - allowed_tables.child AS resource_name, - allowed_tables.parent || ' ' || allowed_tables.child AS search_text, - CASE WHEN catalog_views.view_name IS NULL THEN 20 ELSE 25 END AS sort_key, - 'datasette' AS source, - NULL AS display_name - FROM allowed_tables - LEFT JOIN catalog_views - ON catalog_views.database_name = allowed_tables.parent - AND catalog_views.view_name = allowed_tables.child - """, - params=table_params, - ), - JumpSQL( - sql=f""" - WITH allowed_queries AS ( - {query_sql} - ), - query_display_names AS ( - {query_display_names_sql} - ) - SELECT - 'query' AS type, - allowed_queries.parent || ': ' || allowed_queries.child AS label, - 'Canned query' AS description, - NULL AS url, - allowed_queries.parent AS database_name, - allowed_queries.child AS resource_name, - allowed_queries.parent || ' ' || allowed_queries.child || ' ' || COALESCE(query_display_names.display_name, '') AS search_text, - 30 AS sort_key, - 'datasette' AS source, - query_display_names.display_name AS display_name - FROM allowed_queries - LEFT JOIN query_display_names - ON query_display_names.database_name = allowed_queries.parent - AND query_display_names.query_name = allowed_queries.child - """, - params={**query_params, **query_display_names_params}, - ), - ] - - async def _plugin_fragments(self, request): + async def _fragments(self, request): fragments = [] for hook in pm.hook.jump_items_sql( datasette=self.ds, @@ -1070,8 +959,7 @@ class JumpView(BaseView): q = request.args.get("q", "").strip() terms = q.split() pattern = "%" + "%".join(terms) + "%" if terms else "%" - fragments = await self._core_fragments(request) - fragments.extend(await self._plugin_fragments(request)) + fragments = await self._fragments(request) union_parts = [] all_params = {"q": q, "pattern": pattern} From 09ccab97ccebb7e5570879acdb8b7717c274aa5c Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 23 May 2026 09:34:03 -0700 Subject: [PATCH 072/156] Run cog to update generated plugin docs (#2734) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- docs/plugins.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/plugins.rst b/docs/plugins.rst index d9938dba..90bc9d35 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -216,6 +216,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "register_column_types" ] }, + { + "name": "datasette.default_jump_items", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "jump_items_sql" + ] + }, { "name": "datasette.default_magic_parameters", "static": false, From 1590444fa34c132eaecf49391033925b8fb123a8 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 16:42:29 -0700 Subject: [PATCH 073/156] Simplify by removing _query_display_names_sql See https://github.com/simonw/datasette/pull/2732/changes#r3293627533 --- datasette/default_jump_items.py | 42 +++------------------------------ docs/introspection.rst | 7 ++---- tests/test_jump.py | 15 ++++++------ 3 files changed, 12 insertions(+), 52 deletions(-) diff --git a/datasette/default_jump_items.py b/datasette/default_jump_items.py index 74d8a87f..bacc3649 100644 --- a/datasette/default_jump_items.py +++ b/datasette/default_jump_items.py @@ -2,33 +2,6 @@ from datasette import hookimpl from datasette.jump import JumpSQL -async def _query_display_names_sql(datasette, actor): - selects = [] - params = {} - for database_name in datasette.databases.keys(): - queries = await datasette.get_canned_queries(database_name, actor) - for query_name, query in queries.items(): - display_name = query.get("title") if isinstance(query, dict) else None - if not display_name: - continue - index = len(selects) - params[f"display_database_{index}"] = database_name - params[f"display_query_{index}"] = query_name - params[f"display_name_{index}"] = str(display_name) - selects.append(f""" - SELECT - :display_database_{index} AS database_name, - :display_query_{index} AS query_name, - :display_name_{index} AS display_name - """) - if not selects: - return ( - "SELECT NULL AS database_name, NULL AS query_name, NULL AS display_name WHERE 0", - {}, - ) - return " UNION ALL ".join(selects), params - - @hookimpl def jump_items_sql(datasette, actor, request): async def inner(): @@ -41,9 +14,6 @@ def jump_items_sql(datasette, actor, request): query_sql, query_params = await datasette.allowed_resources_sql( action="view-query", actor=actor ) - query_display_names_sql, query_display_names_params = ( - await _query_display_names_sql(datasette, actor) - ) return [ JumpSQL( sql=f""" @@ -92,9 +62,6 @@ def jump_items_sql(datasette, actor, request): sql=f""" WITH allowed_queries AS ( {query_sql} - ), - query_display_names AS ( - {query_display_names_sql} ) SELECT 'query' AS type, @@ -103,16 +70,13 @@ def jump_items_sql(datasette, actor, request): NULL AS url, allowed_queries.parent AS database_name, allowed_queries.child AS resource_name, - allowed_queries.parent || ' ' || allowed_queries.child || ' ' || COALESCE(query_display_names.display_name, '') AS search_text, + allowed_queries.parent || ' ' || allowed_queries.child AS search_text, 30 AS sort_key, 'datasette' AS source, - query_display_names.display_name AS display_name + NULL AS display_name FROM allowed_queries - LEFT JOIN query_display_names - ON query_display_names.database_name = allowed_queries.parent - AND query_display_names.query_name = allowed_queries.child """, - params={**query_params, **query_display_names_params}, + params=query_params, ), ] diff --git a/docs/introspection.rst b/docs/introspection.rst index a5a9753a..b6ee1690 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -152,10 +152,8 @@ Shows currently attached databases. `Databases example <https://latest.datasette Returns a JSON list of items that the current actor has permission to view for Datasette's jump menu. By default this includes visible databases, tables, views and canned queries, and plugins can contribute additional items. The endpoint supports a ``?q=`` query parameter for filtering items by name. -Canned queries with a configured ``title`` also include a ``display_name`` in -their results, and can be found by searching for that title. Plugins can provide -the same extra field from ``jump_items_sql`` by returning a ``display_name`` -column. +Plugins can provide an optional ``display_name`` field from +``jump_items_sql`` by returning a ``display_name`` column. `Jump example <https://latest.datasette.io/-/jump>`_: @@ -177,7 +175,6 @@ column. }, { "name": "fixtures: recent_releases", - "display_name": "Recent Datasette releases", "url": "/fixtures/recent_releases", "type": "query", "description": "Canned query" diff --git a/tests/test_jump.py b/tests/test_jump.py index acffe4a9..27238695 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -89,10 +89,6 @@ async def test_jump_searches_tables_databases_views_and_canned_queries(ds_for_ju assert ("view", "content: comment_summary") in matches_by_type_and_name assert ("query", "content: recent_comments") in matches_by_type_and_name assert matches_by_type_and_name[("database", "content")]["url"] == "/content" - assert ( - matches_by_type_and_name[("query", "content: recent_comments")]["display_name"] - == "Recent comments" - ) assert ( matches_by_type_and_name[("query", "content: recent_comments")]["url"] == "/content/recent_comments" @@ -100,17 +96,20 @@ async def test_jump_searches_tables_databases_views_and_canned_queries(ds_for_ju @pytest.mark.asyncio -async def test_jump_searches_and_displays_canned_query_titles(ds_for_jump): +async def test_jump_uses_canned_query_names_not_titles(ds_for_jump): response = await ds_for_jump.client.get( "/-/jump.json?q=datasette", actor={"id": "user"} ) assert response.status_code == 200 - data = response.json() + assert response.json()["matches"] == [] - assert data["matches"] == [ + response = await ds_for_jump.client.get( + "/-/jump.json?q=release", actor={"id": "user"} + ) + assert response.status_code == 200 + assert response.json()["matches"] == [ { "name": "content: release_notes", - "display_name": "Recent Datasette releases", "url": "/content/release_notes", "type": "query", "description": "Canned query", From be1b5b2b5ca526bc3441864e0d2b7986012ea300 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 16:57:09 -0700 Subject: [PATCH 074/156] Move debug links into jump menu --- datasette/default_debug_menu.py | 36 ++++++++++++++++ datasette/default_menu_links.py | 41 ------------------ datasette/jump.py | 43 +++++++++++++++++++ datasette/plugins.py | 2 +- docs/authentication.rst | 2 +- docs/changelog.rst | 7 +++- docs/introspection.rst | 2 - docs/plugin_hooks.rst | 25 ++++++----- docs/plugins.rst | 4 +- tests/test_html.py | 2 +- tests/test_jump.py | 73 ++++++++++++++++++++++++++++++++- tests/test_permissions.py | 1 - 12 files changed, 173 insertions(+), 65 deletions(-) create mode 100644 datasette/default_debug_menu.py delete mode 100644 datasette/default_menu_links.py diff --git a/datasette/default_debug_menu.py b/datasette/default_debug_menu.py new file mode 100644 index 00000000..d1c2cd6c --- /dev/null +++ b/datasette/default_debug_menu.py @@ -0,0 +1,36 @@ +from datasette import hookimpl +from datasette.jump import JumpSQL + +DEBUG_MENU_ITEMS = ( + ("/-/databases", "Databases"), + ("/-/plugins", "Installed plugins"), + ("/-/versions", "Version info"), + ("/-/settings", "Settings"), + ("/-/permissions", "Debug permissions"), + ("/-/messages", "Debug messages"), + ("/-/allow-debug", "Debug allow rules"), + ("/-/threads", "Debug threads"), + ("/-/actor", "Debug actor"), + ("/-/patterns", "Pattern portfolio"), +) + + +@hookimpl +def jump_items_sql(datasette, actor, request): + async def inner(): + if not await datasette.allowed(action="debug-menu", actor=actor): + return [] + + return [ + JumpSQL.menu_item( + label=label, + url=datasette.urls.path(path), + description="Debug menu", + source="datasette.default_debug_menu", + sort_key=70 + index, + item_type="debug", + ) + for index, (path, label) in enumerate(DEBUG_MENU_ITEMS) + ] + + return inner diff --git a/datasette/default_menu_links.py b/datasette/default_menu_links.py deleted file mode 100644 index 85032387..00000000 --- a/datasette/default_menu_links.py +++ /dev/null @@ -1,41 +0,0 @@ -from datasette import hookimpl - - -@hookimpl -def menu_links(datasette, actor): - async def inner(): - if not await datasette.allowed(action="debug-menu", actor=actor): - return [] - - return [ - {"href": datasette.urls.path("/-/databases"), "label": "Databases"}, - { - "href": datasette.urls.path("/-/plugins"), - "label": "Installed plugins", - }, - { - "href": datasette.urls.path("/-/versions"), - "label": "Version info", - }, - { - "href": datasette.urls.path("/-/settings"), - "label": "Settings", - }, - { - "href": datasette.urls.path("/-/permissions"), - "label": "Debug permissions", - }, - { - "href": datasette.urls.path("/-/messages"), - "label": "Debug messages", - }, - { - "href": datasette.urls.path("/-/allow-debug"), - "label": "Debug allow rules", - }, - {"href": datasette.urls.path("/-/threads"), "label": "Debug threads"}, - {"href": datasette.urls.path("/-/actor"), "label": "Debug actor"}, - {"href": datasette.urls.path("/-/patterns"), "label": "Pattern portfolio"}, - ] - - return inner diff --git a/datasette/jump.py b/datasette/jump.py index 6ec7ae13..7ef5ce2b 100644 --- a/datasette/jump.py +++ b/datasette/jump.py @@ -10,6 +10,49 @@ class JumpSQL: sql: str params: dict[str, Any] | None = None + @classmethod + def menu_item( + cls, + *, + label: str, + url: str, + description: str = "Menu item", + source: str = "datasette", + sort_key: int = 50, + search_text: str | None = None, + display_name: str | None = None, + item_type: str = "menu", + ) -> "JumpSQL": + if search_text is None: + search_text = " ".join( + text for text in (label, display_name, description) if text is not None + ) + return cls( + sql=""" + SELECT + :type AS type, + :label AS label, + :description AS description, + :url AS url, + NULL AS database_name, + NULL AS resource_name, + :search_text AS search_text, + :sort_key AS sort_key, + :source AS source, + :display_name AS display_name + """, + params={ + "type": item_type, + "label": label, + "description": description, + "url": url, + "search_text": search_text, + "sort_key": sort_key, + "source": source, + "display_name": display_name, + }, + ) + _PARAM_RE = re.compile(r"(?<!:):([A-Za-z_][A-Za-z0-9_]*)") diff --git a/datasette/plugins.py b/datasette/plugins.py index 15041e59..f532ac60 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -28,7 +28,7 @@ DEFAULT_PLUGINS = ( "datasette.default_column_types", "datasette.default_magic_parameters", "datasette.blob_renderer", - "datasette.default_menu_links", + "datasette.default_debug_menu", "datasette.default_jump_items", "datasette.handle_exception", "datasette.forbidden", diff --git a/docs/authentication.rst b/docs/authentication.rst index 7fa3a241..7daefab7 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1398,4 +1398,4 @@ Actor is allowed to view the ``/-/permissions`` debug tools. debug-menu ---------- -Controls if the various debug pages are displayed in the navigation menu. +Controls if the various debug pages are displayed in the jump menu. diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d52d04d..d2479590 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,11 @@ Unreleased - Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) - Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`) - New "Jump to..." menu item, always visible, for triggering the previously undocumented ``/`` menu. (:issue:`2725`) +- The ``/`` jump-to search interface now covers databases, views, canned queries and plugin-provided items in addition to tables. The endpoint backing it has been renamed from ``/-/tables`` to ``/-/jump``. +- New :ref:`plugin_hook_jump_items_sql` plugin hook, allowing plugins to contribute additional items to the jump-to menu by returning SQL that queries the internal catalog. +- ``datasette.jump.JumpSQL.menu_item()`` is a shortcut for adding individual jump menu items that are not backed by resources in the internal catalog. +- New :ref:`javascript_plugins_makeJumpSections` JavaScript plugin hook, allowing plugins to add custom blank-state sections to the jump-to menu before the user has typed a query. +- Debug menu links now appear in the jump-to menu instead of the top-right app menu. - New documented :ref:`datasette.fixtures.populate_fixture_database(conn) <datasette_fixtures_populate_fixture_database>` helper for creating the fixture database tables used by Datasette's own tests, intended for plugin test suites. .. _v1_0_a29: @@ -274,7 +279,7 @@ Other changes ~~~~~~~~~~~~~ - The internal ``catalog_views`` table now tracks SQLite views alongside tables in the introspection database. (:issue:`2495`) -- Hitting the ``/`` brings up a search interface for navigating to databases, tables, views, canned queries and plugin-provided items that the current user can view. A new ``/-/jump`` endpoint supports this functionality, and JavaScript plugins can add custom blank-state sections using ``makeJumpSections()``. (:issue:`2523`) +- Hitting the ``/`` brings up a search interface for navigating to tables that the current user can view. A new ``/-/tables`` endpoint supports this functionality. (:issue:`2523`) - Datasette attempts to detect some configuration errors on startup. - Datasette now supports Python 3.14 and no longer tests against Python 3.9. diff --git a/docs/introspection.rst b/docs/introspection.rst index b6ee1690..8476c22a 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -152,8 +152,6 @@ Shows currently attached databases. `Databases example <https://latest.datasette Returns a JSON list of items that the current actor has permission to view for Datasette's jump menu. By default this includes visible databases, tables, views and canned queries, and plugins can contribute additional items. The endpoint supports a ``?q=`` query parameter for filtering items by name. -Plugins can provide an optional ``display_name`` field from -``jump_items_sql`` by returning a ``display_name`` column. `Jump example <https://latest.datasette.io/-/jump>`_: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index b855d8b2..71d429ac 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1943,22 +1943,21 @@ This example adds a "Plugin dashboard" result for signed-in users: def jump_items_sql(actor): if not actor: return None - return JumpSQL(sql=""" - SELECT - 'dashboard' AS type, - 'plugin-dashboard' AS label, - 'Dashboard' AS description, - '/-/plugin-dashboard' AS url, - NULL AS database_name, - NULL AS resource_name, - 'plugin dashboard' AS search_text, - 80 AS sort_key, - 'my-plugin' AS source, - 'Plugin dashboard' AS display_name - """) + return JumpSQL.menu_item( + item_type="dashboard", + label="plugin-dashboard", + description="Dashboard", + url="/-/plugin-dashboard", + search_text="plugin dashboard", + sort_key=80, + source="my-plugin", + display_name="Plugin dashboard", + ) Use ``params=`` to pass SQL parameters. Datasette will automatically namespace those parameters before combining SQL fragments from different plugins. +``JumpSQL.menu_item(...)`` is a shortcut for adding a single jump menu item that is not backed by a resource in Datasette's internal catalog tables. It returns ``NULL`` for ``database_name`` and ``resource_name`` and accepts the keyword arguments shown above. + .. _plugin_actions: Action hooks diff --git a/docs/plugins.rst b/docs/plugins.rst index 90bc9d35..e79acfe0 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -235,12 +235,12 @@ If you run ``datasette plugins --all`` it will include default plugins that ship ] }, { - "name": "datasette.default_menu_links", + "name": "datasette.default_debug_menu", "static": false, "templates": false, "version": null, "hooks": [ - "menu_links" + "jump_items_sql" ] }, { diff --git a/tests/test_html.py b/tests/test_html.py index 4da321d2..efc1040d 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -994,7 +994,7 @@ def test_edit_sql_link_not_shown_if_user_lacks_permission(has_permission): [ (None, None, None), ("test", None, ["/-/permissions"]), - ("root", ["/-/permissions", "/-/allow-debug"], None), + ("root", None, ["/-/permissions", "/-/allow-debug"]), ], ) async def test_navigation_menu_links( diff --git a/tests/test_jump.py b/tests/test_jump.py index 27238695..af8f4856 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -3,6 +3,7 @@ import pytest_asyncio from datasette import hookimpl from datasette.app import Datasette +from datasette.jump import JumpSQL from datasette.plugins import pm @@ -140,9 +141,77 @@ async def test_jump_respects_resource_permissions(ds_for_jump): @pytest.mark.asyncio -async def test_jump_uses_plugin_sql_with_namespaced_parameters(ds_for_jump): - from datasette.jump import JumpSQL +async def test_jump_sql_menu_item_helper(ds_for_jump): + fragment = JumpSQL.menu_item( + label="Plugin dashboard", + url="/-/plugin-dashboard", + description="Plugin tool", + source="test-plugin", + sort_key=70, + search_text="dashboard plugin", + display_name="Plugin Dashboard", + item_type="plugin", + ) + result = await ds_for_jump.get_internal_database().execute( + fragment.sql, fragment.params + ) + assert dict(result.first()) == { + "type": "plugin", + "label": "Plugin dashboard", + "description": "Plugin tool", + "url": "/-/plugin-dashboard", + "database_name": None, + "resource_name": None, + "search_text": "dashboard plugin", + "sort_key": 70, + "source": "test-plugin", + "display_name": "Plugin Dashboard", + } + +@pytest.mark.asyncio +async def test_debug_menu_items_are_in_jump_for_debug_menu_permission(): + ds = Datasette( + config={ + "permissions": { + "debug-menu": {"id": "debugger"}, + } + } + ) + await ds.invoke_startup() + response = await ds.client.get("/-/jump.json?q=debug", actor={"id": "debugger"}) + assert response.status_code == 200 + debug_matches = [ + match for match in response.json()["matches"] if match["type"] == "debug" + ] + assert {match["name"]: match["url"] for match in debug_matches} == { + "Databases": "/-/databases", + "Installed plugins": "/-/plugins", + "Version info": "/-/versions", + "Settings": "/-/settings", + "Debug permissions": "/-/permissions", + "Debug messages": "/-/messages", + "Debug allow rules": "/-/allow-debug", + "Debug threads": "/-/threads", + "Debug actor": "/-/actor", + "Pattern portfolio": "/-/patterns", + } + assert {match["description"] for match in debug_matches} == {"Debug menu"} + + +@pytest.mark.asyncio +async def test_debug_menu_items_are_hidden_without_debug_menu_permission(): + ds = Datasette() + await ds.invoke_startup() + response = await ds.client.get("/-/jump.json?q=debug", actor={"id": "regular"}) + assert response.status_code == 200 + assert [ + match for match in response.json()["matches"] if match["type"] == "debug" + ] == [] + + +@pytest.mark.asyncio +async def test_jump_uses_plugin_sql_with_namespaced_parameters(ds_for_jump): class JumpPlugin: @hookimpl def jump_items_sql(self, datasette, actor, request): diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 0c09e773..8166532f 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -430,7 +430,6 @@ async def test_permissions_debug(ds_client, filter_): "result": True, "actor": {"id": "root"}, }, - {"action": "debug-menu", "result": False, "actor": None}, { "action": "view-instance", "result": True, From 9c1f8621eb9f339fc0ba59bb6e6ca2cb77942fb0 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 16:59:45 -0700 Subject: [PATCH 075/156] Request is always set for jump_items_sql() hook --- docs/plugin_hooks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 71d429ac..5a0c8af1 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1892,8 +1892,8 @@ jump_items_sql(datasette, actor, request) ``actor`` - dictionary or None The currently authenticated :ref:`actor <authentication_actor>`. -``request`` - :ref:`internals_request` or None - The current HTTP request. This can be ``None`` if the request object is not available. +``request`` - :ref:`internals_request` + The current HTTP request. This hook allows plugins to add extra results to Datasette's ``/`` jump menu, which is powered by the ``/-/jump`` JSON endpoint. From 0f7e4410c11108de4de30e7892bbb48058dbdd24 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 17:07:47 -0700 Subject: [PATCH 076/156] Better test name --- tests/test_allowed_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_allowed_resources.py b/tests/test_allowed_resources.py index 8048ae2c..e247aa78 100644 --- a/tests/test_allowed_resources.py +++ b/tests/test_allowed_resources.py @@ -51,7 +51,7 @@ async def test_ds(): @pytest.mark.asyncio -async def test_tables_endpoint_global_access(test_ds): +async def test_tables_allowed_resources_global_access(test_ds): """Test allowed_resources() with global access permissions""" def rules_callback(datasette, actor, action): From 21a79b34b8eab84f6f9668f1b92d559ad8124f06 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 20:28:02 -0700 Subject: [PATCH 077/156] Improvements to Jump SQL columns - Removed database_name and resource_name - url can now optionally return JSON to reuse datasette.urls. methods - description is now used as a truncated text description --- datasette/default_debug_menu.py | 65 ++++++++++++++++++++----- datasette/default_jump_items.py | 29 ++++++----- datasette/jump.py | 2 - datasette/static/navigation-search.js | 24 ++++++++- datasette/views/special.py | 41 +++++++++++----- docs/changelog.rst | 5 +- docs/introspection.rst | 10 ++-- docs/plugin_hooks.rst | 16 ++---- docs/plugins.rst | 18 +++---- tests/test_jump.py | 70 ++++++++++++++++++++++++--- 10 files changed, 207 insertions(+), 73 deletions(-) diff --git a/datasette/default_debug_menu.py b/datasette/default_debug_menu.py index d1c2cd6c..e9576d99 100644 --- a/datasette/default_debug_menu.py +++ b/datasette/default_debug_menu.py @@ -2,16 +2,56 @@ from datasette import hookimpl from datasette.jump import JumpSQL DEBUG_MENU_ITEMS = ( - ("/-/databases", "Databases"), - ("/-/plugins", "Installed plugins"), - ("/-/versions", "Version info"), - ("/-/settings", "Settings"), - ("/-/permissions", "Debug permissions"), - ("/-/messages", "Debug messages"), - ("/-/allow-debug", "Debug allow rules"), - ("/-/threads", "Debug threads"), - ("/-/actor", "Debug actor"), - ("/-/patterns", "Pattern portfolio"), + ( + "/-/databases", + "Databases", + "List of databases known to this Datasette instance.", + ), + ( + "/-/plugins", + "Installed plugins", + "Review loaded plugins, their versions and their registered hooks.", + ), + ( + "/-/versions", + "Version info", + "Check the Python, SQLite and dependency versions used by this server.", + ), + ( + "/-/settings", + "Settings", + "Inspect the active Datasette settings and configuration values.", + ), + ( + "/-/permissions", + "Debug permissions", + "Test permission checks for actors, actions and resources.", + ), + ( + "/-/messages", + "Debug messages", + "Try out temporary flash messages shown to users.", + ), + ( + "/-/allow-debug", + "Debug allow rules", + "Explore how allow blocks match actors against permission rules.", + ), + ( + "/-/threads", + "Debug threads", + "Inspect worker threads and database tasks.", + ), + ( + "/-/actor", + "Debug actor", + "View the actor object for the current signed-in user.", + ), + ( + "/-/patterns", + "Pattern portfolio", + "Browse Datasette UI patterns.", + ), ) @@ -25,12 +65,13 @@ def jump_items_sql(datasette, actor, request): JumpSQL.menu_item( label=label, url=datasette.urls.path(path), - description="Debug menu", + description=description, source="datasette.default_debug_menu", sort_key=70 + index, + search_text=f"debug {label} {description}", item_type="debug", ) - for index, (path, label) in enumerate(DEBUG_MENU_ITEMS) + for index, (path, label, description) in enumerate(DEBUG_MENU_ITEMS) ] return inner diff --git a/datasette/default_jump_items.py b/datasette/default_jump_items.py index bacc3649..844fb6b6 100644 --- a/datasette/default_jump_items.py +++ b/datasette/default_jump_items.py @@ -23,10 +23,11 @@ def jump_items_sql(datasette, actor, request): SELECT 'database' AS type, parent AS label, - 'Database' AS description, - NULL AS url, - parent AS database_name, - NULL AS resource_name, + NULL AS description, + json_object( + 'method', 'database', + 'database', parent + ) AS url, parent AS search_text, 10 AS sort_key, 'datasette' AS source, @@ -43,10 +44,12 @@ def jump_items_sql(datasette, actor, request): SELECT CASE WHEN catalog_views.view_name IS NULL THEN 'table' ELSE 'view' END AS type, allowed_tables.parent || ': ' || allowed_tables.child AS label, - CASE WHEN catalog_views.view_name IS NULL THEN 'Table' ELSE 'View' END AS description, - NULL AS url, - allowed_tables.parent AS database_name, - allowed_tables.child AS resource_name, + NULL AS description, + json_object( + 'method', 'table', + 'database', allowed_tables.parent, + 'table', allowed_tables.child + ) AS url, allowed_tables.parent || ' ' || allowed_tables.child AS search_text, CASE WHEN catalog_views.view_name IS NULL THEN 20 ELSE 25 END AS sort_key, 'datasette' AS source, @@ -66,10 +69,12 @@ def jump_items_sql(datasette, actor, request): SELECT 'query' AS type, allowed_queries.parent || ': ' || allowed_queries.child AS label, - 'Canned query' AS description, - NULL AS url, - allowed_queries.parent AS database_name, - allowed_queries.child AS resource_name, + NULL AS description, + json_object( + 'method', 'query', + 'database', allowed_queries.parent, + 'query', allowed_queries.child + ) AS url, allowed_queries.parent || ' ' || allowed_queries.child AS search_text, 30 AS sort_key, 'datasette' AS source, diff --git a/datasette/jump.py b/datasette/jump.py index 7ef5ce2b..5a67d49e 100644 --- a/datasette/jump.py +++ b/datasette/jump.py @@ -34,8 +34,6 @@ class JumpSQL: :label AS label, :description AS description, :url AS url, - NULL AS database_name, - NULL AS resource_name, :search_text AS search_text, :sort_key AS sort_key, :source AS source, diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index 34ed2fc3..29a2f143 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -100,6 +100,11 @@ class NavigationSearch extends HTMLElement { background-color: #dbeafe; } + .result-item > div { + flex: 1; + min-width: 0; + } + .jump-start-content { border-bottom: 1px solid #e5e7eb; margin-bottom: 0.5rem; @@ -120,7 +125,7 @@ class NavigationSearch extends HTMLElement { color: #4b5563; } - .result-description { + .result-type { color: #4b5563; font-size: 0.75rem; font-weight: 600; @@ -132,6 +137,17 @@ class NavigationSearch extends HTMLElement { color: #6b7280; } + .result-description { + color: #374151; + display: -webkit-box; + font-size: 0.8125rem; + line-height: 1.35; + margin-top: 0.35rem; + overflow: hidden; + -webkit-box-orient: vertical; + -webkit-line-clamp: 2; + } + .results-heading { color: #4b5563; font-size: 0.75rem; @@ -500,6 +516,9 @@ class NavigationSearch extends HTMLElement { match.display_name && match.display_name !== match.name ? `<div class="result-label">${this.escapeHtml(match.name)}</div>` : ""; + const type = match.type + ? `<div class="result-type">${this.escapeHtml(match.type)}</div>` + : ""; const description = match.description ? `<div class="result-description">${this.escapeHtml( match.description, @@ -513,10 +532,11 @@ class NavigationSearch extends HTMLElement { aria-selected="${index === this.selectedIndex}" > <div> - ${description} + ${type} <div class="result-name">${this.escapeHtml(displayName)}</div> ${label} <div class="result-url">${this.escapeHtml(match.url)}</div> + ${description} </div> </div> `; diff --git a/datasette/views/special.py b/datasette/views/special.py index 2022e4a7..5b468f51 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -944,16 +944,33 @@ class JumpView(BaseView): raise TypeError("jump_items_sql must return JumpSQL instances") return fragments - def _url_for_row(self, row): - if row["url"]: - return row["url"] - if row["type"] == "database": - return self.ds.urls.database(row["database_name"]) - if row["type"] in ("table", "view"): - return self.ds.urls.table(row["database_name"], row["resource_name"]) - if row["type"] == "query": - return self.ds.urls.query(row["database_name"], row["resource_name"]) - return "" + def _resolve_url(self, url): + if not url or url.startswith("/"): + return url + + descriptor = json.loads(url) + if not isinstance(descriptor, dict): + raise TypeError("jump item url JSON must be an object") + method_name = descriptor.get("method") + if not isinstance(method_name, str) or not method_name: + raise TypeError("jump item url JSON must include a method") + if method_name.startswith("_"): + raise AttributeError(f"datasette.urls has no method named {method_name!r}") + try: + method = getattr(self.ds.urls, method_name) + except AttributeError as ex: + raise AttributeError( + f"datasette.urls has no method named {method_name!r}" + ) from ex + if not callable(method): + raise TypeError(f"datasette.urls.{method_name} is not callable") + kwargs = {key: value for key, value in descriptor.items() if key != "method"} + try: + return method(**kwargs) + except TypeError as ex: + raise TypeError( + f"Invalid arguments for datasette.urls.{method_name}(): {ex}" + ) from ex async def get(self, request): q = request.args.get("q", "").strip() @@ -975,8 +992,6 @@ class JumpView(BaseView): label, description, url, - database_name, - resource_name, search_text, sort_key, source, @@ -1016,7 +1031,7 @@ class JumpView(BaseView): for row in rows: match = { "name": row["label"], - "url": self._url_for_row(row), + "url": self._resolve_url(row["url"]), "type": row["type"], "description": row["description"], } diff --git a/docs/changelog.rst b/docs/changelog.rst index d2479590..84015b46 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,10 +14,11 @@ Unreleased - Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`) - New "Jump to..." menu item, always visible, for triggering the previously undocumented ``/`` menu. (:issue:`2725`) - The ``/`` jump-to search interface now covers databases, views, canned queries and plugin-provided items in addition to tables. The endpoint backing it has been renamed from ``/-/tables`` to ``/-/jump``. -- New :ref:`plugin_hook_jump_items_sql` plugin hook, allowing plugins to contribute additional items to the jump-to menu by returning SQL that queries the internal catalog. +- New :ref:`plugin_hook_jump_items_sql` plugin hook, allowing plugins to contribute additional items to the jump-to menu by returning SQL that queries the internal catalog. Each row returned by this hook includes a ``url`` value, which can be a string starting with ``/`` or a JSON object describing a call to one of the :ref:`internals_datasette_urls` methods. - ``datasette.jump.JumpSQL.menu_item()`` is a shortcut for adding individual jump menu items that are not backed by resources in the internal catalog. - New :ref:`javascript_plugins_makeJumpSections` JavaScript plugin hook, allowing plugins to add custom blank-state sections to the jump-to menu before the user has typed a query. -- Debug menu links now appear in the jump-to menu instead of the top-right app menu. +- Jump menu results now show their ``type`` as a category label, and can show optional longer ``description`` text for individual results. +- Debug menu links now appear in the jump-to menu instead of the top-right app menu, with descriptions for each debug item. - New documented :ref:`datasette.fixtures.populate_fixture_database(conn) <datasette_fixtures_populate_fixture_database>` helper for creating the fixture database tables used by Datasette's own tests, intended for plugin test suites. .. _v1_0_a29: diff --git a/docs/introspection.rst b/docs/introspection.rst index 8476c22a..d2eb8efd 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -151,6 +151,8 @@ Shows currently attached databases. `Databases example <https://latest.datasette Returns a JSON list of items that the current actor has permission to view for Datasette's jump menu. By default this includes visible databases, tables, views and canned queries, and plugins can contribute additional items. +Each item includes a ``type`` string used as a category label in the menu. Items can also include an optional ``description`` with longer text describing that individual result. + The endpoint supports a ``?q=`` query parameter for filtering items by name. `Jump example <https://latest.datasette.io/-/jump>`_: @@ -163,19 +165,19 @@ The endpoint supports a ``?q=`` query parameter for filtering items by name. "name": "fixtures", "url": "/fixtures", "type": "database", - "description": "Database" + "description": null }, { "name": "fixtures: facetable", "url": "/fixtures/facetable", "type": "table", - "description": "Table" + "description": null }, { "name": "fixtures: recent_releases", "url": "/fixtures/recent_releases", "type": "query", - "description": "Canned query" + "description": null } ], "truncated": false @@ -191,7 +193,7 @@ Search example with ``?q=facet`` returns only items matching ``.*facet.*``: "name": "fixtures: facetable", "url": "/fixtures/facetable", "type": "table", - "description": "Table" + "description": null } ], "truncated": false diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 5a0c8af1..c0b188cb 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1902,22 +1902,16 @@ Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Ea The SQL query must return these columns: ``type`` - A short type string for the result, for example ``"app"`` or ``"dashboard"``. + A short type string for the result, for example ``"app"`` or ``"dashboard"``. The jump menu displays this above the item as a category label. ``label`` The stable name for the result. This is returned as ``name`` in the JSON API and is used for sorting. ``description`` - A short description shown above the item in the jump menu. + Optional longer text describing this individual item, or ``NULL``. The jump menu displays this below the item's URL when it is present. ``url`` - The URL to navigate to when the item is selected. - -``database_name`` - The database name for Datasette resources, or ``NULL`` for custom plugin results. - -``resource_name`` - The table, view or query name for Datasette resources, or ``NULL`` for custom plugin results. + The URL to navigate to when the item is selected. This can be either a string starting with ``/`` or a JSON object describing a call to one of the :ref:`internals_datasette_urls` methods. For example, ``json_object('method', 'table', 'database', 'fixtures', 'table', 'facetable')`` calls ``datasette.urls.table(database='fixtures', table='facetable')``. Unknown methods or invalid named arguments will result in an error. ``search_text`` Text that should be searched by the ``?q=`` parameter. @@ -1946,7 +1940,7 @@ This example adds a "Plugin dashboard" result for signed-in users: return JumpSQL.menu_item( item_type="dashboard", label="plugin-dashboard", - description="Dashboard", + description="Review plugin status and configuration.", url="/-/plugin-dashboard", search_text="plugin dashboard", sort_key=80, @@ -1956,7 +1950,7 @@ This example adds a "Plugin dashboard" result for signed-in users: Use ``params=`` to pass SQL parameters. Datasette will automatically namespace those parameters before combining SQL fragments from different plugins. -``JumpSQL.menu_item(...)`` is a shortcut for adding a single jump menu item that is not backed by a resource in Datasette's internal catalog tables. It returns ``NULL`` for ``database_name`` and ``resource_name`` and accepts the keyword arguments shown above. +``JumpSQL.menu_item(...)`` is a shortcut for adding a single jump menu item from Python code. It accepts the keyword arguments shown above. .. _plugin_actions: diff --git a/docs/plugins.rst b/docs/plugins.rst index e79acfe0..77958205 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -216,6 +216,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "register_column_types" ] }, + { + "name": "datasette.default_debug_menu", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "jump_items_sql" + ] + }, { "name": "datasette.default_jump_items", "static": false, @@ -234,15 +243,6 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "register_magic_parameters" ] }, - { - "name": "datasette.default_debug_menu", - "static": false, - "templates": false, - "version": null, - "hooks": [ - "jump_items_sql" - ] - }, { "name": "datasette.default_permissions", "static": false, diff --git a/tests/test_jump.py b/tests/test_jump.py index af8f4856..3c5cf1a1 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -5,6 +5,7 @@ from datasette import hookimpl from datasette.app import Datasette from datasette.jump import JumpSQL from datasette.plugins import pm +from datasette.views.special import JumpView @pytest_asyncio.fixture @@ -113,7 +114,7 @@ async def test_jump_uses_canned_query_names_not_titles(ds_for_jump): "name": "content: release_notes", "url": "/content/release_notes", "type": "query", - "description": "Canned query", + "description": None, } ] @@ -160,8 +161,6 @@ async def test_jump_sql_menu_item_helper(ds_for_jump): "label": "Plugin dashboard", "description": "Plugin tool", "url": "/-/plugin-dashboard", - "database_name": None, - "resource_name": None, "search_text": "dashboard plugin", "sort_key": 70, "source": "test-plugin", @@ -196,7 +195,13 @@ async def test_debug_menu_items_are_in_jump_for_debug_menu_permission(): "Debug actor": "/-/actor", "Pattern portfolio": "/-/patterns", } - assert {match["description"] for match in debug_matches} == {"Debug menu"} + descriptions_by_name = { + match["name"]: match["description"] for match in debug_matches + } + assert all(descriptions_by_name.values()) + assert descriptions_by_name["Databases"] == ( + "Inspect the databases, tables, views and columns known to this Datasette instance." + ) @pytest.mark.asyncio @@ -222,8 +227,6 @@ async def test_jump_uses_plugin_sql_with_namespaced_parameters(ds_for_jump): 'plugin-dashboard: ' || :actor_id AS label, 'Plugin supplied item' AS description, '/-/plugin-dashboard' AS url, - NULL AS database_name, - NULL AS resource_name, 'plugin dashboard ' || :actor_id AS search_text, 80 AS sort_key, 'test-plugin' AS source, @@ -256,6 +259,61 @@ async def test_jump_uses_plugin_sql_with_namespaced_parameters(ds_for_jump): ] +@pytest.mark.asyncio +async def test_jump_resolves_url_descriptors_from_sql(ds_for_jump): + class JumpPlugin: + @hookimpl + def jump_items_sql(self, datasette, actor, request): + return JumpSQL(sql=""" + SELECT + 'plugin' AS type, + 'Table descriptor' AS label, + NULL AS description, + json_object( + 'method', 'table', + 'database', 'content', + 'table', 'comments' + ) AS url, + 'table descriptor comments' AS search_text, + 80 AS sort_key, + 'test-plugin' AS source, + NULL AS display_name + """) + + plugin = JumpPlugin() + pm.register(plugin, name="test-jump-url-descriptor-plugin") + try: + response = await ds_for_jump.client.get( + "/-/jump.json?q=descriptor", actor={"id": "alice"} + ) + finally: + pm.unregister(name="test-jump-url-descriptor-plugin") + + assert response.status_code == 200 + plugin_matches = [ + match for match in response.json()["matches"] if match["type"] == "plugin" + ] + assert plugin_matches == [ + { + "name": "Table descriptor", + "url": "/content/comments", + "type": "plugin", + "description": None, + } + ] + + +@pytest.mark.asyncio +async def test_jump_url_descriptor_errors(ds_for_jump): + view = JumpView(ds_for_jump) + with pytest.raises(AttributeError): + view._resolve_url('{"method": "not_a_url_method"}') + with pytest.raises(TypeError): + view._resolve_url( + '{"method": "table", "database_name": "content", "table_name": "comments"}' + ) + + @pytest.mark.asyncio async def test_tables_endpoint_removed(ds_for_jump): response = await ds_for_jump.client.get("/-/tables.json") From c73ed1ee4e619ee9da002443da3e29e0bc93de4f Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 20:30:56 -0700 Subject: [PATCH 078/156] Fixed a test I broke --- tests/test_jump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_jump.py b/tests/test_jump.py index 3c5cf1a1..3282a7c0 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -200,7 +200,7 @@ async def test_debug_menu_items_are_in_jump_for_debug_menu_permission(): } assert all(descriptions_by_name.values()) assert descriptions_by_name["Databases"] == ( - "Inspect the databases, tables, views and columns known to this Datasette instance." + "List of databases known to this Datasette instance" ) From cef6aa85b6a67f29e97d898873fcb8097ca68141 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 20:41:32 -0700 Subject: [PATCH 079/156] Remove source and source_key columns from JumpSQL Refs https://github.com/simonw/datasette/pull/2732#issuecomment-4527290391 --- datasette/default_debug_menu.py | 4 +--- datasette/default_jump_items.py | 6 ------ datasette/jump.py | 6 ------ datasette/views/special.py | 10 +++++++--- docs/plugin_hooks.rst | 8 -------- tests/test_jump.py | 10 +--------- 6 files changed, 9 insertions(+), 35 deletions(-) diff --git a/datasette/default_debug_menu.py b/datasette/default_debug_menu.py index e9576d99..6127b2a6 100644 --- a/datasette/default_debug_menu.py +++ b/datasette/default_debug_menu.py @@ -66,12 +66,10 @@ def jump_items_sql(datasette, actor, request): label=label, url=datasette.urls.path(path), description=description, - source="datasette.default_debug_menu", - sort_key=70 + index, search_text=f"debug {label} {description}", item_type="debug", ) - for index, (path, label, description) in enumerate(DEBUG_MENU_ITEMS) + for path, label, description in DEBUG_MENU_ITEMS ] return inner diff --git a/datasette/default_jump_items.py b/datasette/default_jump_items.py index 844fb6b6..d215e7ec 100644 --- a/datasette/default_jump_items.py +++ b/datasette/default_jump_items.py @@ -29,8 +29,6 @@ def jump_items_sql(datasette, actor, request): 'database', parent ) AS url, parent AS search_text, - 10 AS sort_key, - 'datasette' AS source, NULL AS display_name FROM allowed_databases """, @@ -51,8 +49,6 @@ def jump_items_sql(datasette, actor, request): 'table', allowed_tables.child ) AS url, allowed_tables.parent || ' ' || allowed_tables.child AS search_text, - CASE WHEN catalog_views.view_name IS NULL THEN 20 ELSE 25 END AS sort_key, - 'datasette' AS source, NULL AS display_name FROM allowed_tables LEFT JOIN catalog_views @@ -76,8 +72,6 @@ def jump_items_sql(datasette, actor, request): 'query', allowed_queries.child ) AS url, allowed_queries.parent || ' ' || allowed_queries.child AS search_text, - 30 AS sort_key, - 'datasette' AS source, NULL AS display_name FROM allowed_queries """, diff --git a/datasette/jump.py b/datasette/jump.py index 5a67d49e..17c86bfd 100644 --- a/datasette/jump.py +++ b/datasette/jump.py @@ -17,8 +17,6 @@ class JumpSQL: label: str, url: str, description: str = "Menu item", - source: str = "datasette", - sort_key: int = 50, search_text: str | None = None, display_name: str | None = None, item_type: str = "menu", @@ -35,8 +33,6 @@ class JumpSQL: :description AS description, :url AS url, :search_text AS search_text, - :sort_key AS sort_key, - :source AS source, :display_name AS display_name """, params={ @@ -45,8 +41,6 @@ class JumpSQL: "description": description, "url": url, "search_text": search_text, - "sort_key": sort_key, - "source": source, "display_name": display_name, }, ) diff --git a/datasette/views/special.py b/datasette/views/special.py index 5b468f51..272273aa 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -993,8 +993,6 @@ class JumpView(BaseView): description, url, search_text, - sort_key, - source, display_name FROM ( {fragment_sql} @@ -1016,7 +1014,13 @@ class JumpView(BaseView): WHEN lower(COALESCE(display_name, label)) LIKE lower(:q || '%') THEN 1 ELSE 2 END, - sort_key, + CASE type + WHEN 'database' THEN 10 + WHEN 'table' THEN 20 + WHEN 'view' THEN 25 + WHEN 'query' THEN 30 + ELSE 50 + END, length(COALESCE(display_name, label)), label LIMIT 101 diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index c0b188cb..4d8e980f 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1916,12 +1916,6 @@ The SQL query must return these columns: ``search_text`` Text that should be searched by the ``?q=`` parameter. -``sort_key`` - A numeric value used to order results. - -``source`` - A string identifying the plugin that supplied the result. - ``display_name`` A human-readable label for the result, or ``NULL``. Datasette returns this as ``display_name`` in the JSON API, and the jump menu shows it as the primary readable label with ``name`` shown underneath. @@ -1943,8 +1937,6 @@ This example adds a "Plugin dashboard" result for signed-in users: description="Review plugin status and configuration.", url="/-/plugin-dashboard", search_text="plugin dashboard", - sort_key=80, - source="my-plugin", display_name="Plugin dashboard", ) diff --git a/tests/test_jump.py b/tests/test_jump.py index 3282a7c0..adb8626c 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -147,8 +147,6 @@ async def test_jump_sql_menu_item_helper(ds_for_jump): label="Plugin dashboard", url="/-/plugin-dashboard", description="Plugin tool", - source="test-plugin", - sort_key=70, search_text="dashboard plugin", display_name="Plugin Dashboard", item_type="plugin", @@ -162,8 +160,6 @@ async def test_jump_sql_menu_item_helper(ds_for_jump): "description": "Plugin tool", "url": "/-/plugin-dashboard", "search_text": "dashboard plugin", - "sort_key": 70, - "source": "test-plugin", "display_name": "Plugin Dashboard", } @@ -200,7 +196,7 @@ async def test_debug_menu_items_are_in_jump_for_debug_menu_permission(): } assert all(descriptions_by_name.values()) assert descriptions_by_name["Databases"] == ( - "List of databases known to this Datasette instance" + "List of databases known to this Datasette instance." ) @@ -228,8 +224,6 @@ async def test_jump_uses_plugin_sql_with_namespaced_parameters(ds_for_jump): 'Plugin supplied item' AS description, '/-/plugin-dashboard' AS url, 'plugin dashboard ' || :actor_id AS search_text, - 80 AS sort_key, - 'test-plugin' AS source, 'Plugin dashboard for ' || :actor_id AS display_name """, params={"actor_id": actor["id"] if actor else "anonymous"}, @@ -275,8 +269,6 @@ async def test_jump_resolves_url_descriptors_from_sql(ds_for_jump): 'table', 'comments' ) AS url, 'table descriptor comments' AS search_text, - 80 AS sort_key, - 'test-plugin' AS source, NULL AS display_name """) From c980234c41a9bc7a188f17c7f07e48ef1a7640a7 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 21:00:04 -0700 Subject: [PATCH 080/156] JumpSQL(database=) parameter Refs https://github.com/simonw/datasette/pull/2732#issuecomment-4527304912 --- datasette/jump.py | 3 +- datasette/views/special.py | 76 ++++++++++++++---- docs/changelog.rst | 2 +- docs/plugin_hooks.rst | 10 ++- tests/test_jump.py | 153 +++++++++++++++++++++++++++++++++++++ 5 files changed, 225 insertions(+), 19 deletions(-) diff --git a/datasette/jump.py b/datasette/jump.py index 17c86bfd..d138e827 100644 --- a/datasette/jump.py +++ b/datasette/jump.py @@ -9,6 +9,7 @@ from typing import Any class JumpSQL: sql: str params: dict[str, Any] | None = None + database: str | None = None @classmethod def menu_item( @@ -50,7 +51,7 @@ _PARAM_RE = re.compile(r"(?<!:):([A-Za-z_][A-Za-z0-9_]*)") def namespace_sql_params(sql: str, params: dict[str, Any], prefix: str): - """Rename named SQL parameters so UNION fragments cannot collide.""" + """Rename named SQL parameters so UNION query parameters cannot collide.""" if not params: return sql, {} diff --git a/datasette/views/special.py b/datasette/views/special.py index 272273aa..31e7f0c2 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -972,15 +972,28 @@ class JumpView(BaseView): f"Invalid arguments for datasette.urls.{method_name}(): {ex}" ) from ex - async def get(self, request): - q = request.args.get("q", "").strip() - terms = q.split() - pattern = "%" + "%".join(terms) + "%" if terms else "%" - fragments = await self._fragments(request) + def _sort_key(self, row, q): + display_label = row["display_name"] or row["label"] + display_label_lower = display_label.lower() + q_lower = q.lower() + if display_label_lower == q_lower: + relevance = 0 + elif display_label_lower.startswith(q_lower): + relevance = 1 + else: + relevance = 2 + type_sort = { + "database": 10, + "table": 20, + "view": 25, + "query": 30, + }.get(row["type"], 50) + return (relevance, type_sort, len(display_label), row["label"]) + async def _rows_for_database(self, database_name, indexed_fragments, q, pattern): + params = {"q": q, "pattern": pattern} union_parts = [] - all_params = {"q": q, "pattern": pattern} - for index, fragment in enumerate(fragments): + for index, fragment in indexed_fragments: fragment_sql, fragment_params = namespace_sql_params( fragment.sql, fragment.params or {}, @@ -998,13 +1011,18 @@ class JumpView(BaseView): {fragment_sql} ) """) - all_params.update(fragment_params) - + params.update(fragment_params) sql = f""" WITH jump_items AS ( {" UNION ALL ".join(union_parts)} ) - SELECT * + SELECT + type, + label, + description, + url, + search_text, + display_name FROM jump_items WHERE :q = '' OR search_text LIKE :pattern COLLATE NOCASE @@ -1025,10 +1043,40 @@ class JumpView(BaseView): label LIMIT 101 """ - result = await self.ds.get_internal_database().execute(sql, all_params) - rows = list(result.rows) - truncated = len(rows) > 100 - if truncated: + db = ( + self.ds.get_internal_database() + if database_name is None + else self.ds.get_database(database_name) + ) + result = await db.execute(sql, params) + return list(result.rows) + + async def get(self, request): + q = request.args.get("q", "").strip() + terms = q.split() + pattern = "%" + "%".join(terms) + "%" if terms else "%" + fragments = await self._fragments(request) + + fragments_by_database = {} + for index, fragment in enumerate(fragments): + fragments_by_database.setdefault(fragment.database, []).append( + (index, fragment) + ) + + rows = [] + truncated = False + for database_name, indexed_fragments in fragments_by_database.items(): + database_rows = await self._rows_for_database( + database_name, indexed_fragments, q, pattern + ) + if len(database_rows) > 100: + truncated = True + database_rows = database_rows[:100] + rows.extend(database_rows) + rows.sort(key=lambda row: self._sort_key(row, q)) + + if len(rows) > 100: + truncated = True rows = rows[:100] matches = [] diff --git a/docs/changelog.rst b/docs/changelog.rst index 84015b46..a51684ab 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,7 +14,7 @@ Unreleased - Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`) - New "Jump to..." menu item, always visible, for triggering the previously undocumented ``/`` menu. (:issue:`2725`) - The ``/`` jump-to search interface now covers databases, views, canned queries and plugin-provided items in addition to tables. The endpoint backing it has been renamed from ``/-/tables`` to ``/-/jump``. -- New :ref:`plugin_hook_jump_items_sql` plugin hook, allowing plugins to contribute additional items to the jump-to menu by returning SQL that queries the internal catalog. Each row returned by this hook includes a ``url`` value, which can be a string starting with ``/`` or a JSON object describing a call to one of the :ref:`internals_datasette_urls` methods. +- New :ref:`plugin_hook_jump_items_sql` plugin hook, allowing plugins to contribute additional items to the jump-to menu by returning SQL. ``JumpSQL`` queries run against Datasette's internal database by default, or can target another database using the optional ``database=`` argument. Datasette groups these queries by database and executes one ``UNION ALL`` query for each database. Each row returned by this hook includes a ``url`` value, which can be a string starting with ``/`` or a JSON object describing a call to one of the :ref:`internals_datasette_urls` methods. - ``datasette.jump.JumpSQL.menu_item()`` is a shortcut for adding individual jump menu items that are not backed by resources in the internal catalog. - New :ref:`javascript_plugins_makeJumpSections` JavaScript plugin hook, allowing plugins to add custom blank-state sections to the jump-to menu before the user has typed a query. - Jump menu results now show their ``type`` as a category label, and can show optional longer ``description`` text for individual results. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4d8e980f..e2789112 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1884,7 +1884,7 @@ Examples: `datasette-search-all <https://datasette.io/plugins/datasette-search-a .. _plugin_hook_jump_items_sql: jump_items_sql(datasette, actor, request) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------------------- ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. @@ -1897,7 +1897,11 @@ jump_items_sql(datasette, actor, request) This hook allows plugins to add extra results to Datasette's ``/`` jump menu, which is powered by the ``/-/jump`` JSON endpoint. -Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be combined with Datasette's own databases, tables, views and canned query results. +Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be searched alongside Datasette's own databases, tables, views and canned query results. + +``JumpSQL`` queries run against Datasette's internal database by default. To run a query against another database, pass its name as the optional ``database=`` argument. For example, ``JumpSQL(database="content", sql="...")`` runs against the ``content`` database. + +Datasette groups ``JumpSQL`` queries by database and executes one ``UNION ALL`` query for each database. The SQL query must return these columns: @@ -1940,7 +1944,7 @@ This example adds a "Plugin dashboard" result for signed-in users: display_name="Plugin dashboard", ) -Use ``params=`` to pass SQL parameters. Datasette will automatically namespace those parameters before combining SQL fragments from different plugins. +Use ``params=`` to pass SQL parameters. Datasette will automatically namespace those parameters before adding the SQL fragment to the per-database ``UNION ALL`` query. ``JumpSQL.menu_item(...)`` is a shortcut for adding a single jump menu item from Python code. It accepts the keyword arguments shown above. diff --git a/tests/test_jump.py b/tests/test_jump.py index adb8626c..f60af0fd 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -143,6 +143,10 @@ async def test_jump_respects_resource_permissions(ds_for_jump): @pytest.mark.asyncio async def test_jump_sql_menu_item_helper(ds_for_jump): + assert JumpSQL("SELECT 1").database is None + assert JumpSQL("SELECT 1", database="content").database == "content" + assert JumpSQL("SELECT 1", None, "content").database == "content" + fragment = JumpSQL.menu_item( label="Plugin dashboard", url="/-/plugin-dashboard", @@ -253,6 +257,155 @@ async def test_jump_uses_plugin_sql_with_namespaced_parameters(ds_for_jump): ] +@pytest.mark.asyncio +async def test_jump_sql_unions_fragments_by_database(ds_for_jump, monkeypatch): + class JumpPlugin: + @hookimpl + def jump_items_sql(self, datasette, actor, request): + return [ + JumpSQL(sql=""" + SELECT + 'plugin' AS type, + 'first-unioned-item' AS label, + NULL AS description, + '/-/first-unioned-item' AS url, + 'unioned item' AS search_text, + NULL AS display_name + """), + JumpSQL(sql=""" + SELECT + 'plugin' AS type, + 'second-unioned-item' AS label, + NULL AS description, + '/-/second-unioned-item' AS url, + 'unioned item' AS search_text, + NULL AS display_name + """), + JumpSQL( + """ + SELECT + 'plugin' AS type, + 'content-first-unioned-item' AS label, + NULL AS description, + '/-/content-first-unioned-item' AS url, + 'unioned item' AS search_text, + NULL AS display_name + """, + None, + "content", + ), + JumpSQL( + database="content", + sql=""" + SELECT + 'plugin' AS type, + 'content-second-unioned-item' AS label, + NULL AS description, + '/-/content-second-unioned-item' AS url, + 'unioned item' AS search_text, + NULL AS display_name + """, + ), + ] + + internal_db = ds_for_jump.get_internal_database() + original_execute = internal_db.execute + internal_jump_query_sql = [] + + async def internal_execute_with_recording(sql, *args, **kwargs): + if "unioned-item" in sql: + internal_jump_query_sql.append(sql) + return await original_execute(sql, *args, **kwargs) + + monkeypatch.setattr(internal_db, "execute", internal_execute_with_recording) + + content_db = ds_for_jump.get_database("content") + original_content_execute = content_db.execute + content_jump_query_sql = [] + + async def content_execute_with_recording(sql, *args, **kwargs): + if "unioned-item" in sql: + content_jump_query_sql.append(sql) + return await original_content_execute(sql, *args, **kwargs) + + monkeypatch.setattr(content_db, "execute", content_execute_with_recording) + + plugin = JumpPlugin() + pm.register(plugin, name="test-jump-union-plugin") + try: + response = await ds_for_jump.client.get( + "/-/jump.json?q=unioned", actor={"id": "alice"} + ) + finally: + pm.unregister(name="test-jump-union-plugin") + + assert response.status_code == 200 + assert len(internal_jump_query_sql) == 1 + assert " UNION ALL " in internal_jump_query_sql[0] + assert len(content_jump_query_sql) == 1 + assert " UNION ALL " in content_jump_query_sql[0] + assert {match["name"] for match in response.json()["matches"]} == { + "content-first-unioned-item", + "content-second-unioned-item", + "first-unioned-item", + "second-unioned-item", + } + + +@pytest.mark.asyncio +async def test_jump_sql_can_query_named_database(ds_for_jump): + content_db = ds_for_jump.get_database("content") + await content_db.execute_write( + "INSERT INTO comments (id, body) VALUES (1001, 'Named database jump target')" + ) + + class JumpPlugin: + @hookimpl + def jump_items_sql(self, datasette, actor, request): + return JumpSQL( + database="content", + sql=""" + SELECT + 'comment' AS type, + body AS label, + 'Comment from content database' AS description, + json_object( + 'method', 'table', + 'database', 'content', + 'table', 'comments' + ) AS url, + body AS search_text, + body AS display_name + FROM comments + WHERE id = :comment_id + """, + params={"comment_id": 1001}, + ) + + plugin = JumpPlugin() + pm.register(plugin, name="test-jump-content-db-plugin") + try: + response = await ds_for_jump.client.get( + "/-/jump.json?q=named+database", actor={"id": "alice"} + ) + finally: + pm.unregister(name="test-jump-content-db-plugin") + + assert response.status_code == 200 + plugin_matches = [ + match for match in response.json()["matches"] if match["type"] == "comment" + ] + assert plugin_matches == [ + { + "name": "Named database jump target", + "display_name": "Named database jump target", + "url": "/content/comments", + "type": "comment", + "description": "Comment from content database", + } + ] + + @pytest.mark.asyncio async def test_jump_resolves_url_descriptors_from_sql(ds_for_jump): class JumpPlugin: From c1525cb467c6fe9b83329d130d3576c49149dc5a Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 21:01:18 -0700 Subject: [PATCH 081/156] Improved examples in JumpSQL docs --- docs/plugin_hooks.rst | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index e2789112..8f585cb1 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1897,7 +1897,7 @@ jump_items_sql(datasette, actor, request) This hook allows plugins to add extra results to Datasette's ``/`` jump menu, which is powered by the ``/-/jump`` JSON endpoint. -Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be searched alongside Datasette's own databases, tables, views and canned query results. +Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be searched alongside Datasette's own databases, tables, views and canned query results. The hook can also be an ``async def`` function, or return an awaitable that resolves to one of these values. ``JumpSQL`` queries run against Datasette's internal database by default. To run a query against another database, pass its name as the optional ``database=`` argument. For example, ``JumpSQL(database="content", sql="...")`` runs against the ``content`` database. @@ -1923,7 +1923,9 @@ The SQL query must return these columns: ``display_name`` A human-readable label for the result, or ``NULL``. Datasette returns this as ``display_name`` in the JSON API, and the jump menu shows it as the primary readable label with ``name`` shown underneath. -This example adds a "Plugin dashboard" result for signed-in users: +Use ``params=`` to pass SQL parameters. Datasette will automatically namespace those parameters before adding the SQL fragment to the per-database ``UNION ALL`` query. + +This example returns a SQL fragment that searches rows from a ``dashboards`` table in the ``content`` database. The ``url`` column uses ``json_object()`` to describe a call to ``datasette.urls.row(database='content', table='dashboards', row_path=slug)``: .. code-block:: python @@ -1932,7 +1934,40 @@ This example adds a "Plugin dashboard" result for signed-in users: @hookimpl - def jump_items_sql(actor): + def jump_items_sql(datasette, actor, request): + if not actor: + return None + return JumpSQL( + sql=""" + SELECT + 'dashboard' AS type, + slug AS label, + description AS description, + json_object( + 'method', 'row', + 'database', 'content', + 'table', 'dashboards', + 'row_path', slug + ) AS url, + slug || ' ' || COALESCE(title, '') || ' ' || COALESCE(description, '') AS search_text, + title AS display_name + FROM dashboards + WHERE owner_id = :actor_id + """, + params={"actor_id": actor["id"]}, + database="content", + ) + +This example uses the ``JumpSQL.menu_item()`` shortcut to add a single "Plugin dashboard" result for signed-in users: + +.. code-block:: python + + from datasette import hookimpl + from datasette.jump import JumpSQL + + + @hookimpl + def jump_items_sql(datasette, actor, request): if not actor: return None return JumpSQL.menu_item( @@ -1944,8 +1979,6 @@ This example adds a "Plugin dashboard" result for signed-in users: display_name="Plugin dashboard", ) -Use ``params=`` to pass SQL parameters. Datasette will automatically namespace those parameters before adding the SQL fragment to the per-database ``UNION ALL`` query. - ``JumpSQL.menu_item(...)`` is a shortcut for adding a single jump menu item from Python code. It accepts the keyword arguments shown above. .. _plugin_actions: From b9cb8e9a30417b66ad0f1ad04c926c3f2e6cf164 Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sat, 23 May 2026 21:14:35 -0700 Subject: [PATCH 082/156] Tweaked JumpSQL changelog, refs #2731 --- docs/changelog.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a51684ab..37dd4e9d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,10 +14,9 @@ Unreleased - Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`) - New "Jump to..." menu item, always visible, for triggering the previously undocumented ``/`` menu. (:issue:`2725`) - The ``/`` jump-to search interface now covers databases, views, canned queries and plugin-provided items in addition to tables. The endpoint backing it has been renamed from ``/-/tables`` to ``/-/jump``. -- New :ref:`plugin_hook_jump_items_sql` plugin hook, allowing plugins to contribute additional items to the jump-to menu by returning SQL. ``JumpSQL`` queries run against Datasette's internal database by default, or can target another database using the optional ``database=`` argument. Datasette groups these queries by database and executes one ``UNION ALL`` query for each database. Each row returned by this hook includes a ``url`` value, which can be a string starting with ``/`` or a JSON object describing a call to one of the :ref:`internals_datasette_urls` methods. +- New :ref:`plugin_hook_jump_items_sql` plugin hook, allowing plugins to contribute additional items to the jump-to menu by returning SQL. ``JumpSQL`` queries run against Datasette's internal database by default, or can target another database using the optional ``database=`` argument. - ``datasette.jump.JumpSQL.menu_item()`` is a shortcut for adding individual jump menu items that are not backed by resources in the internal catalog. - New :ref:`javascript_plugins_makeJumpSections` JavaScript plugin hook, allowing plugins to add custom blank-state sections to the jump-to menu before the user has typed a query. -- Jump menu results now show their ``type`` as a category label, and can show optional longer ``description`` text for individual results. - Debug menu links now appear in the jump-to menu instead of the top-right app menu, with descriptions for each debug item. - New documented :ref:`datasette.fixtures.populate_fixture_database(conn) <datasette_fixtures_populate_fixture_database>` helper for creating the fixture database tables used by Datasette's own tests, intended for plugin test suites. From b013aa1f7f133c93f7e2947d0224b4cd9fcd520a Mon Sep 17 00:00:00 2001 From: wheelman <alcaminohot09@gmail.com> Date: Sun, 24 May 2026 09:51:13 +0530 Subject: [PATCH 083/156] Add CORS headers to /db?sql= query redirect (#2730) Closes #2728 --- datasette/views/database.py | 6 ++++-- tests/test_api.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index faf870d0..0cf93832 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -61,8 +61,10 @@ class DatabaseView(View): if request.url_vars.get("format"): redirect_url += "." + request.url_vars.get("format") redirect_url += "?" + request.query_string - return Response.redirect(redirect_url) - return await QueryView()(request, datasette) + response = Response.redirect(redirect_url) + if datasette.cors: + add_cors_headers(response.headers) + return response if format_ not in ("html", "json"): raise NotFound("Invalid format: {}".format(format_)) diff --git a/tests/test_api.py b/tests/test_api.py index 392030d6..f6187529 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -717,6 +717,17 @@ def test_cors( assert "Access-Control-Max-Age" not in response.headers +def test_cors_query_redirect(app_client_with_cors): + # /db?sql= redirects to /db/-/query - the redirect itself needs CORS + # headers, otherwise browsers refuse to follow it cross-origin + response = app_client_with_cors.get( + "/fixtures?sql=select+1", follow_redirects=False + ) + assert response.status == 302 + assert response.headers["Location"] == "/fixtures/-/query?sql=select+1" + assert response.headers["Access-Control-Allow-Origin"] == "*" + + @pytest.mark.parametrize( "path", ( From d11326b250fee1ee3e2782dfc830c59b9b11756d Mon Sep 17 00:00:00 2001 From: Simon Willison <swillison@gmail.com> Date: Sun, 24 May 2026 07:55:40 -0700 Subject: [PATCH 084/156] Fixes for jump to menu accessibility VoiceOver demo video: https://github.com/simonw/datasette/pull/2737#issuecomment-4527447724 HTML explainer: https://gisthost.github.io/?cbdea138b932cdc9cac6dd1e4681a20b Closes #2736 --- datasette/static/navigation-search.js | 267 ++++++++++++++++++++++++-- datasette/templates/base.html | 2 +- 2 files changed, 251 insertions(+), 18 deletions(-) diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index 29a2f143..ec2d23d8 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -1,10 +1,22 @@ +let navigationSearchInstanceCounter = 0; + class NavigationSearch extends HTMLElement { constructor() { super(); + this.instanceId = ++navigationSearchInstanceCounter; + this.inputId = `navigation-search-input-${this.instanceId}`; + this.instructionsId = `navigation-search-instructions-${this.instanceId}`; + this.listboxId = `navigation-search-results-${this.instanceId}`; + this.recentHeadingId = `navigation-search-recent-${this.instanceId}`; + this.statusId = `navigation-search-status-${this.instanceId}`; + this.titleId = `navigation-search-title-${this.instanceId}`; this.attachShadow({ mode: "open" }); this.selectedIndex = -1; this.matches = []; + this.renderedMatches = []; this.debounceTimer = null; + this.restoreFocusTarget = null; + this.shouldRestoreFocus = true; this.render(); this.setupEventListeners(); @@ -59,10 +71,15 @@ class NavigationSearch extends HTMLElement { .search-input-wrapper { padding: 1.25rem; border-bottom: 1px solid #e5e7eb; + display: flex; + gap: 0.5rem; + align-items: center; } .search-input { width: 100%; + flex: 1; + min-width: 0; padding: 0.75rem 1rem; font-size: 1rem; border: 2px solid #e5e7eb; @@ -76,12 +93,36 @@ class NavigationSearch extends HTMLElement { border-color: #2563eb; } + .close-search { + background: transparent; + border: 1px solid transparent; + border-radius: 0.375rem; + color: #4b5563; + cursor: pointer; + flex: 0 0 auto; + font: inherit; + font-size: 1.5rem; + height: 2.75rem; + line-height: 1; + width: 2.75rem; + } + + .close-search:hover, + .close-search:focus { + background-color: #f3f4f6; + border-color: #d1d5db; + } + .results-container { overflow-y: auto; height: calc(80vh - 180px); padding: 0.5rem; } + .results-list:empty { + display: none; + } + .result-item { padding: 0.875rem 1rem; cursor: pointer; @@ -200,6 +241,18 @@ class NavigationSearch extends HTMLElement { font-family: monospace; } + .visually-hidden { + border: 0; + clip: rect(0 0 0 0); + height: 1px; + margin: -1px; + overflow: hidden; + padding: 0; + position: absolute; + white-space: nowrap; + width: 1px; + } + /* Mobile optimizations */ @media (max-width: 640px) { dialog { @@ -227,19 +280,29 @@ class NavigationSearch extends HTMLElement { } </style> - <dialog> + <dialog aria-modal="true" aria-labelledby="${this.titleId}"> <div class="search-container"> + <h2 id="${this.titleId}" class="visually-hidden">Jump to</h2> + <p id="${this.instructionsId}" class="visually-hidden">Type to search. Use up and down arrow keys to move through results, Enter to select a result, and Escape to close this menu.</p> + <div id="${this.statusId}" class="visually-hidden" aria-live="polite" aria-atomic="true"></div> <div class="search-input-wrapper"> <input + id="${this.inputId}" type="text" class="search-input" placeholder="Jump to..." aria-label="Jump to" + aria-describedby="${this.instructionsId}" + role="combobox" + aria-autocomplete="list" + aria-controls="${this.listboxId}" + aria-expanded="false" autocomplete="off" spellcheck="false" > + <button type="button" class="close-search" aria-label="Close jump menu">×</button> </div> - <div class="results-container" role="listbox"></div> + <div class="results-container"></div> <div class="hint-text"> <span><kbd>↑</kbd> <kbd>↓</kbd> Navigate</span> <span><kbd>Enter</kbd> Select</span> @@ -253,6 +316,7 @@ class NavigationSearch extends HTMLElement { setupEventListeners() { const dialog = this.shadowRoot.querySelector("dialog"); const input = this.shadowRoot.querySelector(".search-input"); + const closeButton = this.shadowRoot.querySelector(".close-search"); const resultsContainer = this.shadowRoot.querySelector(".results-container"); @@ -268,8 +332,10 @@ class NavigationSearch extends HTMLElement { const trigger = e.target.closest("[data-navigation-search-open]"); if (trigger) { e.preventDefault(); - trigger.closest("details")?.removeAttribute("open"); - this.openMenu(); + const details = trigger.closest("details"); + const restoreTarget = details?.querySelector("summary") || trigger; + details?.removeAttribute("open"); + this.openMenu(restoreTarget); } }); @@ -294,6 +360,10 @@ class NavigationSearch extends HTMLElement { } }); + closeButton.addEventListener("click", () => { + this.closeMenu(); + }); + // Click on result item resultsContainer.addEventListener("click", (e) => { const clearRecent = e.target.closest("[data-clear-recent-items]"); @@ -317,6 +387,15 @@ class NavigationSearch extends HTMLElement { } }); + dialog.addEventListener("cancel", (e) => { + e.preventDefault(); + this.closeMenu(); + }); + + dialog.addEventListener("close", () => { + this.onMenuClosed(); + }); + // Initial load this.loadInitialData(); } @@ -331,6 +410,106 @@ class NavigationSearch extends HTMLElement { ); } + setElementAttribute(element, name, value) { + if (!element) { + return; + } + if (typeof element.setAttribute === "function") { + element.setAttribute(name, value); + } else { + element[name] = String(value); + } + } + + removeElementAttribute(element, name) { + if (!element) { + return; + } + if (typeof element.removeAttribute === "function") { + element.removeAttribute(name); + } else { + delete element[name]; + } + } + + focusRestoreTarget(trigger) { + if (trigger && typeof trigger.focus === "function") { + return trigger; + } + if ( + document.activeElement && + typeof document.activeElement.focus === "function" + ) { + return document.activeElement; + } + return null; + } + + setNavigationTriggersExpanded(expanded) { + if (typeof document.querySelectorAll !== "function") { + return; + } + document + .querySelectorAll("[data-navigation-search-open]") + .forEach((trigger) => { + this.setElementAttribute( + trigger, + "aria-expanded", + expanded ? "true" : "false", + ); + }); + } + + resultOptionId(index) { + return `${this.listboxId}-option-${index}`; + } + + updateComboboxState() { + const dialog = this.shadowRoot.querySelector("dialog"); + const input = this.shadowRoot.querySelector(".search-input"); + const matches = this.renderedMatches || []; + this.setElementAttribute( + input, + "aria-expanded", + dialog && dialog.open && matches.length > 0 ? "true" : "false", + ); + + if ( + dialog && + dialog.open && + this.selectedIndex >= 0 && + this.selectedIndex < matches.length + ) { + this.setElementAttribute( + input, + "aria-activedescendant", + this.resultOptionId(this.selectedIndex), + ); + } else { + this.removeElementAttribute(input, "aria-activedescendant"); + } + } + + setStatus(message) { + const status = this.shadowRoot.querySelector(`#${this.statusId}`); + if (status) { + status.textContent = message || ""; + } + } + + resultsStatus(count, truncated) { + if (truncated) { + return "More than 100 results. Keep typing to narrow the list."; + } + if (count === 0) { + return "No results found."; + } + if (count === 1) { + return "1 result."; + } + return `${count} results.`; + } + loadInitialData() { const itemsAttr = this.getAttribute("items"); if (itemsAttr) { @@ -347,6 +526,11 @@ class NavigationSearch extends HTMLElement { handleSearch(query) { clearTimeout(this.debounceTimer); + if (query.trim()) { + this.setStatus("Searching..."); + } else { + this.setStatus(""); + } this.debounceTimer = setTimeout(() => { const url = this.getAttribute("url"); @@ -369,10 +553,16 @@ class NavigationSearch extends HTMLElement { this.matches = data.matches || []; this.selectedIndex = this.matches.length > 0 ? 0 : -1; this.renderResults(); + if (query.trim()) { + this.setStatus(this.resultsStatus(this.matches.length, data.truncated)); + } else { + this.setStatus(""); + } } catch (e) { console.error("Failed to fetch search results:", e); this.matches = []; this.renderResults(); + this.setStatus("Search failed."); } } @@ -390,6 +580,11 @@ class NavigationSearch extends HTMLElement { } this.selectedIndex = this.matches.length > 0 ? 0 : -1; this.renderResults(); + if (query.trim()) { + this.setStatus(this.resultsStatus(this.matches.length, false)); + } else { + this.setStatus(""); + } } recentItemsStorageKey() { @@ -466,6 +661,7 @@ class NavigationSearch extends HTMLElement { localStorage.setItem(this.recentItemsStorageKey(), "[]"); } this.renderResults(); + this.setStatus("Recent items cleared."); } jumpSections() { @@ -526,6 +722,7 @@ class NavigationSearch extends HTMLElement { : ""; return ` <div + id="${this.resultOptionId(index)}" class="result-item ${index === this.selectedIndex ? "selected" : ""}" data-index="${index}" role="option" @@ -554,6 +751,7 @@ class NavigationSearch extends HTMLElement { const defaultMatches = showStartContent ? [] : this.matches; const renderedMatches = [...recentItems, ...defaultMatches]; this.renderedMatches = renderedMatches; + const emptyListbox = `<div id="${this.listboxId}" class="results-list" role="listbox" aria-label="Jump results"></div>`; if (renderedMatches.length) { if ( @@ -568,33 +766,43 @@ class NavigationSearch extends HTMLElement { if (renderedMatches.length === 0) { if (startBlock) { - container.innerHTML = startBlock; + container.innerHTML = startBlock + emptyListbox; this.renderJumpSections(container, jumpSections); } else if (showStartContent) { - container.innerHTML = ""; + container.innerHTML = emptyListbox; } else { const message = input.value.trim() ? "No results found" : "Start typing to search..."; - container.innerHTML = `<div class="no-results">${message}</div>`; + container.innerHTML = `${emptyListbox}<div class="no-results">${message}</div>`; } + this.updateComboboxState(); return; } - const recentHtml = recentItems.length - ? `<div class="results-heading">Recent</div>${recentItems + const recentHeading = recentItems.length + ? `<div class="results-heading" id="${this.recentHeadingId}">Recent</div>` + : ""; + const recentGroup = recentItems.length + ? `<div role="group" aria-labelledby="${this.recentHeadingId}">${recentItems .map((match, index) => this.resultItemHtml(match, index)) - .join( - "", - )}<div class="recent-actions"><button type="button" class="clear-recent" data-clear-recent-items>Clear recent</button></div>` + .join("")}</div>` + : ""; + const recentActions = recentItems.length + ? `<div class="recent-actions"><button type="button" class="clear-recent" data-clear-recent-items>Clear recent</button></div>` : ""; const defaultHtml = defaultMatches .map((match, index) => this.resultItemHtml(match, recentItems.length + index), ) .join(""); - container.innerHTML = startBlock + recentHtml + defaultHtml; + container.innerHTML = + startBlock + + recentHeading + + `<div id="${this.listboxId}" class="results-list" role="listbox" aria-label="Jump results">${recentGroup}${defaultHtml}</div>` + + recentActions; this.renderJumpSections(container, jumpSections); + this.updateComboboxState(); // Scroll selected item into view if (this.selectedIndex >= 0) { @@ -641,17 +849,20 @@ class NavigationSearch extends HTMLElement { // Navigate to URL window.location.href = match.url; - this.closeMenu(); + this.closeMenu({ restoreFocus: false }); } } - openMenu() { + openMenu(trigger) { const dialog = this.shadowRoot.querySelector("dialog"); const input = this.shadowRoot.querySelector(".search-input"); + this.restoreFocusTarget = this.focusRestoreTarget(trigger); + this.shouldRestoreFocus = true; if (!dialog.open) { dialog.showModal(); } + this.setNavigationTriggersExpanded(true); input.value = ""; input.focus(); @@ -659,11 +870,33 @@ class NavigationSearch extends HTMLElement { this.matches = []; this.selectedIndex = -1; this.renderResults(); + this.setStatus(""); } - closeMenu() { + closeMenu(options = {}) { const dialog = this.shadowRoot.querySelector("dialog"); - dialog.close(); + this.shouldRestoreFocus = options.restoreFocus !== false; + if (dialog.open) { + dialog.close(); + } else { + this.onMenuClosed(); + } + } + + onMenuClosed() { + const input = this.shadowRoot.querySelector(".search-input"); + this.setElementAttribute(input, "aria-expanded", "false"); + this.removeElementAttribute(input, "aria-activedescendant"); + this.setNavigationTriggersExpanded(false); + this.setStatus(""); + if ( + this.shouldRestoreFocus && + this.restoreFocusTarget && + typeof this.restoreFocusTarget.focus === "function" + ) { + this.restoreFocusTarget.focus(); + } + this.restoreFocusTarget = null; } escapeHtml(text) { diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 819715ba..e1767deb 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -30,7 +30,7 @@ </svg></summary> <div class="nav-menu-inner"> <ul> - <li><button type="button" class="button-as-link" data-navigation-search-open>Jump to... <kbd class="keyboard-shortcut" aria-hidden="true" title="Keyboard shortcut: press / to open Jump to">/</kbd></button></li> + <li><button type="button" class="button-as-link" data-navigation-search-open aria-haspopup="dialog" aria-expanded="false" aria-keyshortcuts="/">Jump to... <kbd class="keyboard-shortcut" aria-hidden="true" title="Keyboard shortcut: press / to open Jump to">/</kbd></button></li> {% for link in links %} <li><a href="{{ link.href }}">{{ link.label }}</a></li> {% endfor %} From 312740b97c93cadaeb1c46e92bce39afe9ecbbe4 Mon Sep 17 00:00:00 2001 From: pintaste <7534292+pintaste@users.noreply.github.com> Date: Mon, 25 May 2026 05:11:04 +0800 Subject: [PATCH 085/156] Keyboard navigation and ARIA attributes for actions menus (#2727) - Add aria-haspopup="menu" and aria-expanded to summary element - Add role="menu" to dropdown ul, role="menuitem" and tabindex="-1" to links - Sync aria-expanded via toggle event listener - Focus first menu item when menu opens - Add ArrowDown/ArrowUp navigation between menu items - Add Escape key to close menu and return focus to summary Refs #2738 --- datasette/templates/_action_menu.html | 6 +-- datasette/templates/_close_open_menus.html | 46 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/datasette/templates/_action_menu.html b/datasette/templates/_action_menu.html index 7d1d4a55..1ae8c173 100644 --- a/datasette/templates/_action_menu.html +++ b/datasette/templates/_action_menu.html @@ -1,7 +1,7 @@ {% if action_links %} <div class="page-action-menu"> <details class="actions-menu-links details-menu"> - <summary> + <summary aria-haspopup="menu" aria-expanded="false"> <div class="icon-text"> <svg class="icon" aria-labelledby="actions-menu-links-title" role="img" style="color: #fff" xmlns="http://www.w3.org/2000/svg" width="28" height="28" viewBox="0 0 28 28" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"> <title id="actions-menu-links-title">{{ action_title }} @@ -13,9 +13,9 @@
-
    +
      {% for link in action_links %} -
    • {{ link.label }} +
    • {{ link.label }} {% if link.description %}

      {{ link.description }}

      {% endif %}       diff --git a/datasette/templates/_close_open_menus.html b/datasette/templates/_close_open_menus.html index 3302d77d..a24633d0 100644 --- a/datasette/templates/_close_open_menus.html +++ b/datasette/templates/_close_open_menus.html @@ -13,4 +13,50 @@ document.body.addEventListener('click', (ev) => { (details) => details.open && details != detailsClickedWithin ).forEach(details => details.open = false); }); + +/* Sync aria-expanded and add keyboard navigation for details-menu elements */ +document.querySelectorAll('details.details-menu').forEach(function(details) { + var summary = details.querySelector('summary'); + details.addEventListener('toggle', function() { + if (summary) { + summary.setAttribute('aria-expanded', details.open ? 'true' : 'false'); + } + if (details.open) { + /* Focus first menu item when menu opens */ + var firstItem = details.querySelector('[role="menuitem"]'); + if (firstItem) { firstItem.focus(); } + } + }); +}); + +document.body.addEventListener('keydown', function(ev) { + /* Keyboard navigation for open details-menu elements */ + var openDetails = Array.from(document.querySelectorAll('details.details-menu[open]')); + if (!openDetails.length) { return; } + + if (ev.key === 'Escape') { + openDetails.forEach(function(details) { + details.open = false; + var summary = details.querySelector('summary'); + if (summary) { summary.focus(); } + }); + return; + } + + if (ev.key === 'ArrowDown' || ev.key === 'ArrowUp') { + var focused = document.activeElement; + openDetails.forEach(function(details) { + var items = Array.from(details.querySelectorAll('[role="menuitem"]')); + if (!items.length) { return; } + var idx = items.indexOf(focused); + if (idx === -1) { return; } + ev.preventDefault(); + if (ev.key === 'ArrowDown') { + items[(idx + 1) % items.length].focus(); + } else { + items[(idx - 1 + items.length) % items.length].focus(); + } + }); + } +}); From 857af9293c7b5cab2b430c92b15ed16fc4e481fa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 14:17:45 -0700 Subject: [PATCH 086/156] Release 1.0a30 Refs #1752, #2723, #2725, #2727 --- datasette/version.py | 2 +- docs/changelog.rst | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index e661e76d..494d2dc0 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a29" +__version__ = "1.0a30" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 37dd4e9d..65530452 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,21 +4,23 @@ Changelog ========= -.. _unreleased: +.. _v1_0_a30: -Unreleased ----------- +1.0a30 (2026-05-24) +------------------- + +The "Jump to" menu, activated by hitting ``/`` or through the application menu, can now be extended by plugins. -- Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) -- Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) -- Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`) - New "Jump to..." menu item, always visible, for triggering the previously undocumented ``/`` menu. (:issue:`2725`) - The ``/`` jump-to search interface now covers databases, views, canned queries and plugin-provided items in addition to tables. The endpoint backing it has been renamed from ``/-/tables`` to ``/-/jump``. - New :ref:`plugin_hook_jump_items_sql` plugin hook, allowing plugins to contribute additional items to the jump-to menu by returning SQL. ``JumpSQL`` queries run against Datasette's internal database by default, or can target another database using the optional ``database=`` argument. - ``datasette.jump.JumpSQL.menu_item()`` is a shortcut for adding individual jump menu items that are not backed by resources in the internal catalog. - New :ref:`javascript_plugins_makeJumpSections` JavaScript plugin hook, allowing plugins to add custom blank-state sections to the jump-to menu before the user has typed a query. - Debug menu links now appear in the jump-to menu instead of the top-right app menu, with descriptions for each debug item. +- Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) +- Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) - New documented :ref:`datasette.fixtures.populate_fixture_database(conn) ` helper for creating the fixture database tables used by Datasette's own tests, intended for plugin test suites. +- Keyboard accessibility and ARIA roles for actions menus, thanks `pintaste `__. (:pr:`2727`) .. _v1_0_a29: From 6aaed2d9b5996151014f45cc7257a6b2c2c434b8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 15:28:13 -0700 Subject: [PATCH 087/156] 2-space indent for HTML version of special JSON views PR #2739 --- datasette/views/special.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/views/special.py b/datasette/views/special.py index 31e7f0c2..6c82983c 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -67,7 +67,7 @@ class JsonDataView(BaseView): context = { "filename": self.filename, "data": data, - "data_json": json.dumps(data, indent=4, default=repr), + "data_json": json.dumps(data, indent=2, default=repr), } # Add has_debug_permission if this view requires permissions-debug if self.permission == "permissions-debug": From f3a34c5012e4a2013bc4995a8fc6eda0b2cc9601 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 15:42:49 -0700 Subject: [PATCH 088/156] Enable root permissions for latest.datasette.io Refs #2740 --- .github/workflows/deploy-latest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 18c01fdc..7d8dd37d 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -116,7 +116,7 @@ jobs: --plugins-dir=plugins \ --branch=$GITHUB_SHA \ --version-note=$GITHUB_SHA \ - --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \ + --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \ --install 'datasette-ephemeral-tables>=0.2.2' \ --service "datasette-latest$SUFFIX" \ --secret $LATEST_DATASETTE_SECRET From f403ea4e53c50319967796128ba73c54fba05edd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 16:47:49 -0700 Subject: [PATCH 089/156] No need to alias description as description --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 8f585cb1..2b8f5eb2 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1942,7 +1942,7 @@ This example returns a SQL fragment that searches rows from a ``dashboards`` tab SELECT 'dashboard' AS type, slug AS label, - description AS description, + description, json_object( 'method', 'row', 'database', 'content', From 6cafdcb6fa57cabe5352a471b6d4b469072beb72 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 21:18:50 -0700 Subject: [PATCH 090/156] Added missing issue reference --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 65530452..329b4769 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,7 @@ The "Jump to" menu, activated by hitting ``/`` or through the application menu, - New "Jump to..." menu item, always visible, for triggering the previously undocumented ``/`` menu. (:issue:`2725`) - The ``/`` jump-to search interface now covers databases, views, canned queries and plugin-provided items in addition to tables. The endpoint backing it has been renamed from ``/-/tables`` to ``/-/jump``. -- New :ref:`plugin_hook_jump_items_sql` plugin hook, allowing plugins to contribute additional items to the jump-to menu by returning SQL. ``JumpSQL`` queries run against Datasette's internal database by default, or can target another database using the optional ``database=`` argument. +- New :ref:`plugin_hook_jump_items_sql` plugin hook, allowing plugins to contribute additional items to the jump-to menu by returning SQL. ``JumpSQL`` queries run against Datasette's internal database by default, or can target another database using the optional ``database=`` argument. (:issue:`2731`) - ``datasette.jump.JumpSQL.menu_item()`` is a shortcut for adding individual jump menu items that are not backed by resources in the internal catalog. - New :ref:`javascript_plugins_makeJumpSections` JavaScript plugin hook, allowing plugins to add custom blank-state sections to the jump-to menu before the user has typed a query. - Debug menu links now appear in the jump-to menu instead of the top-right app menu, with descriptions for each debug item. From a855a1acec758d18b5e3b217e03c3534e979157a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 22:29:49 -0700 Subject: [PATCH 091/156] Database.analyze_sql(sql) method Experimental, we may need this for the upcoming canned query work so that we can tell if a user should be able to save a writable canned query by confirming they have the right permissions to update the affected tables. Refs #2735 --- datasette/database.py | 8 ++ datasette/utils/sql_analysis.py | 99 ++++++++++++++++ tests/test_internals_database.py | 48 ++++++++ tests/test_utils_sql_analysis.py | 195 +++++++++++++++++++++++++++++++ 4 files changed, 350 insertions(+) create mode 100644 datasette/utils/sql_analysis.py create mode 100644 tests/test_utils_sql_analysis.py diff --git a/datasette/database.py b/datasette/database.py index 66d50ffa..e7e9527e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -25,6 +25,7 @@ from .utils import ( table_columns, table_column_details, ) +from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables from .utils.sqlite import sqlite_version from .inspect import inspect_hash @@ -301,6 +302,13 @@ class Database: # Threaded mode - send to write thread return await self._send_to_write_thread(fn, isolated_connection=True) + async def analyze_sql(self, sql, params=None) -> SQLAnalysis: + self._check_not_closed() + + return await self.execute_isolated_fn( + lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name) + ) + async def execute_write_fn(self, fn, block=True, transaction=True, request=None): self._check_not_closed() pending_events = [] diff --git a/datasette/utils/sql_analysis.py b/datasette/utils/sql_analysis.py new file mode 100644 index 00000000..b5317b62 --- /dev/null +++ b/datasette/utils/sql_analysis.py @@ -0,0 +1,99 @@ +from dataclasses import dataclass +from typing import Literal + +from datasette.utils.sqlite import sqlite3 + +SQLTableOperation = Literal["read", "insert", "update", "delete"] + + +@dataclass(frozen=True) +class SQLTableAccess: + operation: SQLTableOperation + database: str | None + table: str + sqlite_schema: str | None + columns: tuple[str, ...] = () + source: str | None = None + + +@dataclass(frozen=True) +class SQLAnalysis: + table_accesses: tuple[SQLTableAccess, ...] + + +_ACTION_TO_OPERATION: dict[int, SQLTableOperation] = { + sqlite3.SQLITE_READ: "read", + sqlite3.SQLITE_INSERT: "insert", + sqlite3.SQLITE_UPDATE: "update", + sqlite3.SQLITE_DELETE: "delete", +} + + +def analyze_sql_tables( + conn, + sql: str, + params=None, + *, + database_name: str | None = None, + schema_to_database: dict[str, str] | None = None, +) -> SQLAnalysis: + """ + Return tables accessed by a SQL statement according to SQLite's authorizer. + + This function is synchronous and connection-based. It temporarily installs a + SQLite authorizer, prepares ``EXPLAIN ``, and returns the table access + callbacks observed while SQLite compiles the statement. + """ + accesses: dict[ + tuple[SQLTableOperation, str | None, str, str | None, str | None], set[str] + ] = {} + + def database_for_schema(sqlite_schema): + if schema_to_database and sqlite_schema in schema_to_database: + return schema_to_database[sqlite_schema] + if sqlite_schema == "main" and database_name is not None: + return database_name + return sqlite_schema + + def authorizer(action, arg1, arg2, sqlite_schema, source): + operation = _ACTION_TO_OPERATION.get(action) + if operation is None or arg1 is None: + return sqlite3.SQLITE_OK + + key = ( + operation, + database_for_schema(sqlite_schema), + arg1, + sqlite_schema, + source, + ) + columns = accesses.setdefault(key, set()) + if operation in ("read", "update") and arg2 is not None: + columns.add(arg2) + return sqlite3.SQLITE_OK + + conn.set_authorizer(authorizer) + try: + conn.execute("EXPLAIN " + sql, params if params is not None else {}).fetchall() + finally: + conn.set_authorizer(None) + + return SQLAnalysis( + table_accesses=tuple( + SQLTableAccess( + operation=operation, + database=database, + table=table, + sqlite_schema=sqlite_schema, + columns=tuple(sorted(columns)), + source=source, + ) + for ( + operation, + database, + table, + sqlite_schema, + source, + ), columns in accesses.items() + ) + ) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 75ae8d39..5481a398 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -688,6 +688,54 @@ async def test_execute_isolated(db, disable_threads): assert not await db.execute_isolated_fn(table_exists_checker("created_by_isolated")) +@pytest.mark.asyncio +async def test_analyze_sql(): + ds = Datasette(memory=True) + db = ds.add_memory_database("test_analyze_sql", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + + analysis = await db.analyze_sql("select name from dogs where id = ?", (1,)) + + assert [ + ( + access.operation, + access.database, + access.sqlite_schema, + access.table, + access.columns, + access.source, + ) + for access in analysis.table_accesses + ] == [ + ("read", "data", "main", "dogs", ("id", "name"), None), + ] + + +@pytest.mark.asyncio +async def test_analyze_sql_insert_select(): + ds = Datasette(memory=True) + db = ds.add_memory_database("test_analyze_sql_insert_select", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write("create table cats (id integer primary key, name text)") + + analysis = await db.analyze_sql("insert into dogs (name) select name from cats") + + assert { + ( + access.operation, + access.database, + access.sqlite_schema, + access.table, + access.columns, + access.source, + ) + for access in analysis.table_accesses + } == { + ("insert", "data", "main", "dogs", (), None), + ("read", "data", "main", "cats", ("name",), None), + } + + @pytest.mark.asyncio async def test_mtime_ns(db): assert isinstance(db.mtime_ns, int) diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py new file mode 100644 index 00000000..c82fb04f --- /dev/null +++ b/tests/test_utils_sql_analysis.py @@ -0,0 +1,195 @@ +import pytest + +from datasette.utils.sqlite import sqlite3 +from datasette.utils.sql_analysis import analyze_sql_tables + + +@pytest.fixture +def conn(): + conn = sqlite3.connect(":memory:") + conn.executescript(""" + create table dogs (id integer primary key, name text, age integer); + create table cats (id integer primary key, name text); + create table log (message text); + create view dog_names as select id, name from dogs; + create trigger dogs_after_insert after insert on dogs begin + update cats set name = new.name where id = new.id; + insert into log (message) values (new.name); + end; + create trigger dog_names_instead_of_update instead of update on dog_names begin + update dogs set name = new.name where id = old.id; + end; + """) + try: + yield conn + finally: + conn.close() + + +def as_tuples(analysis): + return [ + ( + access.operation, + access.database, + access.sqlite_schema, + access.table, + access.columns, + access.source, + ) + for access in analysis.table_accesses + ] + + +def test_analyze_select_tables(conn): + analysis = analyze_sql_tables( + conn, + "select dogs.name, cats.name from dogs join cats on dogs.id = cats.id where dogs.age > ?", + (2,), + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("read", "data", "main", "cats", ("id", "name"), None), + ("read", "data", "main", "dogs", ("age", "id", "name"), None), + } + + +def test_analyze_uses_sqlite_schema_as_default_database(conn): + analysis = analyze_sql_tables(conn, "select name from dogs") + + assert set(as_tuples(analysis)) == { + ("read", "main", "main", "dogs", ("name",), None), + } + + +def test_analyze_insert_tables(conn): + analysis = analyze_sql_tables( + conn, + "insert into dogs (name, age) values (:name, :age)", + {"name": "Cleo", "age": 4}, + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("insert", "data", "main", "dogs", (), None), + ("read", "data", "main", "dogs", ("id", "name"), "dogs_after_insert"), + ("update", "data", "main", "cats", ("name",), "dogs_after_insert"), + ("read", "data", "main", "cats", ("id",), "dogs_after_insert"), + ("insert", "data", "main", "log", (), "dogs_after_insert"), + } + + +def test_analyze_update_tables(conn): + analysis = analyze_sql_tables( + conn, + "update dogs set age = age + 1 where name = ?", + ("Cleo",), + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("update", "data", "main", "dogs", ("age",), None), + ("read", "data", "main", "dogs", ("age", "name"), None), + } + + +def test_analyze_delete_tables(conn): + analysis = analyze_sql_tables( + conn, + "delete from dogs where name = ?", + ("Cleo",), + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("delete", "data", "main", "dogs", (), None), + ("read", "data", "main", "dogs", ("name",), None), + } + + +def test_analyze_insert_select_with_cte(conn): + analysis = analyze_sql_tables( + conn, + """ + with old_dogs as ( + select name from dogs where age > :age + ) + insert into cats (name) + select name from old_dogs + """, + {"age": 10}, + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("insert", "data", "main", "cats", (), None), + ("read", "data", "main", "dogs", ("age", "name"), "old_dogs"), + } + + +def test_analyze_view_with_instead_of_trigger(conn): + analysis = analyze_sql_tables( + conn, + "update dog_names set name = :name where id = :id", + {"name": "Zelda", "id": 1}, + database_name="data", + ) + + assert set(as_tuples(analysis)) == { + ("update", "data", "main", "dog_names", ("name",), None), + ("read", "data", "main", "dogs", ("id", "name"), "dog_names"), + ("read", "data", "main", "dog_names", ("id", "name"), "dog_names"), + ( + "read", + "data", + "main", + "dog_names", + ("id", "name"), + "dog_names_instead_of_update", + ), + ("update", "data", "main", "dogs", ("name",), "dog_names_instead_of_update"), + ("read", "data", "main", "dogs", ("id",), "dog_names_instead_of_update"), + } + + +def test_analyze_attached_database_tables(conn): + conn.execute("attach database ':memory:' as extra") + conn.execute("create table extra.people (id integer primary key, name text)") + + analysis = analyze_sql_tables( + conn, + "insert into extra.people (name) select name from dogs", + database_name="data", + schema_to_database={"extra": "extra_db"}, + ) + + assert set(as_tuples(analysis)) == { + ("insert", "extra_db", "extra", "people", (), None), + ("read", "data", "main", "dogs", ("name",), None), + } + + +def test_analyze_invalid_sql_cleans_up_authorizer(conn): + with pytest.raises(sqlite3.OperationalError): + analyze_sql_tables(conn, "insert into missing_table values (1)") + + conn.execute("select name from dogs").fetchall() + + +def test_analyze_clears_authorizer_on_error(): + class FakeConnection: + def __init__(self): + self.authorizers = [] + + def set_authorizer(self, authorizer): + self.authorizers.append(authorizer) + + def execute(self, sql, params): + raise sqlite3.OperationalError("bad SQL") + + conn = FakeConnection() + + with pytest.raises(sqlite3.OperationalError): + analyze_sql_tables(conn, "bad SQL") + + assert conn.authorizers[-1] is None From daeeca6c6b8f1e46cde33f8ee84b6d73b4478b86 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 22:35:18 -0700 Subject: [PATCH 092/156] Plan internal query storage and management Refs #2735 --- queries-plan.md | 440 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 440 insertions(+) create mode 100644 queries-plan.md diff --git a/queries-plan.md b/queries-plan.md new file mode 100644 index 00000000..283ca866 --- /dev/null +++ b/queries-plan.md @@ -0,0 +1,440 @@ +# Queries in the internal database + +Plan for . + +## Goal + +Move named query definitions into Datasette's internal database, so hundreds or thousands of queries can be listed, searched, permission-filtered, managed, and executed efficiently. + +Terminology change: these are now "queries", not "canned queries". Legacy code and documentation can mention the old name only when describing compatibility or migration. + +## Decisions so far + +- Internal table name: `queries`. +- Query definitions should use real columns, not a JSON blob for all options. +- Query parameter names live in a `parameters` text column as a JSON array. No default values for parameters in this pass. +- No `queries_database_published_idx` index. +- User-created queries require `execute-sql` and `insert-query` on the database. Writable queries additionally require matching table write permissions discovered by `Database.analyze_sql()`. +- `publish-query` is the permission for creating or updating a query so users without `execute-sql` can execute it. +- Add `update-query` and `delete-query`, so administrators can manage queries created by other users. +- Remove the old `canned_queries()` hook from core. If we want compatibility later, build a separate `datasette-old-canned-queries` plugin. +- Writable user-created queries can be supported using `Database.analyze_sql()`, provided we fail closed when analysis cannot prove the required permissions. + +## Current shape + +- Query definitions currently come from `datasette.yaml` or the `canned_queries()` plugin hook. +- `Datasette.get_canned_queries(database_name, actor)` calls that hook every time it needs query definitions. +- `QueryResource.resources_sql()` currently enumerates databases and calls the hook for each one, because permissions and `/-/jump` need query resources. +- Query pages execute if the actor has `view-query` for `QueryResource(database, query)`. +- Arbitrary SQL executes if the actor has `execute-sql` for `DatabaseResource(database)`. + +The main performance and architecture win is making query resource enumeration a direct SQL query against the internal database. + +## Proposed internal schema + +Start with one `queries` table. + +```sql +CREATE TABLE IF NOT EXISTS queries ( + database_name TEXT NOT NULL, + name TEXT NOT NULL, + sql TEXT NOT NULL, + title TEXT, + description TEXT, + description_html TEXT, + hide_sql INTEGER NOT NULL DEFAULT 0 CHECK (hide_sql IN (0, 1)), + fragment TEXT, + parameters TEXT NOT NULL DEFAULT '[]', + is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), + published INTEGER NOT NULL DEFAULT 0 CHECK (published IN (0, 1)), + source TEXT NOT NULL DEFAULT 'user', + owner_id TEXT, + on_success_message TEXT, + on_success_message_sql TEXT, + on_success_redirect TEXT, + on_error_message TEXT, + on_error_redirect TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (database_name, name), + CHECK (is_write = 0 OR published = 0) +); + +CREATE INDEX IF NOT EXISTS queries_owner_idx + ON queries(owner_id); +``` + +Column notes: + +- `database_name`, `name`, and `sql` are the routing and execution core. +- Display fields become columns: `title`, `description`, `description_html`, `hide_sql`, and `fragment`. +- `parameters` is a JSON array of parameter names, stored as text. This preserves explicit parameter order, but does not support labels or default values. +- Existing writable query behavior gets columns too: `is_write`, success/error messages, success/error redirects, and `on_success_message_sql`. +- `published` only applies to read-only queries. A writable query can still be public through explicit `view-query` permissions, but the "publish for users without execute-sql" shortcut should be read-only. +- `source` distinguishes `user`, `config`, and `plugin` rows. +- `owner_id` is the actor id for user-created rows. It is `NULL` for config/plugin rows. + +No separate index is needed on `(database_name, name)` because the primary key already creates one. Do not add a `queries_database_published_idx` index for now. + +`QueryResource.resources_sql()` can become: + +```sql +SELECT q.database_name AS parent, q.name AS child +FROM queries q +JOIN catalog_databases cd ON cd.database_name = q.database_name +``` + +The join keeps persisted queries for detached databases from appearing as live resources. + +## Config and plugin migration + +`datasette.yaml` can continue to support `databases: {db}: queries:` blocks, but core should import them directly into the internal `queries` tables at startup: + +1. Ensure the internal schema exists. +2. Delete previous `source='config'` rows. +3. Read configured query blocks for each live database. +4. Normalize string definitions to `{"sql": ...}`. +5. Insert rows into `queries`, storing explicit `params` as JSON in `parameters`. + +Plugins should move to: + +```python +await datasette.add_query(...) +await datasette.remove_query(...) +``` + +Remove the old `canned_queries()` hookspec and all core calls to it. If compatibility is needed, build `datasette-old-canned-queries` later as a plugin that restores the hook and imports old hook results using `datasette.add_query()`. + +## Permission model + +Add core actions: + +- `insert-query`, database-level, for creating queries in a database. +- `publish-query`, database-level, for marking read-only queries as executable by actors who lack `execute-sql`. +- `update-query`, query-level, for modifying existing query definitions. +- `delete-query`, query-level, for deleting existing query definitions. + +User-created query creation requires: + +- `execute-sql` on `DatabaseResource(database)` +- `insert-query` on `DatabaseResource(database)` +- If analysis shows the query is writable, the table-level write permissions described in the writable query section. + +Setting `published=1` requires: + +- `publish-query` on `DatabaseResource(database)` +- The query must be read-only according to `Database.analyze_sql()`. + +Updating an existing query requires: + +- `update-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. +- If the SQL changes, also require `execute-sql` on the database. +- If the changed SQL is writable, also require the table-level write permissions described in the writable query section. +- If `published` changes from `0` to `1`, also require `publish-query` on the database. + +Deleting an existing query requires: + +- `delete-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. + +Default owner permissions: + +- For `source='user' AND owner_id = actor.id`, grant `update-query` and `delete-query`. +- Do not automatically grant execution if the user no longer has the execution permission described below. + +## Executing queries + +Default execution rule for read-only queries: + +- If `published=0`, the actor needs `execute-sql` on the database. +- If `published=1`, the actor can execute the query without `execute-sql`. + +Default execution rule for user-created writable queries: + +- `published` must be `0`. +- The actor must have `view-query`. +- The actor must currently have every write permission required by fresh `Database.analyze_sql()` results for the query SQL. + +Implementation: + +- Remove `view-query` from the broad `DEFAULT_ALLOW_ACTIONS` set. +- Replace it with query-aware default `view-query` permission SQL. +- For `published=1 AND is_write=0`, emit a child-level `view-query` allow. +- For `published=0 AND is_write=0`, emit child-level `view-query` allows for queries whose parent database is in the actor's `execute-sql` allowed resources. +- For `is_write=1 AND source='user'`, emit `view-query` only for the owner or actors with explicit `view-query` permission, then have `QueryView` perform the fresh analysis/table-permission check before execution. +- For trusted writable queries, preserve current behavior by emitting child-level `view-query` allows for `is_write=1 AND source IN ('config', 'plugin')` when Datasette is not running with `--default-deny`. + +For read-only queries this keeps `QueryView` simple: it checks `view-query` for the query resource, and the default permission hook encodes the relationship with `execute-sql`. User-created writable queries need one additional runtime permission check because their required table permissions are derived from fresh SQL analysis. + +Explicit deny rules should still be able to block a published query. + +## Writable queries + +Writable user-created queries should be in scope, guarded by `Database.analyze_sql()`. + +The secure rule: a user can create, update, or execute a writable user-created query only if they currently have the corresponding write permissions for every table the SQL can affect. + +`Database.analyze_sql(sql, params=None)` runs the SQL through SQLite's authorizer on an isolated connection and returns a `SQLAnalysis` object containing `SQLTableAccess` rows: + +- `operation`: `read`, `insert`, `update`, or `delete` +- `database`: Datasette database name for `main`, or SQLite schema name where no Datasette mapping exists +- `table`: affected table or view +- `columns`: read/updated columns where SQLite reports them +- `source`: trigger/view/CTE source when SQLite reports one + +Validation flow for user-created queries: + +1. Derive named parameters from the SQL and pass harmless placeholder values into `db.analyze_sql()` so SQLite can prepare statements with bindings. +2. If analysis raises a SQLite error, reject the query. +3. If every table access is `read`, treat the query as read-only and require `execute-sql` plus `insert-query`/`update-query` as described above. +4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `published=0`. +5. Reject writable user-created queries that access a database other than the database they are being saved against, until `analyze_sql()` can reliably map attached SQLite schemas back to Datasette database names. +6. For every write access returned by analysis, require the corresponding permission on `TableResource(access.database, access.table)`: + - `insert` -> `insert-row` + - `update` -> `update-row` + - `delete` -> `delete-row` +7. Include write accesses reported from triggers and views, since those are real side effects. +8. Re-run the same analysis and permission checks when SQL changes through `update_query()` or `POST .../-/update`. +9. Re-run analysis before executing user-created writable queries, so schema or trigger changes cannot leave a previously saved query with stale permission assumptions. + +The user-facing API should not trust a submitted `is_write` value. It should derive `is_write` from analysis. + +Trusted configuration and plugin code can still call `datasette.add_query(..., is_write=True, ...)`. Those are treated as deployment/admin-authored queries. They keep the existing execution model: they require `view-query`, and the default `view-query` hook should preserve current default-open behavior for trusted writable queries while still respecting `--default-deny`. + +Fail closed cases for user-created writable queries: + +- Analysis fails. +- Analysis reports any write operation that cannot be mapped to a Datasette table resource. +- Analysis reports writes outside the target database. +- The actor lacks any required table write permission. +- `published=1` is requested. + +This gives us writable user-created queries without letting `execute-sql` alone become a path to create arbitrary write endpoints. + +## HTTP API sketch + +JSON endpoints should follow Datasette's existing write API style: use `POST` plus action paths such as `/-/insert`, `/-/update`, and `/-/delete`, not HTTP `PATCH` or `DELETE`. + +Endpoints: + +- `GET /{database}/-/queries` lists query definitions the actor can view or manage, probably paginated. +- `POST /{database}/-/queries/-/insert` creates a query. +- `GET /{database}/{query}/-/definition` returns one query definition without executing it. +- `POST /{database}/{query}/-/update` updates one query. +- `POST /{database}/{query}/-/delete` deletes one query. + +Create request: + +```json +{ + "query": { + "name": "top_customers", + "sql": "select * from customers order by revenue desc limit 20", + "title": "Top customers", + "description": "Highest revenue customers", + "published": false, + "parameters": ["region"] + } +} +``` + +Successful create returns `201` and the created query definition: + +```json +{ + "ok": true, + "query": { + "database": "fixtures", + "name": "top_customers", + "sql": "select * from customers order by revenue desc limit 20", + "title": "Top customers", + "description": "Highest revenue customers", + "published": false, + "parameters": ["region"] + } +} +``` + +Update request, imitating `RowUpdateView`: + +```json +{ + "update": { + "title": "Top customers by revenue", + "published": true + }, + "return": true +} +``` + +Successful update returns `{"ok": true}` by default. With `"return": true`, return the updated query definition: + +```json +{ + "ok": true, + "query": { + "database": "fixtures", + "name": "top_customers", + "sql": "select * from customers order by revenue desc limit 20", + "title": "Top customers by revenue", + "published": true + } +} +``` + +Delete request: + +```http +POST /{database}/{query}/-/delete +Content-Type: application/json +``` + +Successful delete returns: + +```json +{ + "ok": true +} +``` + +Validation: + +- Update bodies must be dictionaries containing an `update` dictionary, with optional `return`; invalid keys return `{"ok": false, "errors": [...]}`. +- Validate route-safe query names. +- Reject names that collide with a table or view in the same database, since table routes currently win over query routes. +- Analyze user-created SQL with `Database.analyze_sql()`. +- Use `validate_sql_select(sql)` as the read-only fast path when analysis shows only reads, but do not require it for writable queries that pass analysis and permission checks. +- Reject magic parameters such as `:_actor_id`, `:_cookie_*`, and `:_header_*` for user-created queries. +- Reject client-supplied `is_write`; derive it from analysis. +- Reject writable-only success/error fields for read-only queries. + +## Python API sketch + +Add methods on `Datasette`: + +```python +await datasette.add_query( + database, + name, + sql, + title=None, + description=None, + description_html=None, + hide_sql=False, + fragment=None, + parameters=None, + is_write=False, + published=False, + source="plugin", + owner_id=None, + on_success_message=None, + on_success_message_sql=None, + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + replace=True, +) + +await datasette.update_query( + database, + name, + *, + sql=UNCHANGED, + title=UNCHANGED, + description=UNCHANGED, + description_html=UNCHANGED, + hide_sql=UNCHANGED, + fragment=UNCHANGED, + parameters=UNCHANGED, + is_write=UNCHANGED, + published=UNCHANGED, + source=UNCHANGED, + owner_id=UNCHANGED, + on_success_message=UNCHANGED, + on_success_message_sql=UNCHANGED, + on_success_redirect=UNCHANGED, + on_error_message=UNCHANGED, + on_error_redirect=UNCHANGED, +) + +await datasette.remove_query(database, name, source=None) + +await datasette.get_query(database, name) +await datasette.get_queries(database) +``` + +`update_query()` should use an internal sentinel default such as `UNCHANGED = object()` so callers can distinguish "leave this column alone" from "set this column to `NULL`": + +```python +await datasette.update_query( + "fixtures", + "top_customers", + on_success_redirect=None, +) +``` + +That call should set `on_success_redirect` to SQL `NULL`; omitting `on_success_redirect` should leave the existing value unchanged. + +Implementation detail: build the `UPDATE` statement dynamically from fields whose value is not `UNCHANGED`, validate non-nullable fields before writing, and update `updated_at` whenever at least one field changes. + +The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `published`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. + +## Query page save UI + +On `/{database}/-/query`, if the actor has both `execute-sql` and `insert-query`, show a save control for valid read-only SQL. That page already executes read-only arbitrary SQL, so the first UI can stay read-only even though the JSON API can accept writable SQL after `Database.analyze_sql()` validation. + +The save form should call `POST /{database}/-/queries/-/insert` and default to `published=false`. + +If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. + +## Dedicated create query UI + +Add `/{database}/-/queries/-/create` for the fuller query authoring flow, including writable queries. + +This page should require `execute-sql` and `insert-query` to access. It should provide a SQL editor and a mode control: + +- Read-only +- Writable + +Read-only mode can share the same fields as the arbitrary SQL save flow: name, title, description, parameters, and optional published status if the actor has `publish-query`. + +Writable mode should always run `Database.analyze_sql()` and show an analysis panel before saving: + +- detected operation +- database and table +- required permission +- whether the actor has that permission +- source, when the operation comes from a trigger or view + +The Save button should be disabled until analysis succeeds and every required table write permission is allowed. Writable mode should not show a publish control, because user-created writable queries cannot be published. + +The existing edit-SQL flow from query pages can continue to point back to arbitrary SQL. A later enhancement can add "update this query" when the actor owns it or has `update-query`. + +## Test plan + +- Internal schema creates `queries`. +- Query parameters are stored in the `queries.parameters` text column as a JSON array of names. +- Config `queries:` blocks import into internal tables. +- Legacy string query definitions normalize to SQL rows. +- The old `canned_queries()` hook is no longer called by core. +- `QueryResource.resources_sql()` returns rows from `queries`. +- Database page and `/-/jump` list queries from the internal DB. +- `view-query` is no longer globally default-allowed; default query permissions come from the query-aware hook. +- Unpublished read-only query requires `execute-sql` to execute. +- Published read-only query can be executed without `execute-sql`. +- Setting `published=true` requires `publish-query`. +- User-created query requires both `execute-sql` and `insert-query`. +- User-created writable query creation uses `Database.analyze_sql()` and requires matching `insert-row`, `update-row`, and/or `delete-row` permissions for every reported write access. +- `/{database}/-/queries/-/create` provides the writable-query authoring UI with an analysis panel and disabled save until all required write permissions pass. +- User-created writable query execution re-runs `Database.analyze_sql()` and re-checks table write permissions. +- User-created writable query cannot be published. +- Query update uses `POST /{database}/{query}/-/update` with an `{"update": {...}}` body. +- Query delete uses `POST /{database}/{query}/-/delete`. +- There are no `PATCH` or HTTP `DELETE` routes for query management. +- `datasette.update_query(..., field=None)` writes `NULL`, while omitted fields are left unchanged. +- Owner gets default `update-query` and `delete-query` for their own user-created rows. +- Admin can manage other users' queries with `update-query` and `delete-query`. +- User API rejects magic parameters. +- User API rejects writable queries if analysis fails, reports writes outside the target database, or reports writes the actor is not allowed to perform. +- Trusted config/plugin writable queries still execute through `view-query`. +- Trusted config/plugin writable queries are not default-allowed under `--default-deny`. +- Persisted internal DB does not expose queries for detached databases. From 7e1abd0da4cc57be64685b6ea717457ceefbabb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 22:37:34 -0700 Subject: [PATCH 093/156] Add internal query storage APIs Refs #2735 --- datasette/app.py | 200 +++++++++++++++++++++++++++++++++ datasette/utils/internal_db.py | 28 +++++ tests/test_queries.py | 123 ++++++++++++++++++++ 3 files changed, 351 insertions(+) create mode 100644 tests/test_queries.py diff --git a/datasette/app.py b/datasette/app.py index 75f05d88..518215fd 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -268,6 +268,7 @@ DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS} FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" DEFAULT_NOT_SET = object() +UNCHANGED = object() ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) @@ -1007,6 +1008,205 @@ class Datasette: [database_name, resource_name, column_name, key, value], ) + @staticmethod + def _query_row_to_dict(row): + if row is None: + return None + parameters = json.loads(row["parameters"] or "[]") + is_write = bool(row["is_write"]) + return { + "database": row["database_name"], + "name": row["name"], + "sql": row["sql"], + "title": row["title"], + "description": row["description"], + "description_html": row["description_html"], + "hide_sql": bool(row["hide_sql"]), + "fragment": row["fragment"], + "params": parameters, + "parameters": parameters, + "is_write": is_write, + "write": is_write, + "published": bool(row["published"]), + "source": row["source"], + "owner_id": row["owner_id"], + "on_success_message": row["on_success_message"], + "on_success_message_sql": row["on_success_message_sql"], + "on_success_redirect": row["on_success_redirect"], + "on_error_message": row["on_error_message"], + "on_error_redirect": row["on_error_redirect"], + } + + async def add_query( + self, + database, + name, + sql, + *, + title=None, + description=None, + description_html=None, + hide_sql=False, + fragment=None, + parameters=None, + is_write=False, + published=False, + source="plugin", + owner_id=None, + on_success_message=None, + on_success_message_sql=None, + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + replace=True, + ): + parameters_json = json.dumps(list(parameters or [])) + sql_statement = """ + INSERT INTO queries ( + database_name, name, sql, title, description, description_html, + hide_sql, fragment, parameters, is_write, published, source, + owner_id, on_success_message, on_success_message_sql, + on_success_redirect, on_error_message, on_error_redirect + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + if replace: + sql_statement += """ + ON CONFLICT(database_name, name) DO UPDATE SET + sql = excluded.sql, + title = excluded.title, + description = excluded.description, + description_html = excluded.description_html, + hide_sql = excluded.hide_sql, + fragment = excluded.fragment, + parameters = excluded.parameters, + is_write = excluded.is_write, + published = excluded.published, + source = excluded.source, + owner_id = excluded.owner_id, + on_success_message = excluded.on_success_message, + on_success_message_sql = excluded.on_success_message_sql, + on_success_redirect = excluded.on_success_redirect, + on_error_message = excluded.on_error_message, + on_error_redirect = excluded.on_error_redirect, + updated_at = CURRENT_TIMESTAMP + """ + await self.get_internal_database().execute_write( + sql_statement, + [ + database, + name, + sql, + title, + description, + description_html, + int(bool(hide_sql)), + fragment, + parameters_json, + int(bool(is_write)), + int(bool(published)), + source, + owner_id, + on_success_message, + on_success_message_sql, + on_success_redirect, + on_error_message, + on_error_redirect, + ], + ) + + async def update_query( + self, + database, + name, + *, + sql=UNCHANGED, + title=UNCHANGED, + description=UNCHANGED, + description_html=UNCHANGED, + hide_sql=UNCHANGED, + fragment=UNCHANGED, + parameters=UNCHANGED, + is_write=UNCHANGED, + published=UNCHANGED, + source=UNCHANGED, + owner_id=UNCHANGED, + on_success_message=UNCHANGED, + on_success_message_sql=UNCHANGED, + on_success_redirect=UNCHANGED, + on_error_message=UNCHANGED, + on_error_redirect=UNCHANGED, + ): + fields = { + "sql": sql, + "title": title, + "description": description, + "description_html": description_html, + "hide_sql": hide_sql, + "fragment": fragment, + "parameters": parameters, + "is_write": is_write, + "published": published, + "source": source, + "owner_id": owner_id, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + updates = [] + params = [] + for field, value in fields.items(): + if value is UNCHANGED: + continue + if field in {"hide_sql", "is_write", "published"}: + value = int(bool(value)) + elif field == "parameters": + value = json.dumps(list(value or [])) + updates.append(f"{field} = ?") + params.append(value) + if not updates: + return + updates.append("updated_at = CURRENT_TIMESTAMP") + params.extend([database, name]) + await self.get_internal_database().execute_write( + """ + UPDATE queries + SET {} + WHERE database_name = ? AND name = ? + """.format(", ".join(updates)), + params, + ) + + async def remove_query(self, database, name, source=None): + sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" + params = [database, name] + if source is not None: + sql += " AND source = ?" + params.append(source) + await self.get_internal_database().execute_write(sql, params) + + async def get_query(self, database, name): + rows = await self.get_internal_database().execute( + """ + SELECT * FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + return self._query_row_to_dict(rows.first()) + + async def get_queries(self, database): + rows = await self.get_internal_database().execute( + """ + SELECT * FROM queries + WHERE database_name = ? + ORDER BY name + """, + [database], + ) + return {row["name"]: self._query_row_to_dict(row) for row in rows} + # Column types API async def _get_resource_column_details(self, database: str, resource: str): diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index df149928..9008c083 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -112,6 +112,34 @@ async def initialize_metadata_tables(db): config TEXT, PRIMARY KEY (database_name, resource_name, column_name) ); + + CREATE TABLE IF NOT EXISTS queries ( + database_name TEXT NOT NULL, + name TEXT NOT NULL, + sql TEXT NOT NULL, + title TEXT, + description TEXT, + description_html TEXT, + hide_sql INTEGER NOT NULL DEFAULT 0 CHECK (hide_sql IN (0, 1)), + fragment TEXT, + parameters TEXT NOT NULL DEFAULT '[]', + is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), + published INTEGER NOT NULL DEFAULT 0 CHECK (published IN (0, 1)), + source TEXT NOT NULL DEFAULT 'user', + owner_id TEXT, + on_success_message TEXT, + on_success_message_sql TEXT, + on_success_redirect TEXT, + on_error_message TEXT, + on_error_redirect TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (database_name, name), + CHECK (is_write = 0 OR published = 0) + ); + + CREATE INDEX IF NOT EXISTS queries_owner_idx + ON queries(owner_id); """)) diff --git a/tests/test_queries.py b/tests/test_queries.py new file mode 100644 index 00000000..d30fcfe7 --- /dev/null +++ b/tests/test_queries.py @@ -0,0 +1,123 @@ +import pytest + +from datasette.app import Datasette + + +@pytest.mark.asyncio +async def test_queries_internal_table_schema(): + ds = Datasette(memory=True) + await ds.invoke_startup() + internal_db = ds.get_internal_database() + + columns = [ + row["name"] + for row in ( + await internal_db.execute("select name from pragma_table_info('queries')") + ) + ] + + assert columns == [ + "database_name", + "name", + "sql", + "title", + "description", + "description_html", + "hide_sql", + "fragment", + "parameters", + "is_write", + "published", + "source", + "owner_id", + "on_success_message", + "on_success_message_sql", + "on_success_redirect", + "on_error_message", + "on_error_redirect", + "created_at", + "updated_at", + ] + + +@pytest.mark.asyncio +async def test_add_get_and_remove_query(): + ds = Datasette(memory=True) + ds.add_memory_database("query_api", name="data") + await ds.invoke_startup() + + await ds.add_query( + "data", + "top_customers", + "select * from customers where region = :region", + title="Top customers", + description="Customers by region", + hide_sql=True, + fragment="chart", + parameters=["region"], + published=True, + source="user", + owner_id="alice", + ) + + query = await ds.get_query("data", "top_customers") + assert query == { + "database": "data", + "name": "top_customers", + "sql": "select * from customers where region = :region", + "title": "Top customers", + "description": "Customers by region", + "description_html": None, + "hide_sql": True, + "fragment": "chart", + "params": ["region"], + "parameters": ["region"], + "is_write": False, + "write": False, + "published": True, + "source": "user", + "owner_id": "alice", + "on_success_message": None, + "on_success_message_sql": None, + "on_success_redirect": None, + "on_error_message": None, + "on_error_redirect": None, + } + + assert await ds.get_queries("data") == {"top_customers": query} + + await ds.remove_query("data", "top_customers") + assert await ds.get_query("data", "top_customers") is None + assert await ds.get_queries("data") == {} + + +@pytest.mark.asyncio +async def test_update_query_only_updates_provided_fields(): + ds = Datasette(memory=True) + ds.add_memory_database("query_api_update", name="data") + await ds.invoke_startup() + + await ds.add_query( + "data", + "redirect", + "select 1", + title="Original", + on_success_redirect="/original", + parameters=["one"], + ) + + await ds.update_query( + "data", + "redirect", + title="Updated", + parameters=[], + on_success_redirect=None, + ) + + query = await ds.get_query("data", "redirect") + assert query["title"] == "Updated" + assert query["parameters"] == [] + assert query["params"] == [] + assert query["on_success_redirect"] is None + assert query["sql"] == "select 1" + assert query["published"] is False From b4c63966f81599b635a702fd7d971837d01c8bca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 22:40:22 -0700 Subject: [PATCH 094/156] Load saved queries into permission resources Refs #2735 --- datasette/app.py | 51 +++++++++----- datasette/default_permissions/__init__.py | 1 + datasette/default_permissions/defaults.py | 56 ++++++++++++++- datasette/resources.py | 46 ++---------- tests/test_queries.py | 85 +++++++++++++++++++++++ 5 files changed, 179 insertions(+), 60 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 518215fd..d64337d1 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -572,6 +572,35 @@ class Datasette: # TODO(alex) is metadata.json was loaded in, and --internal is not memory, then log # a warning to user that they should delete their metadata.json file + async def apply_queries_config(self): + # Apply configured query entries from datasette.yaml to the internal table. + await self.get_internal_database().execute_write( + "DELETE FROM queries WHERE source = 'config'" + ) + for dbname, db_config in ((self.config or {}).get("databases") or {}).items(): + for query_name, query_config in (db_config.get("queries") or {}).items(): + if not isinstance(query_config, dict): + query_config = {"sql": query_config} + await self.add_query( + dbname, + query_name, + query_config["sql"], + title=query_config.get("title"), + description=query_config.get("description"), + description_html=query_config.get("description_html"), + hide_sql=bool(query_config.get("hide_sql")), + fragment=query_config.get("fragment"), + parameters=query_config.get("params"), + is_write=bool(query_config.get("write")), + published=bool(query_config.get("published")), + source="config", + on_success_message=query_config.get("on_success_message"), + on_success_message_sql=query_config.get("on_success_message_sql"), + on_success_redirect=query_config.get("on_success_redirect"), + on_error_message=query_config.get("on_error_message"), + on_error_redirect=query_config.get("on_error_redirect"), + ) + def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env if request: @@ -732,6 +761,7 @@ class Datasette: await await_me_maybe(hook) # Ensure internal tables and metadata are populated before startup hooks await self._refresh_schemas() + await self.apply_queries_config() # Load column_types from config into internal DB await self._apply_column_types_config() for hook in pm.hook.startup(datasette=self): @@ -1439,27 +1469,10 @@ class Datasette: return self.static_hash("app.css") async def get_canned_queries(self, database_name, actor): - queries = {} - for more_queries in pm.hook.canned_queries( - datasette=self, - database=database_name, - actor=actor, - ): - more_queries = await await_me_maybe(more_queries) - queries.update(more_queries or {}) - # Fix any {"name": "select ..."} queries to be {"name": {"sql": "select ..."}} - for key in queries: - if not isinstance(queries[key], dict): - queries[key] = {"sql": queries[key]} - # Also make sure "name" is available: - queries[key]["name"] = key - return queries + return await self.get_queries(database_name) async def get_canned_query(self, database_name, query_name, actor): - queries = await self.get_canned_queries(database_name, actor) - query = queries.get(query_name) - if query: - return query + return await self.get_query(database_name, query_name) def _prepare_connection(self, conn, database): conn.row_factory = sqlite3.Row diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index 9e3bb648..5a53dbe7 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -35,6 +35,7 @@ from .config import config_permissions_sql as config_permissions_sql from .defaults import ( default_allow_sql_check as default_allow_sql_check, default_action_permissions_sql as default_action_permissions_sql, + default_query_permissions_sql as default_query_permissions_sql, DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, ) diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 4c74219d..2613c4f4 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -21,7 +21,6 @@ DEFAULT_ALLOW_ACTIONS = frozenset( "view-database", "view-database-download", "view-table", - "view-query", "execute-sql", } ) @@ -67,3 +66,58 @@ async def default_action_permissions_sql( return PermissionSQL.allow(reason=reason) return None + + +@hookimpl(specname="permission_resources_sql") +async def default_query_permissions_sql( + datasette: "Datasette", + actor: Optional[dict], + action: str, +) -> Optional[PermissionSQL]: + if action != "view-query": + return None + + execute_sql = await datasette.allowed_resources_sql( + action="execute-sql", actor=actor + ) + sql = execute_sql.sql + params = {} + for key, value in execute_sql.params.items(): + new_key = f"query_execute_sql_{key}" + sql = sql.replace(f":{key}", f":{new_key}") + params[new_key] = value + + trusted_writable_sql = "" + if not datasette.default_deny: + trusted_writable_sql = """ + UNION ALL + SELECT database_name AS parent, name AS child, 1 AS allow, + 'trusted writable query' AS reason + FROM queries + WHERE is_write = 1 + AND source IN ('config', 'plugin') + """ + + return PermissionSQL( + sql=f""" + WITH execute_sql_allowed AS ( + {sql} + ) + SELECT database_name AS parent, name AS child, 1 AS allow, + 'published query' AS reason + FROM queries + WHERE is_write = 0 + AND published = 1 + UNION ALL + SELECT q.database_name AS parent, q.name AS child, 1 AS allow, + 'execute-sql allows query' AS reason + FROM queries q + JOIN execute_sql_allowed es + ON es.parent = q.database_name + AND es.child IS NULL + WHERE q.is_write = 0 + AND q.published = 0 + {trusted_writable_sql} + """, + params=params, + ) diff --git a/datasette/resources.py b/datasette/resources.py index 236b3598..91a46d36 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -41,7 +41,7 @@ class TableResource(Resource): class QueryResource(Resource): - """A canned query in a database.""" + """A saved query in a database.""" name = "query" parent_class = DatabaseResource @@ -51,42 +51,8 @@ class QueryResource(Resource): @classmethod async def resources_sql(cls, datasette, actor=None) -> str: - from datasette.plugins import pm - from datasette.utils import await_me_maybe - - # Get all databases from catalog - db = datasette.get_internal_database() - result = await db.execute("SELECT database_name FROM catalog_databases") - databases = [row[0] for row in result.rows] - - # Gather canned queries for this actor from all databases. - # This keeps allowed_resources("view-query", actor=...) consistent with - # actor-specific canned_queries() implementations. - query_pairs = [] - for database_name in databases: - # Call the hook to get queries (including from config via default plugin) - for queries_result in pm.hook.canned_queries( - datasette=datasette, - database=database_name, - actor=actor, - ): - queries = await await_me_maybe(queries_result) - if queries: - for query_name in queries.keys(): - query_pairs.append((database_name, query_name)) - - # Build SQL - if not query_pairs: - return "SELECT NULL AS parent, NULL AS child WHERE 0" - - # Generate UNION ALL query - selects = [] - for db_name, query_name in query_pairs: - # Escape single quotes by doubling them - db_escaped = db_name.replace("'", "''") - query_escaped = query_name.replace("'", "''") - selects.append( - f"SELECT '{db_escaped}' AS parent, '{query_escaped}' AS child" - ) - - return " UNION ALL ".join(selects) + return """ + SELECT q.database_name AS parent, q.name AS child + FROM queries q + JOIN catalog_databases cd ON cd.database_name = q.database_name + """ diff --git a/tests/test_queries.py b/tests/test_queries.py index d30fcfe7..dcebc2cd 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1,6 +1,7 @@ import pytest from datasette.app import Datasette +from datasette.resources import DatabaseResource, QueryResource @pytest.mark.asyncio @@ -121,3 +122,87 @@ async def test_update_query_only_updates_provided_fields(): assert query["on_success_redirect"] is None assert query["sql"] == "select 1" assert query["published"] is False + + +@pytest.mark.asyncio +async def test_config_queries_imported_to_internal_table(): + ds = Datasette( + memory=True, + config={ + "databases": { + "data": { + "queries": { + "configured": { + "sql": "select :name as name", + "title": "Configured query", + "params": ["name"], + } + } + } + } + }, + ) + ds.add_memory_database("query_config", name="data") + await ds.invoke_startup() + + assert await ds.get_query("data", "configured") == { + "database": "data", + "name": "configured", + "sql": "select :name as name", + "title": "Configured query", + "description": None, + "description_html": None, + "hide_sql": False, + "fragment": None, + "params": ["name"], + "parameters": ["name"], + "is_write": False, + "write": False, + "published": False, + "source": "config", + "owner_id": None, + "on_success_message": None, + "on_success_message_sql": None, + "on_success_redirect": None, + "on_error_message": None, + "on_error_redirect": None, + } + + +@pytest.mark.asyncio +async def test_query_resources_come_from_internal_table(): + ds = Datasette(memory=True) + ds.add_memory_database("query_resources", name="data") + await ds.invoke_startup() + await ds.add_query("data", "internal_query", "select 1", source="user") + + page = await ds.allowed_resources("view-query", actor=None) + + assert [(r.parent, r.child) for r in page.resources] == [ + ("data", "internal_query") + ] + + +@pytest.mark.asyncio +async def test_unpublished_query_requires_execute_sql_but_published_does_not(): + ds = Datasette(memory=True, settings={"default_allow_sql": False}) + ds.add_memory_database("query_permissions", name="data") + await ds.invoke_startup() + await ds.add_query("data", "unpublished", "select 1", published=False) + await ds.add_query("data", "published", "select 1", published=True) + + assert not await ds.allowed( + action="execute-sql", + resource=DatabaseResource("data"), + actor=None, + ) + assert not await ds.allowed( + action="view-query", + resource=QueryResource("data", "unpublished"), + actor=None, + ) + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "published"), + actor=None, + ) From 221be2632e6516e70796c234a1f649018412cdf7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 22:41:56 -0700 Subject: [PATCH 095/156] Add query management actions and write analysis Refs #2735 --- datasette/app.py | 30 +++++++++++++++ datasette/default_actions.py | 26 +++++++++++++ tests/test_queries.py | 73 ++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index d64337d1..9b7fa61e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -93,6 +93,7 @@ from .utils import ( module_from_path, move_plugins_and_allow, move_table_config, + named_parameters, parse_metadata, resolve_env_secrets, resolve_routes, @@ -1237,6 +1238,35 @@ class Datasette: ) return {row["name"]: self._query_row_to_dict(row) for row in rows} + async def ensure_query_write_permissions(self, database, sql, *, actor=None): + write_actions = { + "insert": "insert-row", + "update": "update-row", + "delete": "delete-row", + } + db = self.get_database(database) + params = {name: "" for name in named_parameters(sql)} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise Forbidden(f"Could not analyze query: {ex}") from ex + + for access in analysis.table_accesses: + action = write_actions.get(access.operation) + if action is None: + continue + if access.database != database: + raise Forbidden("Writable queries may not write to attached databases") + if not await self.allowed( + action=action, + resource=TableResource(database=access.database, table=access.table), + actor=actor, + ): + raise Forbidden( + f"Permission denied: need {action} on {access.database}/{access.table}" + ) + return analysis + # Column types API async def _get_resource_column_details(self, database: str, resource: str): diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 149a4e5f..e0e0aee5 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -54,6 +54,20 @@ def register_actions(): description="Create tables", resource_class=DatabaseResource, ), + Action( + name="insert-query", + abbr="iq", + description="Create saved queries", + resource_class=DatabaseResource, + also_requires="execute-sql", + ), + Action( + name="publish-query", + abbr="pq", + description="Publish saved queries for actors without execute-sql", + resource_class=DatabaseResource, + also_requires="insert-query", + ), # Table-level actions (child-level) Action( name="view-table", @@ -104,4 +118,16 @@ def register_actions(): description="View named query results", resource_class=QueryResource, ), + Action( + name="update-query", + abbr="uq", + description="Update saved queries", + resource_class=QueryResource, + ), + Action( + name="delete-query", + abbr="dq", + description="Delete saved queries", + resource_class=QueryResource, + ), ) diff --git a/tests/test_queries.py b/tests/test_queries.py index dcebc2cd..01174a18 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -2,6 +2,7 @@ import pytest from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource +from datasette.utils.asgi import Forbidden @pytest.mark.asyncio @@ -206,3 +207,75 @@ async def test_unpublished_query_requires_execute_sql_but_published_does_not(): resource=QueryResource("data", "published"), actor=None, ) + + +@pytest.mark.asyncio +async def test_query_actions_are_registered(): + ds = Datasette() + await ds.invoke_startup() + + assert ds.get_action("insert-query").resource_class is DatabaseResource + assert ds.get_action("publish-query").resource_class is DatabaseResource + assert ds.get_action("update-query").resource_class is QueryResource + assert ds.get_action("delete-query").resource_class is QueryResource + + +@pytest.mark.asyncio +async def test_analyze_write_query_requires_table_permissions(): + ds = Datasette(memory=True, default_deny=True) + db = ds.add_memory_database("query_write_permissions", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + actor = {"id": "writer"} + await ds.add_query( + "data", + "write_dog", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="writer", + ) + + with pytest.raises(Forbidden): + await ds.ensure_query_write_permissions( + "data", + "insert into dogs (name) values (:name)", + actor=actor, + ) + + ds.config = { + "databases": { + "data": { + "tables": { + "dogs": { + "permissions": { + "insert-row": {"id": "writer"}, + } + } + } + } + } + } + + await ds.ensure_query_write_permissions( + "data", + "insert into dogs (name) values (:name)", + actor=actor, + ) + + +@pytest.mark.asyncio +async def test_analyze_write_query_rejects_writes_to_attached_databases(): + ds = Datasette(memory=True, default_deny=True) + db = ds.add_memory_database("query_attached_writes", name="data") + await db.execute_write("attach database ':memory:' as extra") + await db.execute_write("create table extra.cats (id integer primary key)") + await ds.invoke_startup() + + with pytest.raises(Forbidden): + await ds.ensure_query_write_permissions( + "data", + "insert into extra.cats (id) values (1)", + actor={"id": "writer"}, + ) From 4b5fac9cf7e6de589290007bf6633561eeb3f786 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 22:52:06 -0700 Subject: [PATCH 096/156] Add query management API and create UI Refs #2735 --- datasette/app.py | 37 +- datasette/templates/query.html | 1 + datasette/templates/query_create.html | 71 ++++ datasette/utils/actions_sql.py | 8 + datasette/views/database.py | 528 +++++++++++++++++++++++++- docs/authentication.rst | 50 ++- tests/test_queries.py | 217 ++++++++++- 7 files changed, 904 insertions(+), 8 deletions(-) create mode 100644 datasette/templates/query_create.html diff --git a/datasette/app.py b/datasette/app.py index 9b7fa61e..ce85f447 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -43,7 +43,18 @@ from jinja2.exceptions import TemplateNotFound from .events import Event from .column_types import SQLiteType from .views import Context -from .views.database import database_download, DatabaseView, TableCreateView, QueryView +from .views.database import ( + database_download, + DatabaseView, + TableCreateView, + QueryView, + QueryCreateView, + QueryDeleteView, + QueryDefinitionView, + QueryInsertView, + QueryListView, + QueryUpdateView, +) from .views.index import IndexView from .views.special import ( JsonDataView, @@ -2524,6 +2535,18 @@ class Datasette: r"/(?P[^\/\.]+)(\.(?P\w+))?$", ) add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") + add_route( + QueryListView.as_view(self), + r"/(?P[^\/\.]+)/-/queries$", + ) + add_route( + QueryCreateView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/-/create$", + ) + add_route( + QueryInsertView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/-/insert$", + ) add_route( DatabaseSchemaView.as_view(self), r"/(?P[^\/\.]+)/-/schema(\.(?Pjson|md))?$", @@ -2532,6 +2555,18 @@ class Datasette: wrap_view(QueryView, self), r"/(?P[^\/\.]+)/-/query(\.(?P\w+))?$", ) + add_route( + QueryDefinitionView.as_view(self), + r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/definition$", + ) + add_route( + QueryUpdateView.as_view(self), + r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/update$", + ) + add_route( + QueryDeleteView.as_view(self), + r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/delete$", + ) add_route( wrap_view(table_view, self), r"/(?P[^\/\.]+)/(?P
Name
[^\/\.]+)(\.(?P\w+))?$", diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 8b405da5..7c251e2c 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -67,6 +67,7 @@ {% if not hide_sql %}{% endif %} {{ show_hide_hidden }} + {% if save_query_url %}Save query{% endif %} {% if canned_query and edit_sql_url %}Edit SQL{% endif %}

diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html new file mode 100644 index 00000000..0e6a7b37 --- /dev/null +++ b/datasette/templates/query_create.html @@ -0,0 +1,71 @@ +{% extends "base.html" %} + +{% block title %}Create query{% endblock %} + +{% block extra_head %} +{{- super() -}} +{% include "_codemirror.html" %} +{% endblock %} + +{% block body_class %}query-create db-{{ database|to_css_class }}{% endblock %} + +{% block crumbs %} +{{ crumbs.nav(request=request, database=database) }} +{% endblock %} + +{% block content %} + +

Create query

+ +
+

+

+


+

+ + +

+

+

+ {% if can_publish %} +

+ {% endif %} + +

Analysis

+ {% if analysis_error %} +

{{ analysis_error }}

+ {% elif analysis_rows %} +
+ + + + + + + + + + + + {% for row in analysis_rows %} + + + + + + + + + {% endfor %} + +
OperationDatabaseTablerequired permissionAllowedSource
{{ row.operation }}{{ row.database }}{{ row.table }}{{ row.required_permission }}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}{{ row.source or "" }}
+ {% else %} +

Analysis will show each affected table and required permission.

+ {% endif %} + +

+ + +{% include "_codemirror_foot.html" %} + +{% endblock %} diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index e679ae76..891ee913 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -241,6 +241,14 @@ async def _build_single_action_sql( "),", ] ) + else: + query_parts.extend( + [ + "anon_rules AS (", + " SELECT NULL AS parent, NULL AS child, 0 AS allow, NULL AS reason WHERE 0", + "),", + ] + ) # Continue with the cascading logic query_parts.extend( diff --git a/datasette/views/database.py b/datasette/views/database.py index 0cf93832..f40c434c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -12,7 +12,7 @@ import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted -from datasette.resources import DatabaseResource, QueryResource +from datasette.resources import DatabaseResource, QueryResource, TableResource from datasette.utils import ( add_cors_headers, await_me_maybe, @@ -302,6 +302,9 @@ class QueryContext(Context): allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) + save_query_url: str = field( + metadata={"help": "URL to save the current arbitrary SQL as a query"} + ) tables: list = field(metadata={"help": "List of table objects in the database"}) named_parameter_values: dict = field( metadata={"help": "Dictionary of parameter names/values"} @@ -417,6 +420,510 @@ async def database_download(request, datasette): ) +_query_name_re = re.compile(r"^[^/\.\n]+$") + +_query_fields = { + "sql", + "title", + "description", + "description_html", + "hide_sql", + "fragment", + "parameters", + "params", + "published", + "on_success_message", + "on_success_message_sql", + "on_success_redirect", + "on_error_message", + "on_error_redirect", +} + +_query_create_fields = _query_fields | {"name", "mode", "csrftoken"} +_query_update_fields = _query_fields +_query_write_fields = { + "on_success_message", + "on_success_message_sql", + "on_success_redirect", + "on_error_message", + "on_error_redirect", +} + + +class QueryValidationError(Exception): + def __init__(self, message, status=400): + self.message = message + self.status = status + + +def _actor_id(actor): + if isinstance(actor, dict): + return actor.get("id") + return None + + +def _as_bool(value): + if isinstance(value, bool): + return value + if value is None: + return False + if isinstance(value, int): + return bool(value) + if isinstance(value, str): + return value.lower() in {"1", "true", "t", "yes", "on"} + return bool(value) + + +def _derived_query_parameters(sql): + parameters = [] + seen = set() + for parameter in derive_named_parameters(sql): + if parameter.startswith("_"): + raise QueryValidationError("Magic parameters are not allowed") + if parameter not in seen: + parameters.append(parameter) + seen.add(parameter) + return parameters + + +def _coerce_query_parameters(value, derived): + if value is None: + return derived + if isinstance(value, str): + parameters = [ + parameter.strip() + for parameter in re.split(r"[\s,]+", value) + if parameter.strip() + ] + elif isinstance(value, list): + parameters = value + else: + raise QueryValidationError("parameters must be a list of strings") + if not all(isinstance(parameter, str) for parameter in parameters): + raise QueryValidationError("parameters must be a list of strings") + if any(parameter.startswith("_") for parameter in parameters): + raise QueryValidationError("Magic parameters are not allowed") + if set(parameters) != set(derived): + raise QueryValidationError("parameters must match SQL named parameters") + return parameters + + +async def _json_or_form_payload(request): + content_type = request.headers.get("content-type", "") + if content_type.startswith("application/json"): + body = await request.post_body() + try: + return json.loads(body or b"{}"), True + except json.JSONDecodeError as e: + raise QueryValidationError("Invalid JSON: {}".format(e)) + return await request.post_vars(), False + + +async def _check_query_name(db, name, *, existing=False): + if not name or not isinstance(name, str): + raise QueryValidationError("Query name is required") + if not _query_name_re.match(name): + raise QueryValidationError("Invalid query name") + if not existing and (await db.table_exists(name) or await db.view_exists(name)): + raise QueryValidationError("Query name conflicts with a table or view") + + +async def _analyze_user_query(datasette, db, sql, *, actor, published): + if not sql or not isinstance(sql, str): + raise QueryValidationError("SQL is required") + derived = _derived_query_parameters(sql) + params = {parameter: "" for parameter in derived} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex + + is_write = any( + access.operation in {"insert", "update", "delete"} + for access in analysis.table_accesses + ) + if is_write: + if published: + raise QueryValidationError("Writable queries cannot be published") + try: + await datasette.ensure_query_write_permissions(db.name, sql, actor=actor) + except Forbidden as ex: + raise QueryValidationError(str(ex), status=403) from ex + else: + try: + validate_sql_select(sql) + except InvalidSql as ex: + raise QueryValidationError(str(ex)) from ex + return is_write, derived, analysis + + +def _analysis_rows(analysis): + write_actions = { + "insert": "insert-row", + "update": "update-row", + "delete": "delete-row", + } + return [ + { + "operation": access.operation, + "database": access.database, + "table": access.table, + "required_permission": write_actions.get(access.operation, ""), + "source": access.source, + } + for access in analysis.table_accesses + ] + + +def _apply_query_data_types(data): + typed = dict(data) + for key in ("hide_sql", "published"): + if key in typed: + typed[key] = _as_bool(typed[key]) + return typed + + +async def _prepare_query_create(datasette, request, db, data): + invalid_keys = set(data) - _query_create_fields + if invalid_keys: + raise QueryValidationError("Invalid keys: {}".format(", ".join(invalid_keys))) + + data = _apply_query_data_types(data) + name = data.get("name") + await _check_query_name(db, name) + if await datasette.get_query(db.name, name) is not None: + raise QueryValidationError("Query already exists") + + published = _as_bool(data.get("published")) + is_write, derived, analysis = await _analyze_user_query( + datasette, + db, + data.get("sql"), + actor=request.actor, + published=published, + ) + if published and not await datasette.allowed( + action="publish-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + raise QueryValidationError("Permission denied: need publish-query", status=403) + if not is_write and any(data.get(field) for field in _query_write_fields): + raise QueryValidationError("Writable query fields require writable SQL") + + parameters = _coerce_query_parameters( + data.get("parameters", data.get("params")), + derived, + ) + return { + "name": name, + "sql": data["sql"], + "title": data.get("title"), + "description": data.get("description"), + "description_html": data.get("description_html"), + "hide_sql": _as_bool(data.get("hide_sql")), + "fragment": data.get("fragment"), + "parameters": parameters, + "is_write": is_write, + "published": published, + "source": "user", + "owner_id": _actor_id(request.actor), + "on_success_message": data.get("on_success_message"), + "on_success_message_sql": data.get("on_success_message_sql"), + "on_success_redirect": data.get("on_success_redirect"), + "on_error_message": data.get("on_error_message"), + "on_error_redirect": data.get("on_error_redirect"), + "analysis": analysis, + } + + +async def _prepare_query_update(datasette, request, db, existing, update): + invalid_keys = set(update) - _query_update_fields + if invalid_keys: + raise QueryValidationError("Invalid keys: {}".format(", ".join(invalid_keys))) + + update = _apply_query_data_types(update) + sql = update.get("sql", existing["sql"]) + published = update.get("published", existing["published"]) + query_is_write = existing["is_write"] + derived = _derived_query_parameters(sql) + parameters = None + + if "sql" in update: + query_is_write, derived, _ = await _analyze_user_query( + datasette, + db, + sql, + actor=request.actor, + published=published, + ) + elif published and query_is_write: + raise QueryValidationError("Writable queries cannot be published") + if published and not existing["published"]: + if not await datasette.allowed( + action="publish-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + raise QueryValidationError( + "Permission denied: need publish-query", status=403 + ) + + if "parameters" in update or "params" in update: + parameters = _coerce_query_parameters( + update.get("parameters", update.get("params")), + derived, + ) + elif "sql" in update: + parameters = derived + + if not query_is_write and any(update.get(field) for field in _query_write_fields): + raise QueryValidationError("Writable query fields require writable SQL") + + field_values = { + "sql": sql, + "title": update.get("title"), + "description": update.get("description"), + "description_html": update.get("description_html"), + "hide_sql": update.get("hide_sql"), + "fragment": update.get("fragment"), + "parameters": parameters, + "is_write": query_is_write, + "published": published, + "on_success_message": update.get("on_success_message"), + "on_success_message_sql": update.get("on_success_message_sql"), + "on_success_redirect": update.get("on_success_redirect"), + "on_error_message": update.get("on_error_message"), + "on_error_redirect": update.get("on_error_redirect"), + } + update_kwargs = {} + for field, value in field_values.items(): + if field in update: + update_kwargs[field] = value + if parameters is not None: + update_kwargs["parameters"] = parameters + if "sql" in update: + update_kwargs["is_write"] = query_is_write + return update_kwargs + + +class QueryListView(BaseView): + name = "query-list" + + async def get(self, request): + db = await self.ds.resolve_database(request) + page = await self.ds.allowed_resources( + "view-query", + request.actor, + parent=db.name, + limit=1000, + ) + all_queries = await self.ds.get_queries(db.name) + queries = [ + all_queries[resource.child] + for resource in page.resources + if resource.child in all_queries + ] + return Response.json({"ok": True, "database": db.name, "queries": queries}) + + +class QueryCreateView(BaseView): + name = "query-create" + has_json_alternate = False + + async def get(self, request): + db = await self.ds.resolve_database(request) + await self.ds.ensure_permission( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ) + await self.ds.ensure_permission( + action="insert-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ) + + sql = request.args.get("sql") or "" + analysis_error = None + analysis_rows = [] + parameter_names = [] + if sql: + try: + parameter_names = _derived_query_parameters(sql) + params = {parameter: "" for parameter in parameter_names} + analysis = await db.analyze_sql(sql, params) + rows = _analysis_rows(analysis) + for row in rows: + permission = row["required_permission"] + if permission: + row["allowed"] = await self.ds.allowed( + action=permission, + resource=TableResource(row["database"], row["table"]), + actor=request.actor, + ) + else: + row["allowed"] = None + analysis_rows = rows + except (QueryValidationError, sqlite3.DatabaseError) as ex: + analysis_error = getattr(ex, "message", str(ex)) + + return await self.render( + ["query_create.html"], + request, + { + "database": db.name, + "database_color": db.color, + "sql": sql, + "parameter_names": parameter_names, + "can_publish": await self.ds.allowed( + action="publish-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ), + "analysis_error": analysis_error, + "analysis_rows": analysis_rows, + "save_disabled": bool( + analysis_error + or any(row["allowed"] is False for row in analysis_rows) + ), + }, + ) + + +class QueryInsertView(BaseView): + name = "query-insert" + + async def post(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _error(["Permission denied: need execute-sql"], 403) + if not await self.ds.allowed( + action="insert-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _error(["Permission denied: need insert-query"], 403) + + try: + data, is_json = await _json_or_form_payload(request) + if not isinstance(data, dict): + raise QueryValidationError("JSON must be a dictionary") + query_data = data.get("query") if is_json else data + if not isinstance(query_data, dict): + raise QueryValidationError("JSON must contain a query dictionary") + prepared = await _prepare_query_create(self.ds, request, db, query_data) + except QueryValidationError as ex: + return _error([ex.message], ex.status) + + prepared.pop("analysis") + name = prepared.pop("name") + try: + await self.ds.add_query(db.name, name, replace=False, **prepared) + except sqlite3.IntegrityError as ex: + return _error([str(ex)], 400) + + query = await self.ds.get_query(db.name, name) + if is_json: + return Response.json({"ok": True, "query": query}, status=201) + self.ds.add_message(request, "Query saved", self.ds.INFO) + return Response.redirect(self.ds.urls.path(self.ds.urls.table(db.name, name))) + + +class QueryDefinitionView(BaseView): + name = "query-definition" + + async def get(self, request): + db = await self.ds.resolve_database(request) + query_name = tilde_decode(request.url_vars["query"]) + query = await self.ds.get_query(db.name, query_name) + if query is None: + return _error(["Query not found: {}".format(query_name)], 404) + if not await self.ds.allowed( + action="view-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ): + return _error(["Permission denied"], 403) + return Response.json({"ok": True, "query": query}) + + +class QueryUpdateView(BaseView): + name = "query-update" + + async def post(self, request): + db = await self.ds.resolve_database(request) + query_name = tilde_decode(request.url_vars["query"]) + existing = await self.ds.get_query(db.name, query_name) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + if not await self.ds.allowed( + action="update-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ): + return _error(["Permission denied: need update-query"], 403) + + try: + data, _ = await _json_or_form_payload(request) + if not isinstance(data, dict): + raise QueryValidationError("JSON must be a dictionary") + invalid_keys = set(data) - {"update", "return"} + if invalid_keys: + raise QueryValidationError( + "Invalid keys: {}".format(", ".join(invalid_keys)) + ) + update = data.get("update") + if not isinstance(update, dict): + raise QueryValidationError("JSON must contain an update dictionary") + if "sql" in update and not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + raise QueryValidationError( + "Permission denied: need execute-sql", status=403 + ) + update_kwargs = await _prepare_query_update( + self.ds, request, db, existing, update + ) + except QueryValidationError as ex: + return _error([ex.message], ex.status) + + await self.ds.update_query(db.name, query_name, **update_kwargs) + if data.get("return"): + return Response.json( + { + "ok": True, + "query": await self.ds.get_query(db.name, query_name), + } + ) + return Response.json({"ok": True}) + + +class QueryDeleteView(BaseView): + name = "query-delete" + + async def post(self, request): + db = await self.ds.resolve_database(request) + query_name = tilde_decode(request.url_vars["query"]) + existing = await self.ds.get_query(db.name, query_name) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + if not await self.ds.allowed( + action="delete-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ): + return _error(["Permission denied: need delete-query"], 403) + await self.ds.remove_query(db.name, query_name) + return Response.json({"ok": True}) + + class QueryView(View): async def post(self, request, datasette): from datasette.app import TableNotFound @@ -741,6 +1248,11 @@ class QueryView(View): resource=DatabaseResource(database=database), actor=request.actor, ) + allow_insert_query = await datasette.allowed( + action="insert-query", + resource=DatabaseResource(database=database), + actor=request.actor, + ) show_hide_hidden = "" if canned_query and canned_query.get("hide_sql"): @@ -790,6 +1302,19 @@ class QueryView(View): } ) ) + save_query_url = None + if ( + not canned_query + and allow_execute_sql + and allow_insert_query + and is_validated_sql + and ":_" not in sql + ): + save_query_url = ( + datasette.urls.database(database) + + "/-/queries/-/create?" + + urlencode({"sql": sql}) + ) async def query_actions(): query_actions = [] @@ -827,6 +1352,7 @@ class QueryView(View): show_hide_text=show_hide_text, editable=not canned_query, allow_execute_sql=allow_execute_sql, + save_query_url=save_query_url, tables=await get_tables(datasette, request, db, allowed_dict), named_parameter_values=named_parameter_values, edit_sql_url=edit_sql_url, diff --git a/docs/authentication.rst b/docs/authentication.rst index 7daefab7..543f069b 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1285,12 +1285,56 @@ Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.i view-query ---------- -Actor is allowed to view (and execute) a :ref:`canned query ` page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size - this includes executing :ref:`canned_queries_writable`. +Actor is allowed to view (and execute) a saved query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size - this includes executing :ref:`canned_queries_writable`. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) - - ``query`` is the name of the canned query (string) + + ``query`` is the name of the query (string) + +.. _actions_insert_query: + +insert-query +------------ + +Actor is allowed to create saved queries in a database. + +``resource`` - ``datasette.resources.DatabaseResource(database)`` + ``database`` is the name of the database (string) + +.. _actions_publish_query: + +publish-query +------------- + +Actor is allowed to publish a saved read-only query so actors without ``execute-sql`` can run it. + +``resource`` - ``datasette.resources.DatabaseResource(database)`` + ``database`` is the name of the database (string) + +.. _actions_update_query: + +update-query +------------ + +Actor is allowed to update a saved query. + +``resource`` - ``datasette.resources.QueryResource(database, query)`` + ``database`` is the name of the database (string) + + ``query`` is the name of the query (string) + +.. _actions_delete_query: + +delete-query +------------ + +Actor is allowed to delete a saved query. + +``resource`` - ``datasette.resources.QueryResource(database, query)`` + ``database`` is the name of the database (string) + + ``query`` is the name of the query (string) .. _actions_insert_row: diff --git a/tests/test_queries.py b/tests/test_queries.py index 01174a18..8e802b75 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -179,9 +179,7 @@ async def test_query_resources_come_from_internal_table(): page = await ds.allowed_resources("view-query", actor=None) - assert [(r.parent, r.child) for r in page.resources] == [ - ("data", "internal_query") - ] + assert [(r.parent, r.child) for r in page.resources] == [("data", "internal_query")] @pytest.mark.asyncio @@ -279,3 +277,216 @@ async def test_analyze_write_query_rejects_writes_to_attached_databases(): "insert into extra.cats (id) values (1)", actor={"id": "writer"}, ) + + +@pytest.mark.asyncio +async def test_query_insert_api_creates_read_only_query(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_insert_api", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/queries/-/insert", + actor={"id": "root"}, + json={ + "query": { + "name": "by_name", + "sql": "select * from dogs where name = :name", + "title": "By name", + } + }, + ) + + assert response.status_code == 201 + data = response.json() + assert data["ok"] is True + assert data["query"]["name"] == "by_name" + assert data["query"]["parameters"] == ["name"] + assert data["query"]["is_write"] is False + assert data["query"]["source"] == "user" + assert data["query"]["owner_id"] == "root" + + +@pytest.mark.asyncio +async def test_query_list_and_definition_api(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_api", name="data") + await ds.invoke_startup() + await ds.add_query("data", "listed", "select 1", title="Listed", published=True) + + list_response = await ds.client.get( + "/data/-/queries", + actor={"id": "root"}, + ) + definition_response = await ds.client.get( + "/data/listed/-/definition", + actor={"id": "root"}, + ) + + assert list_response.status_code == 200 + assert list_response.json()["queries"][0]["name"] == "listed" + assert definition_response.status_code == 200 + assert definition_response.json()["query"]["title"] == "Listed" + + +@pytest.mark.asyncio +async def test_query_insert_api_publish_requires_publish_query(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-sql": {"id": "writer"}, + "insert-query": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("query_publish_api", name="data") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/queries/-/insert", + actor={"id": "writer"}, + json={"query": {"name": "public", "sql": "select 1", "published": True}}, + ) + + assert response.status_code == 403 + assert response.json()["errors"] == ["Permission denied: need publish-query"] + + +@pytest.mark.asyncio +async def test_query_insert_api_creates_writable_query(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_write_api", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/queries/-/insert", + actor={"id": "root"}, + json={ + "query": { + "name": "insert_dog", + "sql": "insert into dogs (name) values (:name)", + } + }, + ) + + assert response.status_code == 201 + query = response.json()["query"] + assert query["is_write"] is True + assert query["published"] is False + assert query["parameters"] == ["name"] + + bad_response = await ds.client.post( + "/data/-/queries/-/insert", + actor={"id": "root"}, + json={ + "query": { + "name": "published_insert", + "sql": "insert into dogs (name) values (:name)", + "published": True, + } + }, + ) + + assert bad_response.status_code == 400 + assert bad_response.json()["errors"] == ["Writable queries cannot be published"] + + +@pytest.mark.asyncio +async def test_query_update_and_delete_api(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + ds.add_memory_database("query_update_api", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "editable", + "select 1", + title="Original", + source="user", + owner_id="root", + ) + + update_response = await ds.client.post( + "/data/editable/-/update", + actor={"id": "root"}, + json={ + "update": { + "title": "Updated", + "description": "Fresh", + "on_success_redirect": None, + }, + "return": True, + }, + ) + + assert update_response.status_code == 200 + updated = update_response.json()["query"] + assert updated["title"] == "Updated" + assert updated["description"] == "Fresh" + assert updated["on_success_redirect"] is None + + delete_response = await ds.client.post( + "/data/editable/-/delete", + actor={"id": "root"}, + json={}, + ) + + assert delete_response.status_code == 200 + assert delete_response.json() == {"ok": True} + assert await ds.get_query("data", "editable") is None + + +@pytest.mark.asyncio +async def test_query_insert_api_rejects_magic_parameters(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + ds.add_memory_database("query_magic_api", name="data") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/queries/-/insert", + actor={"id": "root"}, + json={"query": {"name": "magic", "sql": "select :_actor_id"}}, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == ["Magic parameters are not allowed"] + + +@pytest.mark.asyncio +async def test_create_query_ui_and_arbitrary_sql_save_link(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_create_ui", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + create_response = await ds.client.get( + "/data/-/queries/-/create?sql=select+*+from+dogs", + actor={"id": "root"}, + ) + query_response = await ds.client.get( + "/data/-/query?sql=select+*+from+dogs", + actor={"id": "root"}, + ) + + assert create_response.status_code == 200 + assert "Create query" in create_response.text + assert "Read-only" in create_response.text + assert "Writable" in create_response.text + assert "required permission" in create_response.text + assert query_response.status_code == 200 + assert "Save query" in query_response.text + assert "/data/-/queries/-/create?sql=select+%2A+from+dogs" in query_response.text From 040e42ddca047a2e616d412b733fd36de233d2b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 22:58:50 -0700 Subject: [PATCH 097/156] Enforce query ownership and remove canned query hook Refs #2735 --- datasette/default_permissions/__init__.py | 16 ----- datasette/default_permissions/defaults.py | 30 ++++++++ datasette/hookspecs.py | 5 -- datasette/views/database.py | 12 ++++ tests/fixtures.py | 2 - tests/plugins/my_plugin.py | 5 -- tests/plugins/my_plugin_2.py | 26 +++---- tests/test_canned_queries.py | 34 +++++---- tests/test_permissions.py | 40 ++++------- tests/test_plugins.py | 27 ++------ tests/test_queries.py | 84 +++++++++++++++++++++++ 11 files changed, 182 insertions(+), 99 deletions(-) diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index 5a53dbe7..a9f2d8bd 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -17,13 +17,6 @@ UNION/INTERSECT operations. The order of evaluation is: from __future__ import annotations -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from datasette.app import Datasette - -from datasette import hookimpl - # Re-export all hooks and public utilities from .restrictions import ( actor_restrictions_sql as actor_restrictions_sql, @@ -38,12 +31,3 @@ from .defaults import ( default_query_permissions_sql as default_query_permissions_sql, DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS, ) - - -@hookimpl -def canned_queries(datasette: "Datasette", database: str, actor) -> dict: - """Return canned queries defined in datasette.yaml configuration.""" - queries = ( - ((datasette.config or {}).get("databases") or {}).get(database) or {} - ).get("queries") or {} - return queries diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 2613c4f4..9737de96 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -74,6 +74,22 @@ async def default_query_permissions_sql( actor: Optional[dict], action: str, ) -> Optional[PermissionSQL]: + actor_id = actor.get("id") if isinstance(actor, dict) else None + + if action in {"update-query", "delete-query"}: + if actor_id is None: + return None + return PermissionSQL( + sql=""" + SELECT database_name AS parent, name AS child, 1 AS allow, + 'query owner' AS reason + FROM queries + WHERE source = 'user' + AND owner_id = :query_owner_id + """, + params={"query_owner_id": actor_id}, + ) + if action != "view-query": return None @@ -98,6 +114,19 @@ async def default_query_permissions_sql( AND source IN ('config', 'plugin') """ + user_writable_sql = "" + if actor_id is not None: + params["query_owner_id"] = actor_id + user_writable_sql = """ + UNION ALL + SELECT database_name AS parent, name AS child, 1 AS allow, + 'query owner' AS reason + FROM queries + WHERE is_write = 1 + AND source = 'user' + AND owner_id = :query_owner_id + """ + return PermissionSQL( sql=f""" WITH execute_sql_allowed AS ( @@ -118,6 +147,7 @@ async def default_query_permissions_sql( WHERE q.is_write = 0 AND q.published = 0 {trusted_writable_sql} + {user_writable_sql} """, params=params, ) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index cf95abcb..a4067eaa 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -137,11 +137,6 @@ def permission_resources_sql(datasette, actor, action): """ -@hookspec -def canned_queries(datasette, database, actor): - """Return a dictionary of canned query definitions or an awaitable function that returns them""" - - @hookspec def register_magic_parameters(datasette): """Return a list of (name, function) magic parameter functions""" diff --git a/datasette/views/database.py b/datasette/views/database.py index f40c434c..2cdaab9f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -945,6 +945,18 @@ class QueryView(View): # That should not have happened raise DatasetteError("Unexpected table found on POST", status=404) + if not await datasette.allowed( + action="view-query", + resource=QueryResource(database=db.name, query=canned_query["name"]), + actor=request.actor, + ): + raise Forbidden("You do not have permission to view this query") + + if canned_query.get("write") and canned_query.get("source") == "user": + await datasette.ensure_query_write_permissions( + db.name, canned_query["sql"], actor=request.actor + ) + # If database is immutable, return an error if not db.is_mutable: raise Forbidden("Database is immutable") diff --git a/tests/fixtures.py b/tests/fixtures.py index 71884294..8ab3633f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -35,7 +35,6 @@ EXPECTED_PLUGINS = [ "hooks": [ "actor_from_request", "asgi_wrapper", - "canned_queries", "database_actions", "extra_body_script", "extra_css_urls", @@ -68,7 +67,6 @@ EXPECTED_PLUGINS = [ "hooks": [ "actor_from_request", "asgi_wrapper", - "canned_queries", "extra_js_urls", "extra_template_vars", "handle_exception", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 4e401c07..1dd9ed3e 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -314,11 +314,6 @@ def startup(datasette): _ = (Response, Forbidden, NotFound, hookimpl, actor_matches_allow) -@hookimpl -def canned_queries(datasette, database, actor): - return {"from_hook": f"select 1, '{actor['id'] if actor else 'null'}' as actor_id"} - - @hookimpl def register_magic_parameters(): from uuid import uuid4 diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 9e8d9b2b..e3d3e760 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -139,20 +139,20 @@ def startup(datasette): datasette._startup_catalog_databases = [ row["database_name"] for row in catalog_rows ] - - return inner - - -@hookimpl -def canned_queries(datasette, database): - async def inner(): - return { - "from_async_hook": "select {}".format( - ( - await datasette.get_database(database).execute("select 1 + 1") - ).first()[0] + for database in datasette.databases: + await datasette.add_query( + database, + "from_hook", + "select 1, 'null' as actor_id", + source="plugin", + ) + result = await datasette.get_database(database).execute("select 1 + 1") + await datasette.add_query( + database, + "from_async_hook", + "select {}".format(result.first()[0]), + source="plugin", ) - } return inner diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index 5e36a87a..e06ad189 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -1,10 +1,16 @@ from bs4 import BeautifulSoup as Soup +from asgiref.sync import async_to_sync import json import pytest import re from .fixtures import make_app_client +def update_query(client, name, **kwargs): + async_to_sync(client.ds.invoke_startup)() + async_to_sync(client.ds.update_query)("data", name, **kwargs) + + @pytest.fixture def canned_write_client(tmpdir): template_dir = tmpdir / "canned_write_templates" @@ -153,9 +159,7 @@ def test_insert_error(canned_write_client): ) assert [["UNIQUE constraint failed: names.rowid", 3]] == messages # How about with a custom error message? - canned_write_client.ds.config["databases"]["data"]["queries"][ - "add_name_specify_id" - ]["on_error_message"] = "ERROR" + update_query(canned_write_client, "add_name_specify_id", on_error_message="ERROR") response = canned_write_client.post( "/data/add_name_specify_id", {"rowid": 1, "name": "Should fail"}, @@ -327,12 +331,16 @@ def magic_parameters_client(): ], ) def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re): - magic_parameters_client.ds.config["databases"]["data"]["queries"]["runme_post"][ - "sql" - ] = f"insert into logs (line) values (:{magic_parameter})" - magic_parameters_client.ds.config["databases"]["data"]["queries"]["runme_get"][ - "sql" - ] = f"select :{magic_parameter} as result" + update_query( + magic_parameters_client, + "runme_post", + sql=f"insert into logs (line) values (:{magic_parameter})", + ) + update_query( + magic_parameters_client, + "runme_get", + sql=f"select :{magic_parameter} as result", + ) cookies = { "ds_actor": magic_parameters_client.actor_cookie({"id": "root"}), "foo": "bar", @@ -366,9 +374,11 @@ def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re) @pytest.mark.parametrize("use_csrf", [True, False]) @pytest.mark.parametrize("return_json", [True, False]) def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_json): - magic_parameters_client.ds.config["databases"]["data"]["queries"]["runme_post"][ - "sql" - ] = "insert into logs (line) values (:_header_host)" + update_query( + magic_parameters_client, + "runme_post", + sql="insert into logs (line) values (:_header_host)", + ) qs = "" if return_json: qs = "?_json=1" diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 8166532f..04800ed3 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,4 +1,5 @@ import collections +from asgiref.sync import async_to_sync from datasette.app import Datasette from datasette.cli import cli from datasette.default_permissions import restrictions_allow_action @@ -609,6 +610,10 @@ def test_padlocks_on_database_page(cascade_app_client): previous_config = cascade_app_client.ds.config try: cascade_app_client.ds.config = config + async_to_sync(cascade_app_client.ds.invoke_startup)() + async_to_sync(cascade_app_client.ds.add_query)( + "fixtures", "query_two", "select 2", source="config" + ) response = cascade_app_client.get( "/fixtures", cookies={"ds_actor": cascade_app_client.actor_cookie({"id": "test"})}, @@ -624,6 +629,7 @@ def test_padlocks_on_database_page(cascade_app_client): assert ">simple_view" in response.text finally: cascade_app_client.ds.config = previous_config + async_to_sync(cascade_app_client.ds.remove_query)("fixtures", "query_two") @pytest.mark.asyncio @@ -954,39 +960,20 @@ async def test_permissions_in_config( @pytest.mark.asyncio -async def test_allowed_resources_view_query_includes_actor_specific_canned_queries(): - """ - Actor-specific canned queries should be listed by allowed_resources("view-query"). - - This test is intentionally explicit about the previous bug: - - the canned query only exists for actor "alice" - - the permission rule only allows actor "alice" to view it - - allowed() succeeds for that specific query resource - - allowed_resources("view-query", actor) must include the same query - - Before the fix, QueryResource.resources_sql() called canned_queries(..., actor=None), - so the query was omitted from resource enumeration and allowed_resources() returned - an empty list even though allowed() returned True. - """ +async def test_allowed_resources_view_query_includes_actor_specific_query_permissions(): from datasette import hookimpl from datasette.permissions import PermissionSQL from datasette.resources import QueryResource - class ActorSpecificQueryPlugin: - __name__ = "ActorSpecificQueryPlugin" - - @hookimpl - def canned_queries(self, datasette, database, actor): - if database == "testdb" and actor and actor.get("id") == "alice": - return {"user_only": {"sql": "select 1 as n"}} - return {} + class ActorSpecificQueryPermissionPlugin: + __name__ = "ActorSpecificQueryPermissionPlugin" @hookimpl def permission_resources_sql(self, datasette, actor, action): if action == "view-query" and actor and actor.get("id") == "alice": return PermissionSQL(sql=""" SELECT 'testdb' AS parent, 'user_only' AS child, 1 AS allow, - 'alice can view her actor-specific canned query' AS reason + 'alice can view this query' AS reason """) return None @@ -994,9 +981,10 @@ async def test_allowed_resources_view_query_includes_actor_specific_canned_queri await ds.invoke_startup() ds.add_memory_database("testdb") await ds._refresh_schemas() + await ds.add_query("testdb", "user_only", "select 1 as n") - plugin = ActorSpecificQueryPlugin() - ds.pm.register(plugin, name="actor_specific_query_plugin") + plugin = ActorSpecificQueryPermissionPlugin() + ds.pm.register(plugin, name="actor_specific_query_permission_plugin") try: actor = {"id": "alice"} @@ -1012,7 +1000,7 @@ async def test_allowed_resources_view_query_includes_actor_specific_canned_queri ("testdb", "user_only") ] finally: - ds.pm.unregister(name="actor_specific_query_plugin") + ds.pm.unregister(name="actor_specific_query_permission_plugin") @pytest.mark.asyncio diff --git a/tests/test_plugins.py b/tests/test_plugins.py index c5b9aef0..b5a13ae5 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -885,40 +885,27 @@ async def test_hook_startup_catalog_populated(ds_client): @pytest.mark.asyncio -async def test_hook_canned_queries(ds_client): +async def test_plugin_startup_queries(ds_client): queries = (await ds_client.get("/fixtures.json")).json()["queries"] queries_by_name = {q["name"]: q for q in queries} - assert { - "sql": "select 2", - "name": "from_async_hook", - "private": False, - } == queries_by_name["from_async_hook"] - assert { - "sql": "select 1, 'null' as actor_id", - "name": "from_hook", - "private": False, - } == queries_by_name["from_hook"] + assert queries_by_name["from_async_hook"]["sql"] == "select 2" + assert queries_by_name["from_async_hook"]["private"] is False + assert queries_by_name["from_hook"]["sql"] == "select 1, 'null' as actor_id" + assert queries_by_name["from_hook"]["private"] is False @pytest.mark.asyncio -async def test_hook_canned_queries_non_async(ds_client): +async def test_plugin_startup_query_from_hook(ds_client): response = await ds_client.get("/fixtures/from_hook.json?_shape=array") assert [{"1": 1, "actor_id": "null"}] == response.json() @pytest.mark.asyncio -async def test_hook_canned_queries_async(ds_client): +async def test_plugin_startup_query_from_async_hook(ds_client): response = await ds_client.get("/fixtures/from_async_hook.json?_shape=array") assert [{"2": 2}] == response.json() -@pytest.mark.asyncio -async def test_hook_canned_queries_actor(ds_client): - assert ( - await ds_client.get("/fixtures/from_hook.json?_bot=1&_shape=array") - ).json() == [{"1": 1, "actor_id": "bot"}] - - def test_hook_register_magic_parameters(restore_working_directory): with make_app_client( extra_databases={"data.db": "create table logs (line text)"}, diff --git a/tests/test_queries.py b/tests/test_queries.py index 8e802b75..c6685d6c 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -490,3 +490,87 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): assert query_response.status_code == 200 assert "Save query" in query_response.text assert "/data/-/queries/-/create?sql=select+%2A+from+dogs" in query_response.text + + +@pytest.mark.asyncio +async def test_query_owner_gets_update_delete_and_writable_view_defaults(): + ds = Datasette(memory=True, default_deny=True) + ds.add_memory_database("query_owner_defaults", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "insert_dog", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="alice", + ) + + for action in ("view-query", "update-query", "delete-query"): + assert await ds.allowed( + action=action, + resource=QueryResource("data", "insert_dog"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action=action, + resource=QueryResource("data", "insert_dog"), + actor={"id": "bob"}, + ) + + +@pytest.mark.asyncio +async def test_user_writable_query_execution_rechecks_table_permissions(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "tables": { + "dogs": { + "permissions": { + "insert-row": {"id": "alice"}, + } + } + } + } + } + }, + ) + db = ds.add_memory_database("query_write_execution", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "insert_dog", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "insert_cat", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="bob", + ) + + allowed_response = await ds.client.post( + "/data/insert_dog?_json=1", + actor={"id": "alice"}, + data={"name": "Cleo"}, + ) + denied_response = await ds.client.post( + "/data/insert_cat?_json=1", + actor={"id": "bob"}, + data={"name": "Milo"}, + ) + + assert allowed_response.status_code == 200 + assert allowed_response.json()["ok"] is True + assert denied_response.status_code == 403 + rows = (await db.execute("select name from dogs")).dicts() + assert rows == [{"name": "Cleo"}] From 3b26b7aff03ed78fae6a17a5a65edc5b83415dee Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 23:00:00 -0700 Subject: [PATCH 098/156] Document canned query hook removal Refs #2735 --- docs/plugin_hooks.rst | 73 ++----------------------------------------- docs/plugins.rst | 1 - 2 files changed, 2 insertions(+), 72 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 2b8f5eb2..b2676b3e 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1212,77 +1212,8 @@ Examples: `datasette-saved-queries `. - -Use this hook to return a dictionary of additional :ref:`canned query ` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query ` documentation. - -.. code-block:: python - - from datasette import hookimpl - - - @hookimpl - def canned_queries(datasette, database): - if database == "mydb": - return { - "my_query": { - "sql": "select * from my_table where id > :min_id" - } - } - -The hook can alternatively return an awaitable function that returns a list. Here's an example that returns queries that have been stored in the ``saved_queries`` database table, if one exists: - -.. code-block:: python - - from datasette import hookimpl - - - @hookimpl - def canned_queries(datasette, database): - async def inner(): - db = datasette.get_database(database) - if await db.table_exists("saved_queries"): - results = await db.execute( - "select name, sql from saved_queries" - ) - return { - result["name"]: {"sql": result["sql"]} - for result in results - } - - return inner - -The actor parameter can be used to include the currently authenticated actor in your decision. Here's an example that returns saved queries that were saved by that actor: - -.. code-block:: python - - from datasette import hookimpl - - - @hookimpl - def canned_queries(datasette, database, actor): - async def inner(): - db = datasette.get_database(database) - if actor is not None and await db.table_exists( - "saved_queries" - ): - results = await db.execute( - "select name, sql from saved_queries where actor_id = :id", - {"id": actor["id"]}, - ) - return { - result["name"]: {"sql": result["sql"]} - for result in results - } - - return inner +This hook has been removed. Plugins that need to add saved queries should use +the :ref:`plugin_hook_startup` hook and call ``await datasette.add_query(...)``. Example: `datasette-saved-queries `__ diff --git a/docs/plugins.rst b/docs/plugins.rst index 77958205..8fa49d6d 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -249,7 +249,6 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "templates": false, "version": null, "hooks": [ - "canned_queries", "permission_resources_sql" ] }, From 2d77e3334b48417c5e27355bb4016c7c76acf30e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 24 May 2026 23:06:01 -0700 Subject: [PATCH 099/156] Clean up query management test coverage Refs #2735 --- datasette/views/database.py | 6 ++-- docs/json_api.rst | 42 +++++++++++++++++++++++ tests/plugins/my_plugin_2.py | 14 -------- tests/test_canned_queries.py | 4 --- tests/test_html.py | 2 -- tests/test_permissions.py | 1 - tests/test_plugins.py | 65 ++++++++++++++++++++++++++++-------- 7 files changed, 96 insertions(+), 38 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 2cdaab9f..d521f7ad 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -697,9 +697,9 @@ async def _prepare_query_update(datasette, request, db, existing, update): "on_error_redirect": update.get("on_error_redirect"), } update_kwargs = {} - for field, value in field_values.items(): - if field in update: - update_kwargs[field] = value + for field_name, value in field_values.items(): + if field_name in update: + update_kwargs[field_name] = value if parameters is not None: update_kwargs["parameters"] = parameters if "sql" in update: diff --git a/docs/json_api.rst b/docs/json_api.rst index 48c70af6..d5cd231c 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -505,6 +505,48 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`. +.. _QueryListView: + +Listing saved queries +~~~~~~~~~~~~~~~~~~~~~ + +``GET //-/queries`` returns saved query definitions the actor can view. + +.. _QueryCreateView: + +Creating saved queries in the UI +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``GET //-/queries/-/create`` provides a form for creating saved queries. + +.. _QueryInsertView: + +Creating saved queries +~~~~~~~~~~~~~~~~~~~~~~ + +``POST //-/queries/-/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. + +.. _QueryDefinitionView: + +Getting a saved query definition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``GET ///-/definition`` returns a saved query definition without executing it. + +.. _QueryUpdateView: + +Updating saved queries +~~~~~~~~~~~~~~~~~~~~~~ + +``POST ///-/update`` updates a saved query using a JSON body with an ``"update"`` object. + +.. _QueryDeleteView: + +Deleting saved queries +~~~~~~~~~~~~~~~~~~~~~~ + +``POST ///-/delete`` deletes a saved query. + .. _TableInsertView: Inserting rows diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index e3d3e760..864637a6 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -139,20 +139,6 @@ def startup(datasette): datasette._startup_catalog_databases = [ row["database_name"] for row in catalog_rows ] - for database in datasette.databases: - await datasette.add_query( - database, - "from_hook", - "select 1, 'null' as actor_id", - source="plugin", - ) - result = await datasette.get_database(database).execute("select 1 + 1") - await datasette.add_query( - database, - "from_async_hook", - "select {}".format(result.first()[0]), - source="plugin", - ) return inner diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index e06ad189..c46fd86f 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -254,10 +254,8 @@ def test_canned_query_permissions_on_database_page(canned_write_client): } assert query_names == { "add_name_specify_id_with_error_in_on_success_message_sql", - "from_hook", "update_name", "add_name_specify_id", - "from_async_hook", "canned_read", "add_name", } @@ -284,8 +282,6 @@ def test_canned_query_permissions_on_database_page(canned_write_client): }, {"name": "canned_read", "private": False}, {"name": "delete_name", "private": True}, - {"name": "from_async_hook", "private": False}, - {"name": "from_hook", "private": False}, {"name": "update_name", "private": False}, ] diff --git a/tests/test_html.py b/tests/test_html.py index efc1040d..e5f00e17 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -158,8 +158,6 @@ async def test_database_page(ds_client): queries_ul = soup.find("h2", string="Queries").find_next_sibling("ul") assert queries_ul is not None assert [ - ("/fixtures/from_async_hook", "from_async_hook"), - ("/fixtures/from_hook", "from_hook"), ("/fixtures/magic_parameters", "magic_parameters"), ("/fixtures/neighborhood_search#fragment-goes-here", "Search neighborhoods"), ("/fixtures/pragma_cache_size", "pragma_cache_size"), diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 04800ed3..22f294bb 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -622,7 +622,6 @@ def test_padlocks_on_database_page(cascade_app_client): assert ">123_starts_with_digits" in response.text assert ">Table With Space In Name 🔒" in response.text # Queries - assert ">from_async_hook 🔒" in response.text assert ">query_two" in response.text # Views assert ">paginated_view 🔒" in response.text diff --git a/tests/test_plugins.py b/tests/test_plugins.py index b5a13ae5..f7adbd66 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -885,24 +885,61 @@ async def test_hook_startup_catalog_populated(ds_client): @pytest.mark.asyncio -async def test_plugin_startup_queries(ds_client): - queries = (await ds_client.get("/fixtures.json")).json()["queries"] +async def test_plugin_startup_can_add_queries(): + ds = Datasette(memory=True) + ds.add_memory_database("plugin_startup_queries", name="data") + + class AddQueriesPlugin: + __name__ = "AddQueriesPlugin" + + @hookimpl + def startup(self, datasette): + async def inner(): + result = await datasette.get_database("data").execute("select 1 + 1") + await datasette.add_query( + "data", + "from_startup", + "select {}".format(result.first()[0]), + source="plugin", + ) + + return inner + + ds.pm.register(AddQueriesPlugin(), name="add_queries_plugin") + try: + response = await ds.client.get("/data.json") + finally: + ds.pm.unregister(name="add_queries_plugin") + + queries = response.json()["queries"] queries_by_name = {q["name"]: q for q in queries} - assert queries_by_name["from_async_hook"]["sql"] == "select 2" - assert queries_by_name["from_async_hook"]["private"] is False - assert queries_by_name["from_hook"]["sql"] == "select 1, 'null' as actor_id" - assert queries_by_name["from_hook"]["private"] is False + assert queries_by_name["from_startup"]["sql"] == "select 2" + assert queries_by_name["from_startup"]["private"] is False @pytest.mark.asyncio -async def test_plugin_startup_query_from_hook(ds_client): - response = await ds_client.get("/fixtures/from_hook.json?_shape=array") - assert [{"1": 1, "actor_id": "null"}] == response.json() +async def test_plugin_startup_query_can_execute(): + ds = Datasette(memory=True) + ds.add_memory_database("plugin_startup_query_execute", name="data") + class AddQueryPlugin: + __name__ = "AddQueryPlugin" + + @hookimpl + def startup(self, datasette): + async def inner(): + await datasette.add_query( + "data", "from_startup", "select 2", source="plugin" + ) + + return inner + + ds.pm.register(AddQueryPlugin(), name="add_query_plugin") + try: + response = await ds.client.get("/data/from_startup.json?_shape=array") + finally: + ds.pm.unregister(name="add_query_plugin") -@pytest.mark.asyncio -async def test_plugin_startup_query_from_async_hook(ds_client): - response = await ds_client.get("/fixtures/from_async_hook.json?_shape=array") assert [{"2": 2}] == response.json() @@ -1514,9 +1551,9 @@ async def test_hook_top_query(ds_client): async def test_hook_top_canned_query(ds_client): try: pm.register(SlotPlugin(), name="SlotPlugin") - response = await ds_client.get("/fixtures/from_hook?z=xyz") + response = await ds_client.get("/fixtures/magic_parameters?z=xyz") assert response.status_code == 200 - assert "Xtop_query:fixtures:from_hook:xyz" in response.text + assert "Xtop_query:fixtures:magic_parameters:xyz" in response.text finally: pm.unregister(name="SlotPlugin") From ef43c103880fe819206f4e0dd12fa62add1c927c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 08:30:49 -0700 Subject: [PATCH 100/156] Add arbitrary write SQL execution page Refs #2735 --- datasette/app.py | 21 +- datasette/default_actions.py | 7 + datasette/templates/execute_write.html | 71 +++++++ datasette/templates/query_create.html | 3 + datasette/views/database.py | 266 +++++++++++++++++++++++-- docs/authentication.rst | 12 +- docs/json_api.rst | 9 + tests/test_queries.py | 122 ++++++++++++ 8 files changed, 487 insertions(+), 24 deletions(-) create mode 100644 datasette/templates/execute_write.html diff --git a/datasette/app.py b/datasette/app.py index ce85f447..409aed23 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -46,6 +46,7 @@ from .views import Context from .views.database import ( database_download, DatabaseView, + ExecuteWriteView, TableCreateView, QueryView, QueryCreateView, @@ -1249,18 +1250,22 @@ class Datasette: ) return {row["name"]: self._query_row_to_dict(row) for row in rows} - async def ensure_query_write_permissions(self, database, sql, *, actor=None): + async def ensure_query_write_permissions( + self, database, sql, *, actor=None, params=None, analysis=None + ): write_actions = { "insert": "insert-row", "update": "update-row", "delete": "delete-row", } db = self.get_database(database) - params = {name: "" for name in named_parameters(sql)} - try: - analysis = await db.analyze_sql(sql, params) - except sqlite3.DatabaseError as ex: - raise Forbidden(f"Could not analyze query: {ex}") from ex + if analysis is None: + if params is None: + params = {name: "" for name in named_parameters(sql)} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise Forbidden(f"Could not analyze query: {ex}") from ex for access in analysis.table_accesses: action = write_actions.get(access.operation) @@ -2547,6 +2552,10 @@ class Datasette: QueryInsertView.as_view(self), r"/(?P[^\/\.]+)/-/queries/-/insert$", ) + add_route( + ExecuteWriteView.as_view(self), + r"/(?P[^\/\.]+)/-/execute-write$", + ) add_route( DatabaseSchemaView.as_view(self), r"/(?P[^\/\.]+)/-/schema(\.(?Pjson|md))?$", diff --git a/datasette/default_actions.py b/datasette/default_actions.py index e0e0aee5..6787b80e 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -48,6 +48,13 @@ def register_actions(): resource_class=DatabaseResource, also_requires="view-database", ), + Action( + name="execute-write-sql", + abbr="ews", + description="Execute writable SQL queries", + resource_class=DatabaseResource, + also_requires="view-database", + ), Action( name="create-table", abbr="ct", diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html new file mode 100644 index 00000000..5b4f30d9 --- /dev/null +++ b/datasette/templates/execute_write.html @@ -0,0 +1,71 @@ +{% extends "base.html" %} + +{% block title %}Execute write SQL{% endblock %} + +{% block extra_head %} +{{- super() -}} +{% include "_codemirror.html" %} +{% endblock %} + +{% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %} + +{% block crumbs %} +{{ crumbs.nav(request=request, database=database) }} +{% endblock %} + +{% block content %} + +

Execute write SQL

+ +{% if execution_message %} +

{{ execution_message }}

+{% endif %} + +
+

+ + {% if parameter_names %} +

Parameters

+ {% for parameter in parameter_names %} +

+ {% endfor %} + {% endif %} + +

Analysis

+ {% if analysis_error %} +

{{ analysis_error }}

+ {% elif analysis_rows %} +
+ + + + + + + + + + + + {% for row in analysis_rows %} + + + + + + + + + {% endfor %} + +
OperationDatabaseTablerequired permissionAllowedSource
{{ row.operation }}{{ row.database }}{{ row.table }}{{ row.required_permission }}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}{{ row.source or "" }}
+ {% else %} +

Analysis will show each affected table and required permission.

+ {% endif %} + +

+
+ +{% include "_codemirror_foot.html" %} + +{% endblock %} diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index 0e6a7b37..1b3d30a8 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -30,6 +30,9 @@ {% if can_publish %}

{% endif %} + {% if sql and analysis_is_write %} +

Execute write SQL

+ {% endif %}

Analysis

{% if analysis_error %} diff --git a/datasette/views/database.py b/datasette/views/database.py index d521f7ad..a90d889e 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -508,6 +508,27 @@ def _coerce_query_parameters(value, derived): return parameters +def _analysis_is_write(analysis): + return any( + access.operation in {"insert", "update", "delete"} + for access in analysis.table_accesses + ) + + +def _block_framing(response): + response.headers["Content-Security-Policy"] = "frame-ancestors 'none'" + response.headers["X-Frame-Options"] = "DENY" + return response + + +def _wants_json(request, is_json, data): + return ( + is_json + or request.headers.get("accept") == "application/json" + or (isinstance(data, dict) and data.get("_json")) + ) + + async def _json_or_form_payload(request): content_type = request.headers.get("content-type", "") if content_type.startswith("application/json"): @@ -538,15 +559,14 @@ async def _analyze_user_query(datasette, db, sql, *, actor, published): except sqlite3.DatabaseError as ex: raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex - is_write = any( - access.operation in {"insert", "update", "delete"} - for access in analysis.table_accesses - ) + is_write = _analysis_is_write(analysis) if is_write: if published: raise QueryValidationError("Writable queries cannot be published") try: - await datasette.ensure_query_write_permissions(db.name, sql, actor=actor) + await datasette.ensure_query_write_permissions( + db.name, sql, actor=actor, analysis=analysis + ) except Forbidden as ex: raise QueryValidationError(str(ex), status=403) from ex else: @@ -575,6 +595,69 @@ def _analysis_rows(analysis): ] +async def _analysis_rows_with_permissions(datasette, analysis, actor): + rows = _analysis_rows(analysis) + for row in rows: + permission = row["required_permission"] + if permission: + row["allowed"] = await datasette.allowed( + action=permission, + resource=TableResource(row["database"], row["table"]), + actor=actor, + ) + else: + row["allowed"] = None + return rows + + +def _coerce_execute_write_payload(data, is_json): + if not isinstance(data, dict): + raise QueryValidationError("JSON must be a dictionary") + if is_json: + invalid_keys = set(data) - {"sql", "params"} + if invalid_keys: + raise QueryValidationError( + "Invalid keys: {}".format(", ".join(sorted(invalid_keys))) + ) + params = data.get("params") or {} + else: + params = { + key: value + for key, value in data.items() + if key not in {"sql", "csrftoken", "_json"} + } + if not isinstance(params, dict): + raise QueryValidationError("params must be a dictionary") + return data.get("sql"), params + + +async def _prepare_execute_write(datasette, db, sql, params, actor): + if not sql or not isinstance(sql, str): + raise QueryValidationError("SQL is required") + parameter_names = _derived_query_parameters(sql) + extra_params = set(params) - set(parameter_names) + if extra_params: + raise QueryValidationError( + "Unknown parameters: {}".format(", ".join(sorted(extra_params))) + ) + params = {name: params.get(name, "") for name in parameter_names} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex + if not _analysis_is_write(analysis): + raise QueryValidationError( + "Use /-/query for read-only SQL; this endpoint only executes writes" + ) + try: + await datasette.ensure_query_write_permissions( + db.name, sql, actor=actor, analysis=analysis + ) + except Forbidden as ex: + raise QueryValidationError(str(ex), status=403) from ex + return parameter_names, params, analysis + + def _apply_query_data_types(data): typed = dict(data) for key in ("hide_sql", "published"): @@ -707,6 +790,160 @@ async def _prepare_query_update(datasette, request, db, existing, update): return update_kwargs +class ExecuteWriteView(BaseView): + name = "execute-write" + has_json_alternate = False + + async def _render_form( + self, + request, + db, + *, + sql="", + parameter_values=None, + analysis=None, + analysis_error=None, + execution_message=None, + execution_ok=None, + status=200, + ): + parameter_values = parameter_values or {} + parameter_names = [] + analysis_rows = [] + if sql and analysis_error is None: + try: + parameter_names = _derived_query_parameters(sql) + if analysis is None: + params = {parameter: "" for parameter in parameter_names} + analysis = await db.analyze_sql(sql, params) + if _analysis_is_write(analysis): + analysis_rows = await _analysis_rows_with_permissions( + self.ds, analysis, request.actor + ) + else: + analysis_error = ( + "Use /-/query for read-only SQL; " + "this endpoint only executes writes" + ) + except (QueryValidationError, sqlite3.DatabaseError) as ex: + analysis_error = getattr(ex, "message", str(ex)) + + response = await self.render( + ["execute_write.html"], + request, + { + "database": db.name, + "database_color": db.color, + "sql": sql, + "parameter_names": parameter_names, + "parameter_values": parameter_values, + "analysis_error": analysis_error, + "analysis_rows": analysis_rows, + "execution_message": execution_message, + "execution_ok": execution_ok, + "execute_disabled": bool( + (not sql) + or analysis_error + or any(row["allowed"] is False for row in analysis_rows) + ), + }, + ) + response.status = status + return _block_framing(response) + + async def get(self, request): + db = await self.ds.resolve_database(request) + await self.ds.ensure_permission( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ) + return await self._render_form( + request, + db, + sql=request.args.get("sql") or "", + ) + + async def post(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing( + _error(["Permission denied: need execute-write-sql"], 403) + ) + if not db.is_mutable: + return _block_framing(_error(["Database is immutable"], 403)) + + data = {} + is_json = request.headers.get("content-type", "").startswith("application/json") + sql = "" + provided_params = {} + try: + data, is_json = await _json_or_form_payload(request) + sql, provided_params = _coerce_execute_write_payload(data, is_json) + parameter_names, params, analysis = await _prepare_execute_write( + self.ds, db, sql, provided_params, request.actor + ) + except QueryValidationError as ex: + if _wants_json(request, is_json, data): + return _block_framing(_error([ex.message], ex.status)) + return await self._render_form( + request, + db, + sql=sql or "", + parameter_values=provided_params, + analysis_error=ex.message, + execution_message=ex.message, + execution_ok=False, + status=ex.status, + ) + + try: + cursor = await db.execute_write(sql, params, request=request) + except sqlite3.DatabaseError as ex: + message = str(ex) + if _wants_json(request, is_json, data): + return _block_framing(_error([message], 400)) + return await self._render_form( + request, + db, + sql=sql, + parameter_values=params, + analysis=analysis, + execution_message=message, + execution_ok=False, + status=400, + ) + + message = "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) + if _wants_json(request, is_json, data): + return _block_framing( + Response.json( + { + "ok": True, + "message": message, + "rowcount": cursor.rowcount, + "analysis": _analysis_rows(analysis), + } + ) + ) + + return await self._render_form( + request, + db, + sql=sql, + parameter_values={name: params.get(name, "") for name in parameter_names}, + analysis=analysis, + execution_message=message, + execution_ok=True, + ) + + class QueryListView(BaseView): name = "query-list" @@ -753,18 +990,9 @@ class QueryCreateView(BaseView): parameter_names = _derived_query_parameters(sql) params = {parameter: "" for parameter in parameter_names} analysis = await db.analyze_sql(sql, params) - rows = _analysis_rows(analysis) - for row in rows: - permission = row["required_permission"] - if permission: - row["allowed"] = await self.ds.allowed( - action=permission, - resource=TableResource(row["database"], row["table"]), - actor=request.actor, - ) - else: - row["allowed"] = None - analysis_rows = rows + analysis_rows = await _analysis_rows_with_permissions( + self.ds, analysis, request.actor + ) except (QueryValidationError, sqlite3.DatabaseError) as ex: analysis_error = getattr(ex, "message", str(ex)) @@ -783,6 +1011,10 @@ class QueryCreateView(BaseView): ), "analysis_error": analysis_error, "analysis_rows": analysis_rows, + "analysis_is_write": bool( + analysis_rows + and any(row["required_permission"] for row in analysis_rows) + ), "save_disabled": bool( analysis_error or any(row["allowed"] is False for row in analysis_rows) diff --git a/docs/authentication.rst b/docs/authentication.rst index 543f069b..b6a4cb7e 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1423,13 +1423,23 @@ Actor is allowed to drop a database table. execute-sql ----------- -Actor is allowed to run arbitrary SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100 +Actor is allowed to run arbitrary read-only SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100 ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) See also :ref:`the default_allow_sql setting `. +.. _actions_execute_write_sql: + +execute-write-sql +----------------- + +Actor is allowed to run arbitrary writable SQL queries against a specific database, subject to table-level write permissions such as ``insert-row``, ``update-row`` and ``delete-row``. + +``resource`` - ``datasette.resources.DatabaseResource(database)`` + ``database`` is the name of the database (string) + .. _actions_permissions_debug: permissions-debug diff --git a/docs/json_api.rst b/docs/json_api.rst index d5cd231c..e4c9e86e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -526,6 +526,15 @@ Creating saved queries ``POST //-/queries/-/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. +.. _ExecuteWriteView: + +Executing write SQL +~~~~~~~~~~~~~~~~~~~ + +``GET //-/execute-write`` displays a form for executing writable SQL. A ``?sql=`` query string pre-populates the form without executing it. + +``POST //-/execute-write`` executes writable SQL. This requires ``execute-write-sql`` for the database plus the relevant table-level write permissions. + .. _QueryDefinitionView: Getting a saved query definition diff --git a/tests/test_queries.py b/tests/test_queries.py index c6685d6c..05bc5ee1 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -212,6 +212,7 @@ async def test_query_actions_are_registered(): ds = Datasette() await ds.invoke_startup() + assert ds.get_action("execute-write-sql").resource_class is DatabaseResource assert ds.get_action("insert-query").resource_class is DatabaseResource assert ds.get_action("publish-query").resource_class is DatabaseResource assert ds.get_action("update-query").resource_class is QueryResource @@ -492,6 +493,127 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): assert "/data/-/queries/-/create?sql=select+%2A+from+dogs" in query_response.text +@pytest.mark.asyncio +async def test_execute_write_get_prepopulates_without_executing(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_get", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.get( + "/data/-/execute-write?sql=insert+into+dogs+(name)+values+('Cleo')", + actor={"id": "root"}, + ) + + assert response.status_code == 200 + assert response.headers["content-security-policy"] == "frame-ancestors 'none'" + assert response.headers["x-frame-options"] == "DENY" + assert "Execute write SQL" in response.text + assert 'action="/data/-/execute-write"' in response.text + assert "insert into dogs (name) values ('Cleo')" in response.text + assert (await db.execute("select count(*) from dogs")).first()[0] == 0 + + +@pytest.mark.asyncio +async def test_execute_write_post_requires_database_and_table_permissions(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + db = ds.add_memory_database("execute_write_permissions", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + no_database_permission = await ds.client.post( + "/data/-/execute-write", + actor={"id": "outsider"}, + json={ + "sql": "insert into dogs (name) values (:name)", + "params": {"name": "Cleo"}, + }, + ) + no_table_permission = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": "insert into dogs (name) values (:name)", + "params": {"name": "Cleo"}, + }, + ) + + assert no_database_permission.status_code == 403 + assert no_database_permission.json()["errors"] == [ + "Permission denied: need execute-write-sql" + ] + assert no_table_permission.status_code == 403 + assert no_table_permission.json()["errors"] == [ + "Permission denied: need insert-row on data/dogs" + ] + + ds.config = { + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + }, + "tables": { + "dogs": { + "permissions": { + "insert-row": {"id": "writer"}, + } + } + }, + } + } + } + allowed = await ds.client.post( + "/data/-/execute-write", + actor={"id": "writer"}, + json={ + "sql": "insert into dogs (name) values (:name)", + "params": {"name": "Cleo"}, + }, + ) + + assert allowed.status_code == 200 + assert allowed.json()["ok"] is True + assert allowed.json()["rowcount"] == 1 + assert allowed.json()["analysis"][0]["operation"] == "insert" + assert (await db.execute("select name from dogs")).first()[0] == "Cleo" + + +@pytest.mark.asyncio +async def test_execute_write_post_rejects_read_only_sql(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_read_only", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + json={"sql": "select * from dogs"}, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == [ + "Use /-/query for read-only SQL; this endpoint only executes writes" + ] + + @pytest.mark.asyncio async def test_query_owner_gets_update_delete_and_writable_view_defaults(): ds = Datasette(memory=True, default_deny=True) From b7505a9fc22fd96f0c6aad60c8b149bc1978d7b0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 08:49:18 -0700 Subject: [PATCH 101/156] Add execute write SQL database action Refs #2735 --- datasette/default_database_actions.py | 22 +++++++++++++++++ datasette/plugins.py | 1 + tests/test_queries.py | 34 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 datasette/default_database_actions.py diff --git a/datasette/default_database_actions.py b/datasette/default_database_actions.py new file mode 100644 index 00000000..78055392 --- /dev/null +++ b/datasette/default_database_actions.py @@ -0,0 +1,22 @@ +from datasette import hookimpl +from datasette.resources import DatabaseResource + + +@hookimpl +def database_actions(datasette, actor, database, request): + async def inner(): + if not await datasette.allowed( + action="execute-write-sql", + resource=DatabaseResource(database), + actor=actor, + ): + return [] + return [ + { + "href": datasette.urls.database(database) + "/-/execute-write", + "label": "Execute write SQL", + "description": "Run writable SQL with table permission checks.", + } + ] + + return inner diff --git a/datasette/plugins.py b/datasette/plugins.py index f532ac60..5a31cdad 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -30,6 +30,7 @@ DEFAULT_PLUGINS = ( "datasette.blob_renderer", "datasette.default_debug_menu", "datasette.default_jump_items", + "datasette.default_database_actions", "datasette.handle_exception", "datasette.forbidden", "datasette.events", diff --git a/tests/test_queries.py b/tests/test_queries.py index 05bc5ee1..1c9175cc 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -515,6 +515,40 @@ async def test_execute_write_get_prepopulates_without_executing(): assert (await db.execute("select count(*) from dogs")).first()[0] == 0 +@pytest.mark.asyncio +async def test_database_action_menu_links_to_execute_write_for_permitted_actor(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": { + "id": ["writer", "viewer"], + }, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + ds.add_memory_database("execute_write_menu", name="data") + await ds.invoke_startup() + + anonymous_response = await ds.client.get("/data") + viewer_response = await ds.client.get("/data", actor={"id": "viewer"}) + writer_response = await ds.client.get("/data", actor={"id": "writer"}) + + assert anonymous_response.status_code == 403 + assert viewer_response.status_code == 200 + assert "Execute write SQL" not in viewer_response.text + assert writer_response.status_code == 200 + assert "Database actions" in writer_response.text + assert 'href="/data/-/execute-write"' in writer_response.text + assert "Execute write SQL" in writer_response.text + + @pytest.mark.asyncio async def test_execute_write_post_requires_database_and_table_permissions(): ds = Datasette( From e0d39ba69f677be1af1cf580beb83dbc56c8ef87 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 09:41:32 -0700 Subject: [PATCH 102/156] Store query options as JSON Refs #2735 --- datasette/app.py | 105 ++++++++++++++++++++++++--------- datasette/utils/internal_db.py | 8 +-- docs/internals.rst | 20 +++++++ queries-plan.md | 19 +++--- tests/test_queries.py | 45 +++++++++++--- 5 files changed, 143 insertions(+), 54 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 409aed23..023568dd 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -283,6 +283,16 @@ FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" DEFAULT_NOT_SET = object() UNCHANGED = object() +QUERY_OPTION_FIELDS = ( + "hide_sql", + "fragment", + "on_success_message", + "on_success_message_sql", + "on_success_redirect", + "on_error_message", + "on_error_redirect", +) + ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) @@ -1056,6 +1066,7 @@ class Datasette: if row is None: return None parameters = json.loads(row["parameters"] or "[]") + options = json.loads(row["options"] or "{}") is_write = bool(row["is_write"]) return { "database": row["database_name"], @@ -1064,8 +1075,8 @@ class Datasette: "title": row["title"], "description": row["description"], "description_html": row["description_html"], - "hide_sql": bool(row["hide_sql"]), - "fragment": row["fragment"], + "hide_sql": bool(options.get("hide_sql")), + "fragment": options.get("fragment"), "params": parameters, "parameters": parameters, "is_write": is_write, @@ -1073,13 +1084,25 @@ class Datasette: "published": bool(row["published"]), "source": row["source"], "owner_id": row["owner_id"], - "on_success_message": row["on_success_message"], - "on_success_message_sql": row["on_success_message_sql"], - "on_success_redirect": row["on_success_redirect"], - "on_error_message": row["on_error_message"], - "on_error_redirect": row["on_error_redirect"], + "on_success_message": options.get("on_success_message"), + "on_success_message_sql": options.get("on_success_message_sql"), + "on_success_redirect": options.get("on_success_redirect"), + "on_error_message": options.get("on_error_message"), + "on_error_redirect": options.get("on_error_redirect"), } + @staticmethod + def _query_options_json(options): + options_dict = {} + for field in QUERY_OPTION_FIELDS: + value = options.get(field) + if field == "hide_sql": + if value: + options_dict[field] = True + elif value is not None: + options_dict[field] = value + return json.dumps(options_dict, sort_keys=True) + async def add_query( self, database, @@ -1104,13 +1127,22 @@ class Datasette: replace=True, ): parameters_json = json.dumps(list(parameters or [])) + options_json = self._query_options_json( + { + "hide_sql": hide_sql, + "fragment": fragment, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + ) sql_statement = """ INSERT INTO queries ( database_name, name, sql, title, description, description_html, - hide_sql, fragment, parameters, is_write, published, source, - owner_id, on_success_message, on_success_message_sql, - on_success_redirect, on_error_message, on_error_redirect - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + options, parameters, is_write, published, source, owner_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ if replace: sql_statement += """ @@ -1119,18 +1151,12 @@ class Datasette: title = excluded.title, description = excluded.description, description_html = excluded.description_html, - hide_sql = excluded.hide_sql, - fragment = excluded.fragment, + options = excluded.options, parameters = excluded.parameters, is_write = excluded.is_write, published = excluded.published, source = excluded.source, owner_id = excluded.owner_id, - on_success_message = excluded.on_success_message, - on_success_message_sql = excluded.on_success_message_sql, - on_success_redirect = excluded.on_success_redirect, - on_error_message = excluded.on_error_message, - on_error_redirect = excluded.on_error_redirect, updated_at = CURRENT_TIMESTAMP """ await self.get_internal_database().execute_write( @@ -1142,18 +1168,12 @@ class Datasette: title, description, description_html, - int(bool(hide_sql)), - fragment, + options_json, parameters_json, int(bool(is_write)), int(bool(published)), source, owner_id, - on_success_message, - on_success_message_sql, - on_success_redirect, - on_error_message, - on_error_redirect, ], ) @@ -1184,13 +1204,15 @@ class Datasette: "title": title, "description": description, "description_html": description_html, - "hide_sql": hide_sql, - "fragment": fragment, "parameters": parameters, "is_write": is_write, "published": published, "source": source, "owner_id": owner_id, + } + option_fields = { + "hide_sql": hide_sql, + "fragment": fragment, "on_success_message": on_success_message, "on_success_message_sql": on_success_message_sql, "on_success_redirect": on_success_redirect, @@ -1202,12 +1224,39 @@ class Datasette: for field, value in fields.items(): if value is UNCHANGED: continue - if field in {"hide_sql", "is_write", "published"}: + if field in {"is_write", "published"}: value = int(bool(value)) elif field == "parameters": value = json.dumps(list(value or [])) updates.append(f"{field} = ?") params.append(value) + changed_options = { + field: value + for field, value in option_fields.items() + if value is not UNCHANGED + } + if changed_options: + rows = await self.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + row = rows.first() + options = json.loads(row["options"] or "{}") if row is not None else {} + for field, value in changed_options.items(): + if field == "hide_sql": + if value: + options[field] = True + else: + options.pop(field, None) + elif value is None: + options.pop(field, None) + else: + options[field] = value + updates.append("options = ?") + params.append(json.dumps(options, sort_keys=True)) if not updates: return updates.append("updated_at = CURRENT_TIMESTAMP") diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 9008c083..854e8784 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -120,18 +120,12 @@ async def initialize_metadata_tables(db): title TEXT, description TEXT, description_html TEXT, - hide_sql INTEGER NOT NULL DEFAULT 0 CHECK (hide_sql IN (0, 1)), - fragment TEXT, + options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), published INTEGER NOT NULL DEFAULT 0 CHECK (published IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, - on_success_message TEXT, - on_success_message_sql TEXT, - on_success_redirect TEXT, - on_error_message TEXT, - on_error_redirect TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (database_name, name), diff --git a/docs/internals.rst b/docs/internals.rst index e0123a7b..a0845ade 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2148,6 +2148,26 @@ The internal database schema is as follows: config TEXT, PRIMARY KEY (database_name, resource_name, column_name) ); + CREATE TABLE queries ( + database_name TEXT NOT NULL, + name TEXT NOT NULL, + sql TEXT NOT NULL, + title TEXT, + description TEXT, + description_html TEXT, + options TEXT NOT NULL DEFAULT '{}', + parameters TEXT NOT NULL DEFAULT '[]', + is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), + published INTEGER NOT NULL DEFAULT 0 CHECK (published IN (0, 1)), + source TEXT NOT NULL DEFAULT 'user', + owner_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (database_name, name), + CHECK (is_write = 0 OR published = 0) + ); + CREATE INDEX queries_owner_idx + ON queries(owner_id); .. [[[end]]] diff --git a/queries-plan.md b/queries-plan.md index 283ca866..dbc46101 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -42,18 +42,12 @@ CREATE TABLE IF NOT EXISTS queries ( title TEXT, description TEXT, description_html TEXT, - hide_sql INTEGER NOT NULL DEFAULT 0 CHECK (hide_sql IN (0, 1)), - fragment TEXT, + options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), published INTEGER NOT NULL DEFAULT 0 CHECK (published IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, - on_success_message TEXT, - on_success_message_sql TEXT, - on_success_redirect TEXT, - on_error_message TEXT, - on_error_redirect TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (database_name, name), @@ -67,9 +61,10 @@ CREATE INDEX IF NOT EXISTS queries_owner_idx Column notes: - `database_name`, `name`, and `sql` are the routing and execution core. -- Display fields become columns: `title`, `description`, `description_html`, `hide_sql`, and `fragment`. +- Display fields become columns: `title`, `description`, and `description_html`. +- Less common presentation and writable-query behavior lives in `options`, stored as a JSON object. That covers `hide_sql`, `fragment`, `on_success_message`, `on_success_message_sql`, `on_success_redirect`, `on_error_message`, and `on_error_redirect`. - `parameters` is a JSON array of parameter names, stored as text. This preserves explicit parameter order, but does not support labels or default values. -- Existing writable query behavior gets columns too: `is_write`, success/error messages, success/error redirects, and `on_success_message_sql`. +- Existing writable query behavior gets `is_write` as a column. Success/error messages, success/error redirects, and `on_success_message_sql` are stored in `options`. - `published` only applies to read-only queries. A writable query can still be public through explicit `view-query` permissions, but the "publish for users without execute-sql" shortcut should be read-only. - `source` distinguishes `user`, `config`, and `plugin` rows. - `owner_id` is the actor id for user-created rows. It is `NULL` for config/plugin rows. @@ -372,11 +367,11 @@ await datasette.update_query( ) ``` -That call should set `on_success_redirect` to SQL `NULL`; omitting `on_success_redirect` should leave the existing value unchanged. +For column-backed fields, `None` should write SQL `NULL`. For option fields, `None` should remove that key from the JSON object so `get_query()` returns `None`; omitting the field should leave the existing option unchanged. Implementation detail: build the `UPDATE` statement dynamically from fields whose value is not `UNCHANGED`, validate non-nullable fields before writing, and update `updated_at` whenever at least one field changes. -The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `published`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. +The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `published`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. ## Query page save UI @@ -430,7 +425,7 @@ The existing edit-SQL flow from query pages can continue to point back to arbitr - Query update uses `POST /{database}/{query}/-/update` with an `{"update": {...}}` body. - Query delete uses `POST /{database}/{query}/-/delete`. - There are no `PATCH` or HTTP `DELETE` routes for query management. -- `datasette.update_query(..., field=None)` writes `NULL`, while omitted fields are left unchanged. +- `datasette.update_query(..., field=None)` writes `NULL` for column-backed fields and removes JSON keys for option fields, while omitted fields are left unchanged. - Owner gets default `update-query` and `delete-query` for their own user-created rows. - Admin can manage other users' queries with `update-query` and `delete-query`. - User API rejects magic parameters. diff --git a/tests/test_queries.py b/tests/test_queries.py index 1c9175cc..edb9484a 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1,3 +1,5 @@ +import json + import pytest from datasette.app import Datasette @@ -25,18 +27,12 @@ async def test_queries_internal_table_schema(): "title", "description", "description_html", - "hide_sql", - "fragment", + "options", "parameters", "is_write", "published", "source", "owner_id", - "on_success_message", - "on_success_message_sql", - "on_success_redirect", - "on_error_message", - "on_error_redirect", "created_at", "updated_at", ] @@ -62,6 +58,20 @@ async def test_add_get_and_remove_query(): owner_id="alice", ) + options_row = ( + await ds.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + ["data", "top_customers"], + ) + ).first() + assert json.loads(options_row["options"]) == { + "fragment": "chart", + "hide_sql": True, + } + query = await ds.get_query("data", "top_customers") assert query == { "database": "data", @@ -108,6 +118,17 @@ async def test_update_query_only_updates_provided_fields(): parameters=["one"], ) + options_row = ( + await ds.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + ["data", "redirect"], + ) + ).first() + assert json.loads(options_row["options"]) == {"on_success_redirect": "/original"} + await ds.update_query( "data", "redirect", @@ -123,6 +144,16 @@ async def test_update_query_only_updates_provided_fields(): assert query["on_success_redirect"] is None assert query["sql"] == "select 1" assert query["published"] is False + options_row = ( + await ds.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + ["data", "redirect"], + ) + ).first() + assert json.loads(options_row["options"]) == {} @pytest.mark.asyncio From e62a5ea3378095832b0388ac5c6014c23127a577 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 09:46:39 -0700 Subject: [PATCH 103/156] Rename query publication flag Refs #2735 --- datasette/app.py | 18 ++++----- datasette/default_permissions/defaults.py | 4 +- datasette/templates/query_create.html | 2 +- datasette/utils/internal_db.py | 4 +- datasette/views/database.py | 26 ++++++------- docs/internals.rst | 4 +- queries-plan.md | 46 +++++++++++------------ tests/test_queries.py | 22 +++++------ 8 files changed, 63 insertions(+), 63 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 023568dd..40877802 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -615,7 +615,7 @@ class Datasette: fragment=query_config.get("fragment"), parameters=query_config.get("params"), is_write=bool(query_config.get("write")), - published=bool(query_config.get("published")), + is_published=bool(query_config.get("is_published")), source="config", on_success_message=query_config.get("on_success_message"), on_success_message_sql=query_config.get("on_success_message_sql"), @@ -1081,7 +1081,7 @@ class Datasette: "parameters": parameters, "is_write": is_write, "write": is_write, - "published": bool(row["published"]), + "is_published": bool(row["is_published"]), "source": row["source"], "owner_id": row["owner_id"], "on_success_message": options.get("on_success_message"), @@ -1116,7 +1116,7 @@ class Datasette: fragment=None, parameters=None, is_write=False, - published=False, + is_published=False, source="plugin", owner_id=None, on_success_message=None, @@ -1141,7 +1141,7 @@ class Datasette: sql_statement = """ INSERT INTO queries ( database_name, name, sql, title, description, description_html, - options, parameters, is_write, published, source, owner_id + options, parameters, is_write, is_published, source, owner_id ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ if replace: @@ -1154,7 +1154,7 @@ class Datasette: options = excluded.options, parameters = excluded.parameters, is_write = excluded.is_write, - published = excluded.published, + is_published = excluded.is_published, source = excluded.source, owner_id = excluded.owner_id, updated_at = CURRENT_TIMESTAMP @@ -1171,7 +1171,7 @@ class Datasette: options_json, parameters_json, int(bool(is_write)), - int(bool(published)), + int(bool(is_published)), source, owner_id, ], @@ -1190,7 +1190,7 @@ class Datasette: fragment=UNCHANGED, parameters=UNCHANGED, is_write=UNCHANGED, - published=UNCHANGED, + is_published=UNCHANGED, source=UNCHANGED, owner_id=UNCHANGED, on_success_message=UNCHANGED, @@ -1206,7 +1206,7 @@ class Datasette: "description_html": description_html, "parameters": parameters, "is_write": is_write, - "published": published, + "is_published": is_published, "source": source, "owner_id": owner_id, } @@ -1224,7 +1224,7 @@ class Datasette: for field, value in fields.items(): if value is UNCHANGED: continue - if field in {"is_write", "published"}: + if field in {"is_write", "is_published"}: value = int(bool(value)) elif field == "parameters": value = json.dumps(list(value or [])) diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 9737de96..58deea01 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -136,7 +136,7 @@ async def default_query_permissions_sql( 'published query' AS reason FROM queries WHERE is_write = 0 - AND published = 1 + AND is_published = 1 UNION ALL SELECT q.database_name AS parent, q.name AS child, 1 AS allow, 'execute-sql allows query' AS reason @@ -145,7 +145,7 @@ async def default_query_permissions_sql( ON es.parent = q.database_name AND es.child IS NULL WHERE q.is_write = 0 - AND q.published = 0 + AND q.is_published = 0 {trusted_writable_sql} {user_writable_sql} """, diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index 1b3d30a8..fb2599d2 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -28,7 +28,7 @@

{% if can_publish %} -

+

{% endif %} {% if sql and analysis_is_write %}

Execute write SQL

diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 854e8784..0f84e886 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -123,13 +123,13 @@ async def initialize_metadata_tables(db): options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - published INTEGER NOT NULL DEFAULT 0 CHECK (published IN (0, 1)), + is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (database_name, name), - CHECK (is_write = 0 OR published = 0) + CHECK (is_write = 0 OR is_published = 0) ); CREATE INDEX IF NOT EXISTS queries_owner_idx diff --git a/datasette/views/database.py b/datasette/views/database.py index a90d889e..ed38189b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -431,7 +431,7 @@ _query_fields = { "fragment", "parameters", "params", - "published", + "is_published", "on_success_message", "on_success_message_sql", "on_success_redirect", @@ -549,7 +549,7 @@ async def _check_query_name(db, name, *, existing=False): raise QueryValidationError("Query name conflicts with a table or view") -async def _analyze_user_query(datasette, db, sql, *, actor, published): +async def _analyze_user_query(datasette, db, sql, *, actor, is_published): if not sql or not isinstance(sql, str): raise QueryValidationError("SQL is required") derived = _derived_query_parameters(sql) @@ -561,7 +561,7 @@ async def _analyze_user_query(datasette, db, sql, *, actor, published): is_write = _analysis_is_write(analysis) if is_write: - if published: + if is_published: raise QueryValidationError("Writable queries cannot be published") try: await datasette.ensure_query_write_permissions( @@ -660,7 +660,7 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): def _apply_query_data_types(data): typed = dict(data) - for key in ("hide_sql", "published"): + for key in ("hide_sql", "is_published"): if key in typed: typed[key] = _as_bool(typed[key]) return typed @@ -677,15 +677,15 @@ async def _prepare_query_create(datasette, request, db, data): if await datasette.get_query(db.name, name) is not None: raise QueryValidationError("Query already exists") - published = _as_bool(data.get("published")) + is_published = _as_bool(data.get("is_published")) is_write, derived, analysis = await _analyze_user_query( datasette, db, data.get("sql"), actor=request.actor, - published=published, + is_published=is_published, ) - if published and not await datasette.allowed( + if is_published and not await datasette.allowed( action="publish-query", resource=DatabaseResource(db.name), actor=request.actor, @@ -708,7 +708,7 @@ async def _prepare_query_create(datasette, request, db, data): "fragment": data.get("fragment"), "parameters": parameters, "is_write": is_write, - "published": published, + "is_published": is_published, "source": "user", "owner_id": _actor_id(request.actor), "on_success_message": data.get("on_success_message"), @@ -727,7 +727,7 @@ async def _prepare_query_update(datasette, request, db, existing, update): update = _apply_query_data_types(update) sql = update.get("sql", existing["sql"]) - published = update.get("published", existing["published"]) + is_published = update.get("is_published", existing["is_published"]) query_is_write = existing["is_write"] derived = _derived_query_parameters(sql) parameters = None @@ -738,11 +738,11 @@ async def _prepare_query_update(datasette, request, db, existing, update): db, sql, actor=request.actor, - published=published, + is_published=is_published, ) - elif published and query_is_write: + elif is_published and query_is_write: raise QueryValidationError("Writable queries cannot be published") - if published and not existing["published"]: + if is_published and not existing["is_published"]: if not await datasette.allowed( action="publish-query", resource=DatabaseResource(db.name), @@ -772,7 +772,7 @@ async def _prepare_query_update(datasette, request, db, existing, update): "fragment": update.get("fragment"), "parameters": parameters, "is_write": query_is_write, - "published": published, + "is_published": is_published, "on_success_message": update.get("on_success_message"), "on_success_message_sql": update.get("on_success_message_sql"), "on_success_redirect": update.get("on_success_redirect"), diff --git a/docs/internals.rst b/docs/internals.rst index a0845ade..892cf64c 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2158,13 +2158,13 @@ The internal database schema is as follows: options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - published INTEGER NOT NULL DEFAULT 0 CHECK (published IN (0, 1)), + is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (database_name, name), - CHECK (is_write = 0 OR published = 0) + CHECK (is_write = 0 OR is_published = 0) ); CREATE INDEX queries_owner_idx ON queries(owner_id); diff --git a/queries-plan.md b/queries-plan.md index dbc46101..0fbddecd 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -13,7 +13,7 @@ Terminology change: these are now "queries", not "canned queries". Legacy code a - Internal table name: `queries`. - Query definitions should use real columns, not a JSON blob for all options. - Query parameter names live in a `parameters` text column as a JSON array. No default values for parameters in this pass. -- No `queries_database_published_idx` index. +- No `queries_database_is_published_idx` index. - User-created queries require `execute-sql` and `insert-query` on the database. Writable queries additionally require matching table write permissions discovered by `Database.analyze_sql()`. - `publish-query` is the permission for creating or updating a query so users without `execute-sql` can execute it. - Add `update-query` and `delete-query`, so administrators can manage queries created by other users. @@ -45,13 +45,13 @@ CREATE TABLE IF NOT EXISTS queries ( options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - published INTEGER NOT NULL DEFAULT 0 CHECK (published IN (0, 1)), + is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (database_name, name), - CHECK (is_write = 0 OR published = 0) + CHECK (is_write = 0 OR is_published = 0) ); CREATE INDEX IF NOT EXISTS queries_owner_idx @@ -65,11 +65,11 @@ Column notes: - Less common presentation and writable-query behavior lives in `options`, stored as a JSON object. That covers `hide_sql`, `fragment`, `on_success_message`, `on_success_message_sql`, `on_success_redirect`, `on_error_message`, and `on_error_redirect`. - `parameters` is a JSON array of parameter names, stored as text. This preserves explicit parameter order, but does not support labels or default values. - Existing writable query behavior gets `is_write` as a column. Success/error messages, success/error redirects, and `on_success_message_sql` are stored in `options`. -- `published` only applies to read-only queries. A writable query can still be public through explicit `view-query` permissions, but the "publish for users without execute-sql" shortcut should be read-only. +- `is_published` only applies to read-only queries. A writable query can still be public through explicit `view-query` permissions, but the "publish for users without execute-sql" shortcut should be read-only. - `source` distinguishes `user`, `config`, and `plugin` rows. - `owner_id` is the actor id for user-created rows. It is `NULL` for config/plugin rows. -No separate index is needed on `(database_name, name)` because the primary key already creates one. Do not add a `queries_database_published_idx` index for now. +No separate index is needed on `(database_name, name)` because the primary key already creates one. Do not add a `queries_database_is_published_idx` index for now. `QueryResource.resources_sql()` can become: @@ -115,7 +115,7 @@ User-created query creation requires: - `insert-query` on `DatabaseResource(database)` - If analysis shows the query is writable, the table-level write permissions described in the writable query section. -Setting `published=1` requires: +Setting `is_published=1` requires: - `publish-query` on `DatabaseResource(database)` - The query must be read-only according to `Database.analyze_sql()`. @@ -125,7 +125,7 @@ Updating an existing query requires: - `update-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. - If the SQL changes, also require `execute-sql` on the database. - If the changed SQL is writable, also require the table-level write permissions described in the writable query section. -- If `published` changes from `0` to `1`, also require `publish-query` on the database. +- If `is_published` changes from `0` to `1`, also require `publish-query` on the database. Deleting an existing query requires: @@ -140,12 +140,12 @@ Default owner permissions: Default execution rule for read-only queries: -- If `published=0`, the actor needs `execute-sql` on the database. -- If `published=1`, the actor can execute the query without `execute-sql`. +- If `is_published=0`, the actor needs `execute-sql` on the database. +- If `is_published=1`, the actor can execute the query without `execute-sql`. Default execution rule for user-created writable queries: -- `published` must be `0`. +- `is_published` must be `0`. - The actor must have `view-query`. - The actor must currently have every write permission required by fresh `Database.analyze_sql()` results for the query SQL. @@ -153,8 +153,8 @@ Implementation: - Remove `view-query` from the broad `DEFAULT_ALLOW_ACTIONS` set. - Replace it with query-aware default `view-query` permission SQL. -- For `published=1 AND is_write=0`, emit a child-level `view-query` allow. -- For `published=0 AND is_write=0`, emit child-level `view-query` allows for queries whose parent database is in the actor's `execute-sql` allowed resources. +- For `is_published=1 AND is_write=0`, emit a child-level `view-query` allow. +- For `is_published=0 AND is_write=0`, emit child-level `view-query` allows for queries whose parent database is in the actor's `execute-sql` allowed resources. - For `is_write=1 AND source='user'`, emit `view-query` only for the owner or actors with explicit `view-query` permission, then have `QueryView` perform the fresh analysis/table-permission check before execution. - For trusted writable queries, preserve current behavior by emitting child-level `view-query` allows for `is_write=1 AND source IN ('config', 'plugin')` when Datasette is not running with `--default-deny`. @@ -181,7 +181,7 @@ Validation flow for user-created queries: 1. Derive named parameters from the SQL and pass harmless placeholder values into `db.analyze_sql()` so SQLite can prepare statements with bindings. 2. If analysis raises a SQLite error, reject the query. 3. If every table access is `read`, treat the query as read-only and require `execute-sql` plus `insert-query`/`update-query` as described above. -4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `published=0`. +4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `is_published=0`. 5. Reject writable user-created queries that access a database other than the database they are being saved against, until `analyze_sql()` can reliably map attached SQLite schemas back to Datasette database names. 6. For every write access returned by analysis, require the corresponding permission on `TableResource(access.database, access.table)`: - `insert` -> `insert-row` @@ -201,7 +201,7 @@ Fail closed cases for user-created writable queries: - Analysis reports any write operation that cannot be mapped to a Datasette table resource. - Analysis reports writes outside the target database. - The actor lacks any required table write permission. -- `published=1` is requested. +- `is_published=1` is requested. This gives us writable user-created queries without letting `execute-sql` alone become a path to create arbitrary write endpoints. @@ -226,7 +226,7 @@ Create request: "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers", "description": "Highest revenue customers", - "published": false, + "is_published": false, "parameters": ["region"] } } @@ -243,7 +243,7 @@ Successful create returns `201` and the created query definition: "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers", "description": "Highest revenue customers", - "published": false, + "is_published": false, "parameters": ["region"] } } @@ -255,7 +255,7 @@ Update request, imitating `RowUpdateView`: { "update": { "title": "Top customers by revenue", - "published": true + "is_published": true }, "return": true } @@ -271,7 +271,7 @@ Successful update returns `{"ok": true}` by default. With `"return": true`, retu "name": "top_customers", "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers by revenue", - "published": true + "is_published": true } } ``` @@ -318,7 +318,7 @@ await datasette.add_query( fragment=None, parameters=None, is_write=False, - published=False, + is_published=False, source="plugin", owner_id=None, on_success_message=None, @@ -341,7 +341,7 @@ await datasette.update_query( fragment=UNCHANGED, parameters=UNCHANGED, is_write=UNCHANGED, - published=UNCHANGED, + is_published=UNCHANGED, source=UNCHANGED, owner_id=UNCHANGED, on_success_message=UNCHANGED, @@ -371,13 +371,13 @@ For column-backed fields, `None` should write SQL `NULL`. For option fields, `No Implementation detail: build the `UPDATE` statement dynamically from fields whose value is not `UNCHANGED`, validate non-nullable fields before writing, and update `updated_at` whenever at least one field changes. -The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `published`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. +The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `is_published`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. ## Query page save UI On `/{database}/-/query`, if the actor has both `execute-sql` and `insert-query`, show a save control for valid read-only SQL. That page already executes read-only arbitrary SQL, so the first UI can stay read-only even though the JSON API can accept writable SQL after `Database.analyze_sql()` validation. -The save form should call `POST /{database}/-/queries/-/insert` and default to `published=false`. +The save form should call `POST /{database}/-/queries/-/insert` and default to `is_published=false`. If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. @@ -416,7 +416,7 @@ The existing edit-SQL flow from query pages can continue to point back to arbitr - `view-query` is no longer globally default-allowed; default query permissions come from the query-aware hook. - Unpublished read-only query requires `execute-sql` to execute. - Published read-only query can be executed without `execute-sql`. -- Setting `published=true` requires `publish-query`. +- Setting `is_published=true` requires `publish-query`. - User-created query requires both `execute-sql` and `insert-query`. - User-created writable query creation uses `Database.analyze_sql()` and requires matching `insert-row`, `update-row`, and/or `delete-row` permissions for every reported write access. - `/{database}/-/queries/-/create` provides the writable-query authoring UI with an analysis panel and disabled save until all required write permissions pass. diff --git a/tests/test_queries.py b/tests/test_queries.py index edb9484a..df4131b9 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -30,7 +30,7 @@ async def test_queries_internal_table_schema(): "options", "parameters", "is_write", - "published", + "is_published", "source", "owner_id", "created_at", @@ -53,7 +53,7 @@ async def test_add_get_and_remove_query(): hide_sql=True, fragment="chart", parameters=["region"], - published=True, + is_published=True, source="user", owner_id="alice", ) @@ -86,7 +86,7 @@ async def test_add_get_and_remove_query(): "parameters": ["region"], "is_write": False, "write": False, - "published": True, + "is_published": True, "source": "user", "owner_id": "alice", "on_success_message": None, @@ -143,7 +143,7 @@ async def test_update_query_only_updates_provided_fields(): assert query["params"] == [] assert query["on_success_redirect"] is None assert query["sql"] == "select 1" - assert query["published"] is False + assert query["is_published"] is False options_row = ( await ds.get_internal_database().execute( """ @@ -190,7 +190,7 @@ async def test_config_queries_imported_to_internal_table(): "parameters": ["name"], "is_write": False, "write": False, - "published": False, + "is_published": False, "source": "config", "owner_id": None, "on_success_message": None, @@ -218,8 +218,8 @@ async def test_unpublished_query_requires_execute_sql_but_published_does_not(): ds = Datasette(memory=True, settings={"default_allow_sql": False}) ds.add_memory_database("query_permissions", name="data") await ds.invoke_startup() - await ds.add_query("data", "unpublished", "select 1", published=False) - await ds.add_query("data", "published", "select 1", published=True) + await ds.add_query("data", "unpublished", "select 1", is_published=False) + await ds.add_query("data", "published", "select 1", is_published=True) assert not await ds.allowed( action="execute-sql", @@ -347,7 +347,7 @@ async def test_query_list_and_definition_api(): ds.root_enabled = True ds.add_memory_database("query_list_api", name="data") await ds.invoke_startup() - await ds.add_query("data", "listed", "select 1", title="Listed", published=True) + await ds.add_query("data", "listed", "select 1", title="Listed", is_published=True) list_response = await ds.client.get( "/data/-/queries", @@ -387,7 +387,7 @@ async def test_query_insert_api_publish_requires_publish_query(): response = await ds.client.post( "/data/-/queries/-/insert", actor={"id": "writer"}, - json={"query": {"name": "public", "sql": "select 1", "published": True}}, + json={"query": {"name": "public", "sql": "select 1", "is_published": True}}, ) assert response.status_code == 403 @@ -416,7 +416,7 @@ async def test_query_insert_api_creates_writable_query(): assert response.status_code == 201 query = response.json()["query"] assert query["is_write"] is True - assert query["published"] is False + assert query["is_published"] is False assert query["parameters"] == ["name"] bad_response = await ds.client.post( @@ -426,7 +426,7 @@ async def test_query_insert_api_creates_writable_query(): "query": { "name": "published_insert", "sql": "insert into dogs (name) values (:name)", - "published": True, + "is_published": True, } }, ) From 2d07c3b99e654b54c604df4af601ebe27f52b017 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 09:47:12 -0700 Subject: [PATCH 104/156] Ran cog --- datasette/utils/internal_db.py | 3 +-- docs/plugins.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 0f84e886..9c693b0a 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -128,8 +128,7 @@ async def initialize_metadata_tables(db): owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (database_name, name), - CHECK (is_write = 0 OR is_published = 0) + PRIMARY KEY (database_name, name) ); CREATE INDEX IF NOT EXISTS queries_owner_idx diff --git a/docs/plugins.rst b/docs/plugins.rst index 8fa49d6d..d578e9e2 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -216,6 +216,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "register_column_types" ] }, + { + "name": "datasette.default_database_actions", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "database_actions" + ] + }, { "name": "datasette.default_debug_menu", "static": false, From 539ff9ddfcdec0283758138987ddb362485e6ad7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 09:49:21 -0700 Subject: [PATCH 105/156] Drop query publication check from docs Refs #2735 --- docs/internals.rst | 3 +-- queries-plan.md | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 892cf64c..b5da7cbf 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2163,8 +2163,7 @@ The internal database schema is as follows: owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (database_name, name), - CHECK (is_write = 0 OR is_published = 0) + PRIMARY KEY (database_name, name) ); CREATE INDEX queries_owner_idx ON queries(owner_id); diff --git a/queries-plan.md b/queries-plan.md index 0fbddecd..a58ace70 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -50,8 +50,7 @@ CREATE TABLE IF NOT EXISTS queries ( owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (database_name, name), - CHECK (is_write = 0 OR is_published = 0) + PRIMARY KEY (database_name, name) ); CREATE INDEX IF NOT EXISTS queries_owner_idx From 4a70b893559897034625bd797c8fccc80116844a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 10:11:46 -0700 Subject: [PATCH 106/156] Add cursor-paginated query browser Refs #2735 --- datasette/app.py | 129 +++++++++++++++++++++++++--- datasette/templates/database.html | 3 + datasette/templates/query_list.html | 55 ++++++++++++ datasette/views/database.py | 125 ++++++++++++++++++++------- docs/json_api.rst | 2 +- queries-plan.md | 18 +++- tests/test_queries.py | 107 +++++++++++++++++++++-- 7 files changed, 389 insertions(+), 50 deletions(-) create mode 100644 datasette/templates/query_list.html diff --git a/datasette/app.py b/datasette/app.py index 40877802..bdbf9389 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1288,16 +1288,122 @@ class Datasette: ) return self._query_row_to_dict(rows.first()) - async def get_queries(self, database): - rows = await self.get_internal_database().execute( - """ - SELECT * FROM queries - WHERE database_name = ? - ORDER BY name - """, - [database], + async def list_queries( + self, + database, + *, + actor=None, + limit=50, + cursor=None, + q=None, + is_write=None, + is_published=None, + source=None, + owner_id=None, + include_private=False, + ): + limit = min(max(1, int(limit)), 1000) + allowed_sql, allowed_params = await self.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + include_is_private=include_private, ) - return {row["name"]: self._query_row_to_dict(row) for row in rows} + params = dict(allowed_params) + params.update({"query_database": database, "limit": limit + 1}) + sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" + where_clauses = ["q.database_name = :query_database"] + + if cursor: + try: + components = urlsafe_components(cursor) + except ValueError: + components = [] + if len(components) == 2: + where_clauses.append(""" + ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_sort_key"] = components[0] + params["cursor_name"] = components[1] + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_published is not None: + where_clauses.append("q.is_published = :query_is_published") + params["query_is_published"] = int(bool(is_published)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + private_select = ", allowed.is_private AS private" if include_private else "" + rows = list( + ( + await self.get_internal_database().execute( + """ + SELECT q.*, {sort_key_sql} AS sort_key{private_select} + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + ORDER BY sort_key, q.name + LIMIT :limit + """.format( + allowed_sql=allowed_sql, + private_select=private_select, + sort_key_sql=sort_key_sql, + where=" AND ".join(where_clauses), + ), + params, + ) + ).rows + ) + has_more = len(rows) > limit + if has_more: + rows = rows[:limit] + + queries = [] + for row in rows: + query = self._query_row_to_dict(row) + if include_private: + query["private"] = bool(row["private"]) + queries.append(query) + + next_token = None + if has_more and rows: + last_row = rows[-1] + next_token = "{},{}".format( + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + return { + "queries": queries, + "next": next_token, + "has_more": has_more, + "limit": limit, + } async def ensure_query_write_permissions( self, database, sql, *, actor=None, params=None, analysis=None @@ -1564,7 +1670,8 @@ class Datasette: return self.static_hash("app.css") async def get_canned_queries(self, database_name, actor): - return await self.get_queries(database_name) + page = await self.list_queries(database_name, actor=actor, limit=1000) + return {query["name"]: query for query in page["queries"]} async def get_canned_query(self, database_name, query_name, actor): return await self.get_query(database_name, query_name) @@ -2591,7 +2698,7 @@ class Datasette: add_route(TableCreateView.as_view(self), r"/(?P[^\/\.]+)/-/create$") add_route( QueryListView.as_view(self), - r"/(?P[^\/\.]+)/-/queries$", + r"/(?P[^\/\.]+)/-/queries(\.(?Pjson))?$", ) add_route( QueryCreateView.as_view(self), diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 42b4ca0b..a39d6ad7 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -53,6 +53,9 @@
  • {{ query.title or query.name }}{% if query.private %} 🔒{% endif %}
    • {% endfor %} + {% if queries_more %} +

    View all queries

    + {% endif %} {% endif %} {% if tables %} diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html new file mode 100644 index 00000000..ef5da0d5 --- /dev/null +++ b/datasette/templates/query_list.html @@ -0,0 +1,55 @@ +{% extends "base.html" %} + +{% block title %}{{ database }}: queries{% endblock %} + +{% block body_class %}query-list db-{{ database|to_css_class }}{% endblock %} + +{% block crumbs %} +{{ crumbs.nav(request=request, database=database) }} +{% endblock %} + +{% block content %} + +

    Queries

    + +
    +

    + + + +

    +

    + + + + +

    +
    + +{% if queries %} +
      + {% for query in queries %} +
    • + {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} + {% if query.is_write %}Writable{% endif %} + {% if query.is_published %}Published{% endif %} +
      • + {% endfor %} +
    +{% else %} +

    No queries found.

    +{% endif %} + +{% if next_url %} +

    Next page

    +{% endif %} + +{% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index ed38189b..edbc315e 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -92,24 +92,14 @@ class DatabaseView(View): tables = await get_tables(datasette, request, db, allowed_dict) - # Get allowed queries using the new permission system - allowed_query_page = await datasette.allowed_resources( - "view-query", - request.actor, - parent=database, - include_is_private=True, - limit=1000, + queries_page = await datasette.list_queries( + database, + actor=request.actor, + limit=20, + include_private=True, ) - - # Build canned_queries list by looking up each allowed query - all_queries = await datasette.get_canned_queries(database, request.actor) - canned_queries = [] - for query_resource in allowed_query_page.resources: - query_name = query_resource.child - if query_name in all_queries: - canned_queries.append( - dict(all_queries[query_name], private=query_resource.private) - ) + canned_queries = queries_page["queries"] + queries_more = queries_page["has_more"] async def database_actions(): links = [] @@ -141,6 +131,7 @@ class DatabaseView(View): "hidden_count": len([t for t in tables if t["hidden"]]), "views": sql_views, "queries": canned_queries, + "queries_more": queries_more, "allow_execute_sql": allow_execute_sql, "table_columns": ( await _table_columns(datasette, database) if allow_execute_sql else {} @@ -174,6 +165,7 @@ class DatabaseView(View): hidden_count=len([t for t in tables if t["hidden"]]), views=sql_views, queries=canned_queries, + queries_more=queries_more, allow_execute_sql=allow_execute_sql, table_columns=( await _table_columns(datasette, database) @@ -222,6 +214,9 @@ class DatabaseContext(Context): hidden_count: int = field(metadata={"help": "Count of hidden tables"}) views: list = field(metadata={"help": "List of view objects in the database"}) queries: list = field(metadata={"help": "List of canned query objects"}) + queries_more: bool = field( + metadata={"help": "Boolean indicating if more saved queries are available"} + ) allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) @@ -474,6 +469,31 @@ def _as_bool(value): return bool(value) +def _as_optional_bool(value, name): + if value is None or value == "": + return None + if isinstance(value, bool): + return value + if isinstance(value, int): + return bool(value) + if isinstance(value, str): + lowered = value.lower() + if lowered in {"1", "true", "t", "yes", "on"}: + return True + if lowered in {"0", "false", "f", "no", "off"}: + return False + raise QueryValidationError("{} must be 0 or 1".format(name)) + + +def _query_list_limit(value): + if value in (None, ""): + return 50 + try: + return min(max(1, int(value)), 1000) + except ValueError as ex: + raise QueryValidationError("_size must be an integer") from ex + + def _derived_query_parameters(sql): parameters = [] seen = set() @@ -949,19 +969,66 @@ class QueryListView(BaseView): async def get(self, request): db = await self.ds.resolve_database(request) - page = await self.ds.allowed_resources( - "view-query", - request.actor, - parent=db.name, - limit=1000, + format_ = request.url_vars.get("format") or "html" + try: + limit = _query_list_limit(request.args.get("_size")) + is_write = _as_optional_bool(request.args.get("is_write"), "is_write") + is_published = _as_optional_bool( + request.args.get("is_published"), "is_published" + ) + except QueryValidationError as ex: + return _error([ex.message], ex.status) + + page = await self.ds.list_queries( + db.name, + actor=request.actor, + limit=limit, + cursor=request.args.get("_next"), + q=request.args.get("q") or None, + is_write=is_write, + is_published=is_published, + source=request.args.get("source") or None, + owner_id=request.args.get("owner_id") or None, + include_private=True, + ) + next_url = None + if page["next"]: + pairs = [ + (key, value) + for key, value in parse_qsl( + request.query_string, keep_blank_values=True + ) + if key != "_next" + ] + pairs.append(("_next", page["next"])) + next_url = "{}?{}".format( + self.ds.urls.database(db.name) + "/-/queries", + urlencode(pairs), + ) + + data = { + "ok": True, + "database": db.name, + "queries": page["queries"], + "next": page["next"], + "next_url": next_url, + "has_more": page["has_more"], + "limit": page["limit"], + "filters": { + "q": request.args.get("q") or "", + "is_write": request.args.get("is_write") or "", + "is_published": request.args.get("is_published") or "", + "source": request.args.get("source") or "", + "owner_id": request.args.get("owner_id") or "", + }, + } + if format_ == "json": + return Response.json(data) + return await self.render( + ["query_list.html"], + request, + data, ) - all_queries = await self.ds.get_queries(db.name) - queries = [ - all_queries[resource.child] - for resource in page.resources - if resource.child in all_queries - ] - return Response.json({"ok": True, "database": db.name, "queries": queries}) class QueryCreateView(BaseView): diff --git a/docs/json_api.rst b/docs/json_api.rst index e4c9e86e..ece430c2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -510,7 +510,7 @@ Datasette provides a write API for JSON data. This is a POST-only API that requi Listing saved queries ~~~~~~~~~~~~~~~~~~~~~ -``GET //-/queries`` returns saved query definitions the actor can view. +``GET //-/queries.json`` returns saved query definitions the actor can view. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. .. _QueryCreateView: diff --git a/queries-plan.md b/queries-plan.md index a58ace70..671fc29c 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -210,7 +210,7 @@ JSON endpoints should follow Datasette's existing write API style: use `POST` pl Endpoints: -- `GET /{database}/-/queries` lists query definitions the actor can view or manage, probably paginated. +- `GET /{database}/-/queries` shows a searchable HTML query browser. `GET /{database}/-/queries.json` returns query definitions the actor can view, using cursor pagination with `_next` and `_size`. - `POST /{database}/-/queries/-/insert` creates a query. - `GET /{database}/{query}/-/definition` returns one query definition without executing it. - `POST /{database}/{query}/-/update` updates one query. @@ -353,9 +353,21 @@ await datasette.update_query( await datasette.remove_query(database, name, source=None) await datasette.get_query(database, name) -await datasette.get_queries(database) +await datasette.list_queries( + database, + actor=None, + limit=50, + cursor=None, + q=None, + is_write=None, + is_published=None, + source=None, + owner_id=None, +) ``` +`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. + `update_query()` should use an internal sentinel default such as `UNCHANGED = object()` so callers can distinguish "leave this column alone" from "set this column to `NULL`": ```python @@ -380,6 +392,8 @@ The save form should call `POST /{database}/-/queries/-/insert` and default to ` If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. +On `/{database}`, show a preview of the first 20 visible queries using `list_queries(..., limit=20)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. + ## Dedicated create query UI Add `/{database}/-/queries/-/create` for the fuller query authoring flow, including writable queries. diff --git a/tests/test_queries.py b/tests/test_queries.py index df4131b9..dd906faf 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -7,6 +7,20 @@ from datasette.resources import DatabaseResource, QueryResource from datasette.utils.asgi import Forbidden +async def add_numbered_queries(ds, database, count): + for i in range(1, count + 1): + await ds.add_query( + database, + "demo_query_{:02d}".format(i), + "select {} as query_number".format(i), + title="Demo query {:02d}".format(i), + description="Seeded demo query number {:02d}".format(i), + is_published=True, + source="user", + owner_id="root", + ) + + @pytest.mark.asyncio async def test_queries_internal_table_schema(): ds = Datasette(memory=True) @@ -96,11 +110,15 @@ async def test_add_get_and_remove_query(): "on_error_redirect": None, } - assert await ds.get_queries("data") == {"top_customers": query} + queries_page = await ds.list_queries("data", actor=None) + assert queries_page["queries"] == [query] + assert queries_page["next"] is None await ds.remove_query("data", "top_customers") assert await ds.get_query("data", "top_customers") is None - assert await ds.get_queries("data") == {} + queries_page = await ds.list_queries("data", actor=None) + assert queries_page["queries"] == [] + assert queries_page["next"] is None @pytest.mark.asyncio @@ -238,6 +256,24 @@ async def test_unpublished_query_requires_execute_sql_but_published_does_not(): ) +@pytest.mark.asyncio +async def test_database_page_query_preview_is_limited(): + ds = Datasette(memory=True) + ds.add_memory_database("query_preview", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 25) + + html_response = await ds.client.get("/data") + json_response = await ds.client.get("/data.json") + + assert html_response.status_code == 200 + assert "Demo query 20" in html_response.text + assert "Demo query 21" not in html_response.text + assert 'href="/data/-/queries"' in html_response.text + assert len(json_response.json()["queries"]) == 20 + assert json_response.json()["queries_more"] is True + + @pytest.mark.asyncio async def test_query_actions_are_registered(): ds = Datasette() @@ -347,21 +383,78 @@ async def test_query_list_and_definition_api(): ds.root_enabled = True ds.add_memory_database("query_list_api", name="data") await ds.invoke_startup() - await ds.add_query("data", "listed", "select 1", title="Listed", is_published=True) + await add_numbered_queries(ds, "data", 12) list_response = await ds.client.get( - "/data/-/queries", + "/data/-/queries.json?_size=5", + actor={"id": "root"}, + ) + next_response = await ds.client.get( + "/data/-/queries.json?_size=5&_next={}".format(list_response.json()["next"]), actor={"id": "root"}, ) definition_response = await ds.client.get( - "/data/listed/-/definition", + "/data/demo_query_01/-/definition", actor={"id": "root"}, ) assert list_response.status_code == 200 - assert list_response.json()["queries"][0]["name"] == "listed" + assert [query["name"] for query in list_response.json()["queries"]] == [ + "demo_query_01", + "demo_query_02", + "demo_query_03", + "demo_query_04", + "demo_query_05", + ] + assert list_response.json()["next"] + assert [query["name"] for query in next_response.json()["queries"]] == [ + "demo_query_06", + "demo_query_07", + "demo_query_08", + "demo_query_09", + "demo_query_10", + ] assert definition_response.status_code == 200 - assert definition_response.json()["query"]["title"] == "Listed" + assert definition_response.json()["query"]["title"] == "Demo query 01" + + +@pytest.mark.asyncio +async def test_query_list_search_filter_and_html(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_html", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 3) + await ds.add_query( + "data", + "private_query", + "select 'private'", + title="Private query", + is_published=False, + source="user", + owner_id="root", + ) + + html_response = await ds.client.get( + "/data/-/queries?q=02", + actor={"id": "root"}, + ) + json_response = await ds.client.get( + "/data/-/queries.json?q=02", + actor={"id": "root"}, + ) + filtered_response = await ds.client.get( + "/data/-/queries.json?is_published=0", + actor={"id": "root"}, + ) + + assert html_response.status_code == 200 + assert "Demo query 02" in html_response.text + assert "Demo query 01" not in html_response.text + assert json_response.json()["queries"][0]["name"] == "demo_query_02" + assert [query["name"] for query in filtered_response.json()["queries"]] == [ + "private_query" + ] @pytest.mark.asyncio From 310c36ae94c54d4b859925d4977554c2a2618534 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 10:18:36 -0700 Subject: [PATCH 107/156] Limit database query preview to five Refs #2735 --- datasette/views/database.py | 2 +- queries-plan.md | 2 +- tests/test_canned_queries.py | 35 ++++++++++++++++++++++++++++++----- tests/test_queries.py | 6 +++--- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index edbc315e..353cfcf2 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -95,7 +95,7 @@ class DatabaseView(View): queries_page = await datasette.list_queries( database, actor=request.actor, - limit=20, + limit=5, include_private=True, ) canned_queries = queries_page["queries"] diff --git a/queries-plan.md b/queries-plan.md index 671fc29c..82ef3260 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -392,7 +392,7 @@ The save form should call `POST /{database}/-/queries/-/insert` and default to ` If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. -On `/{database}`, show a preview of the first 20 visible queries using `list_queries(..., limit=20)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. +On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. ## Dedicated create query UI diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index c46fd86f..a9d22036 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -248,10 +248,9 @@ def test_json_response(canned_write_client, headers, body, querystring): def test_canned_query_permissions_on_database_page(canned_write_client): - # Without auth only shows three queries - query_names = { - q["name"] for q in canned_write_client.get("/data.json").json["queries"] - } + # Without auth shows the five public queries + anon_response = canned_write_client.get("/data.json") + query_names = {q["name"] for q in anon_response.json["queries"]} assert query_names == { "add_name_specify_id_with_error_in_on_success_message_sql", "update_name", @@ -259,8 +258,9 @@ def test_canned_query_permissions_on_database_page(canned_write_client): "canned_read", "add_name", } + assert anon_response.json["queries_more"] is False - # With auth shows four + # With auth the database page preview shows the first five queries response = canned_write_client.get( "/data.json", cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, @@ -273,6 +273,31 @@ def test_canned_query_permissions_on_database_page(canned_write_client): ], key=lambda q: q["name"], ) + assert query_names_and_private == [ + {"name": "add_name", "private": False}, + {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, + {"name": "canned_read", "private": False}, + {"name": "delete_name", "private": True}, + ] + assert response.json["queries_more"] is True + + # The full query list endpoint includes the remaining query + response = canned_write_client.get( + "/data/-/queries.json?_size=10", + cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, + ) + assert response.status == 200 + query_names_and_private = sorted( + [ + {"name": q["name"], "private": q["private"]} + for q in response.json["queries"] + ], + key=lambda q: q["name"], + ) assert query_names_and_private == [ {"name": "add_name", "private": False}, {"name": "add_name_specify_id", "private": False}, diff --git a/tests/test_queries.py b/tests/test_queries.py index dd906faf..2b46e00f 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -267,10 +267,10 @@ async def test_database_page_query_preview_is_limited(): json_response = await ds.client.get("/data.json") assert html_response.status_code == 200 - assert "Demo query 20" in html_response.text - assert "Demo query 21" not in html_response.text + assert "Demo query 05" in html_response.text + assert "Demo query 06" not in html_response.text assert 'href="/data/-/queries"' in html_response.text - assert len(json_response.json()["queries"]) == 20 + assert len(json_response.json()["queries"]) == 5 assert json_response.json()["queries_more"] is True From 6eee6c81e8c21737e2391af55baf24866429038d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 10:24:42 -0700 Subject: [PATCH 108/156] Add global query browser Refs #2735 --- datasette/app.py | 57 +++++++++++++++++++----- datasette/templates/query_list.html | 11 +++-- datasette/views/database.py | 27 ++++++++++-- docs/json_api.rst | 3 +- queries-plan.md | 6 +-- tests/test_queries.py | 67 +++++++++++++++++++++++++++++ 6 files changed, 149 insertions(+), 22 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index bdbf9389..c047fde9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -52,6 +52,7 @@ from .views.database import ( QueryCreateView, QueryDeleteView, QueryDefinitionView, + GlobalQueryListView, QueryInsertView, QueryListView, QueryUpdateView, @@ -1290,7 +1291,7 @@ class Datasette: async def list_queries( self, - database, + database=None, *, actor=None, limit=50, @@ -1310,16 +1311,40 @@ class Datasette: include_is_private=include_private, ) params = dict(allowed_params) - params.update({"query_database": database, "limit": limit + 1}) + params.update({"limit": limit + 1}) sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" - where_clauses = ["q.database_name = :query_database"] + where_clauses = [] + order_by = "q.database_name, sort_key, q.name" + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + order_by = "sort_key, q.name" if cursor: try: components = urlsafe_components(cursor) except ValueError: components = [] - if len(components) == 2: + if database is None and len(components) == 3: + where_clauses.append(""" + ( + q.database_name > :cursor_database + OR ( + q.database_name = :cursor_database + AND ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_database"] = components[0] + params["cursor_sort_key"] = components[1] + params["cursor_name"] = components[2] + elif database is not None and len(components) == 2: where_clauses.append(""" ( {sort_key_sql} > :cursor_sort_key @@ -1368,13 +1393,14 @@ class Datasette: ON allowed.parent = q.database_name AND allowed.child = q.name WHERE {where} - ORDER BY sort_key, q.name + ORDER BY {order_by} LIMIT :limit """.format( allowed_sql=allowed_sql, private_select=private_select, sort_key_sql=sort_key_sql, - where=" AND ".join(where_clauses), + where=" AND ".join(where_clauses) or "1 = 1", + order_by=order_by, ), params, ) @@ -1394,10 +1420,17 @@ class Datasette: next_token = None if has_more and rows: last_row = rows[-1] - next_token = "{},{}".format( - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) + if database is None: + next_token = "{},{},{}".format( + tilde_encode(last_row["database_name"]), + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + else: + next_token = "{},{}".format( + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) return { "queries": queries, "next": next_token, @@ -2651,6 +2684,10 @@ class Datasette: JumpView.as_view(self), r"/-/jump(\.(?Pjson))?$", ) + add_route( + GlobalQueryListView.as_view(self), + r"/-/queries(\.(?Pjson))?$", + ) add_route( InstanceSchemaView.as_view(self), r"/-/schema(\.(?Pjson|md))?$", diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index ef5da0d5..af974550 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -1,8 +1,8 @@ {% extends "base.html" %} -{% block title %}{{ database }}: queries{% endblock %} +{% block title %}{% if database %}{{ database }}: {% endif %}queries{% endblock %} -{% block body_class %}query-list db-{{ database|to_css_class }}{% endblock %} +{% block body_class %}query-list{% if database %} db-{{ database|to_css_class }}{% endif %}{% endblock %} {% block crumbs %} {{ crumbs.nav(request=request, database=database) }} @@ -12,7 +12,7 @@

    Queries

    -
    +

    @@ -38,7 +38,10 @@

      {% for query in queries %}
    • - {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} + {% if show_database %} + {{ query.database }}: + {% endif %} + {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} {% if query.is_write %}Writable{% endif %} {% if query.is_published %}Published{% endif %}
      • diff --git a/datasette/views/database.py b/datasette/views/database.py index 353cfcf2..1576b6a9 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -967,8 +967,14 @@ class ExecuteWriteView(BaseView): class QueryListView(BaseView): name = "query-list" + async def database_name(self, request): + return (await self.ds.resolve_database(request)).name + + def query_list_path(self, database): + return self.ds.urls.database(database) + "/-/queries" + async def get(self, request): - db = await self.ds.resolve_database(request) + database = await self.database_name(request) format_ = request.url_vars.get("format") or "html" try: limit = _query_list_limit(request.args.get("_size")) @@ -980,7 +986,7 @@ class QueryListView(BaseView): return _error([ex.message], ex.status) page = await self.ds.list_queries( - db.name, + database, actor=request.actor, limit=limit, cursor=request.args.get("_next"), @@ -991,6 +997,7 @@ class QueryListView(BaseView): owner_id=request.args.get("owner_id") or None, include_private=True, ) + query_list_path = self.query_list_path(database) next_url = None if page["next"]: pairs = [ @@ -1002,18 +1009,20 @@ class QueryListView(BaseView): ] pairs.append(("_next", page["next"])) next_url = "{}?{}".format( - self.ds.urls.database(db.name) + "/-/queries", + query_list_path, urlencode(pairs), ) data = { "ok": True, - "database": db.name, + "database": database, "queries": page["queries"], "next": page["next"], "next_url": next_url, "has_more": page["has_more"], "limit": page["limit"], + "query_list_path": query_list_path, + "show_database": database is None, "filters": { "q": request.args.get("q") or "", "is_write": request.args.get("is_write") or "", @@ -1031,6 +1040,16 @@ class QueryListView(BaseView): ) +class GlobalQueryListView(QueryListView): + name = "global-query-list" + + async def database_name(self, request): + return None + + def query_list_path(self, database): + return self.ds.urls.path("/-/queries") + + class QueryCreateView(BaseView): name = "query-create" has_json_alternate = False diff --git a/docs/json_api.rst b/docs/json_api.rst index ece430c2..f44a39fe 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -505,12 +505,13 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`. +.. _GlobalQueryListView: .. _QueryListView: Listing saved queries ~~~~~~~~~~~~~~~~~~~~~ -``GET //-/queries.json`` returns saved query definitions the actor can view. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. +``GET /-/queries.json`` returns saved query definitions across every database that the actor can view. ``GET //-/queries.json`` returns saved query definitions for a specific database. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. .. _QueryCreateView: diff --git a/queries-plan.md b/queries-plan.md index 82ef3260..a708e887 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -210,7 +210,7 @@ JSON endpoints should follow Datasette's existing write API style: use `POST` pl Endpoints: -- `GET /{database}/-/queries` shows a searchable HTML query browser. `GET /{database}/-/queries.json` returns query definitions the actor can view, using cursor pagination with `_next` and `_size`. +- `GET /-/queries` and `GET /{database}/-/queries` show searchable HTML query browsers. `GET /-/queries.json` lists query definitions across every database the actor can view; `GET /{database}/-/queries.json` scopes that list to one database. Both JSON endpoints use cursor pagination with `_next` and `_size`. - `POST /{database}/-/queries/-/insert` creates a query. - `GET /{database}/{query}/-/definition` returns one query definition without executing it. - `POST /{database}/{query}/-/update` updates one query. @@ -366,7 +366,7 @@ await datasette.list_queries( ) ``` -`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. +`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. Passing `database=None` lists visible queries across all live databases, still filtered through `view-query` permission SQL. `update_query()` should use an internal sentinel default such as `UNCHANGED = object()` so callers can distinguish "leave this column alone" from "set this column to `NULL`": @@ -392,7 +392,7 @@ The save form should call `POST /{database}/-/queries/-/insert` and default to ` If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. -On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. +On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. The global `/-/queries` page reuses the same interface and shows the database for each query. ## Dedicated create query UI diff --git a/tests/test_queries.py b/tests/test_queries.py index 2b46e00f..bc04bb51 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -457,6 +457,73 @@ async def test_query_list_search_filter_and_html(): ] +@pytest.mark.asyncio +async def test_global_query_list_api_and_html(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_global_alpha", name="alpha") + ds.add_memory_database("query_list_global_beta", name="beta") + await ds.invoke_startup() + await ds.add_query( + "alpha", + "alpha_first", + "select 1", + title="Alpha first", + is_published=True, + source="user", + owner_id="root", + ) + await ds.add_query( + "alpha", + "alpha_second", + "select 2", + title="Alpha second", + is_published=True, + source="user", + owner_id="root", + ) + await ds.add_query( + "beta", + "beta_first", + "select 3", + title="Beta first", + is_published=True, + source="user", + owner_id="root", + ) + + list_response = await ds.client.get( + "/-/queries.json?_size=2", + actor={"id": "root"}, + ) + next_response = await ds.client.get( + "/-/queries.json?_size=2&_next={}".format(list_response.json()["next"]), + actor={"id": "root"}, + ) + html_response = await ds.client.get( + "/-/queries?q=Beta", + actor={"id": "root"}, + ) + + assert list_response.status_code == 200 + assert [ + (query["database"], query["name"]) for query in list_response.json()["queries"] + ] == [ + ("alpha", "alpha_first"), + ("alpha", "alpha_second"), + ] + assert list_response.json()["next"] + assert [ + (query["database"], query["name"]) for query in next_response.json()["queries"] + ] == [ + ("beta", "beta_first"), + ] + assert html_response.status_code == 200 + assert 'href="/beta">beta:' in html_response.text + assert "Beta first" in html_response.text + assert "Alpha first" not in html_response.text + + @pytest.mark.asyncio async def test_query_insert_api_publish_requires_publish_query(): ds = Datasette( From f0b59971f7c8c0f4435a18b4f4e9c8053c2683fe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 10:39:56 -0700 Subject: [PATCH 109/156] Delete unnecessary test --- tests/test_utils_sql_analysis.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_utils_sql_analysis.py b/tests/test_utils_sql_analysis.py index c82fb04f..5730cd0d 100644 --- a/tests/test_utils_sql_analysis.py +++ b/tests/test_utils_sql_analysis.py @@ -169,13 +169,6 @@ def test_analyze_attached_database_tables(conn): } -def test_analyze_invalid_sql_cleans_up_authorizer(conn): - with pytest.raises(sqlite3.OperationalError): - analyze_sql_tables(conn, "insert into missing_table values (1)") - - conn.execute("select name from dogs").fetchall() - - def test_analyze_clears_authorizer_on_error(): class FakeConnection: def __init__(self): From 2b5b4ed66b86bae0080e9d8f4881cad8e57bbdb3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 11:11:08 -0700 Subject: [PATCH 110/156] Much improved "Write to this database" UI - Start with a template option, letting you pick table and operation - SQL textarea defaults to 4 empty lines at start - Query operations table is simpler and looks nicer Refs #2742 --- datasette/templates/execute_write.html | 240 +++++++++++++++++++++++-- datasette/views/database.py | 13 +- tests/test_queries.py | 32 +++- 3 files changed, 271 insertions(+), 14 deletions(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 5b4f30d9..90845910 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -1,10 +1,80 @@ {% extends "base.html" %} -{% block title %}Execute write SQL{% endblock %} +{% block title %}Write to this database{% endblock %} {% block extra_head %} {{- super() -}} {% include "_codemirror.html" %} + {% endblock %} {% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %} @@ -15,13 +85,34 @@ {% block content %} -

      Execute write SQL

      +

      Write to this database

      + +

      Execute SQL to insert, update or delete rows in this database.

      {% if execution_message %}

      {{ execution_message }}

      {% endif %} + {% if write_template_tables %} +
      +
      + Start with a template +

      + + + + + +

      +
      +
      + {% endif %} +

      {% if parameter_names %} @@ -31,30 +122,28 @@ {% endfor %} {% endif %} -

      Analysis

      +

      Query operations

      {% if analysis_error %}

      {{ analysis_error }}

      {% elif analysis_rows %} -
      +
      - + - {% for row in analysis_rows %} - - - - - - + + + + + {% endfor %} @@ -66,6 +155,133 @@

      + + {% include "_codemirror_foot.html" %} +{% if write_template_tables %} + +{% endif %} + {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 1576b6a9..fb3bdfdb 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -830,6 +830,13 @@ class ExecuteWriteView(BaseView): parameter_values = parameter_values or {} parameter_names = [] analysis_rows = [] + table_columns = await _table_columns(self.ds, db.name) + hidden_table_names = set(await db.hidden_table_names()) + write_template_tables = { + table: columns + for table, columns in table_columns.items() + if columns and table not in hidden_table_names + } if sql and analysis_error is None: try: parameter_names = _derived_query_parameters(sql) @@ -858,7 +865,9 @@ class ExecuteWriteView(BaseView): "parameter_names": parameter_names, "parameter_values": parameter_values, "analysis_error": analysis_error, - "analysis_rows": analysis_rows, + "analysis_rows": [ + row for row in analysis_rows if row["operation"] != "read" + ], "execution_message": execution_message, "execution_ok": execution_ok, "execute_disabled": bool( @@ -866,6 +875,8 @@ class ExecuteWriteView(BaseView): or analysis_error or any(row["allowed"] is False for row in analysis_rows) ), + "table_columns": table_columns, + "write_template_tables": write_template_tables, }, ) response.status = status diff --git a/tests/test_queries.py b/tests/test_queries.py index bc04bb51..684454fc 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -690,6 +690,14 @@ async def test_execute_write_get_prepopulates_without_executing(): ds.root_enabled = True db = ds.add_memory_database("execute_write_get", name="data") await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write("create table cats (id integer primary key, name text)") + await db.execute_write("create table log (message text)") + await db.execute_write(""" + create trigger dogs_after_insert after insert on dogs begin + update cats set name = new.name where id = new.id; + insert into log (message) values (new.name); + end + """) await ds.invoke_startup() response = await ds.client.get( @@ -700,11 +708,33 @@ async def test_execute_write_get_prepopulates_without_executing(): assert response.status_code == 200 assert response.headers["content-security-policy"] == "frame-ancestors 'none'" assert response.headers["x-frame-options"] == "DENY" - assert "Execute write SQL" in response.text + assert "Write to this database" in response.text + assert ( + "Execute SQL to insert, update or delete rows in this database." + in response.text + ) + assert "

      Query operations

      " in response.text + assert "Start with a template" in response.text + assert '' in response.text + assert 'data-sql-template="insert"' in response.text + assert 'data-sql-template="update"' in response.text + assert 'data-sql-template="delete"' in response.text + assert '
      Operation Database Tablerequired permissionRequired permission AllowedSource
      {{ row.operation }}{{ row.database }}{{ row.table }}{{ row.required_permission }}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}{{ row.source or "" }}{{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% endif %}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}
      ' in response.text + assert '' in response.text + assert "" in response.text + assert "" in response.text + assert "" not in response.text assert 'action="/data/-/execute-write"' in response.text assert "insert into dogs (name) values ('Cleo')" in response.text assert (await db.execute("select count(*) from dogs")).first()[0] == 0 + empty_response = await ds.client.get( + "/data/-/execute-write", + actor={"id": "root"}, + ) + assert '' in empty_response.text + assert 'executeWriteSqlInput.value = "\\n\\n\\n";' in empty_response.text + @pytest.mark.asyncio async def test_database_action_menu_links_to_execute_write_for_permitted_actor(): From 1bce34a33869709e1dea21b6182327a105895285 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 11:22:24 -0700 Subject: [PATCH 111/156] If just a single insert, link to row page Refs #2742 --- datasette/templates/execute_write.html | 2 +- datasette/views/database.py | 49 ++++++++++++++++++++++++++ tests/test_queries.py | 42 ++++++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 90845910..705181d8 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -90,7 +90,7 @@

      Execute SQL to insert, update or delete rows in this database.

      {% if execution_message %} -

      {{ execution_message }}

      +

      {{ execution_message }}{% for link in execution_links %} {{ link.label }}{% endfor %}

      {% endif %}
      diff --git a/datasette/views/database.py b/datasette/views/database.py index fb3bdfdb..2b3920f7 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -18,8 +18,10 @@ from datasette.utils import ( await_me_maybe, call_with_supported_arguments, named_parameters as derive_named_parameters, + escape_sqlite, format_bytes, make_slot_function, + path_from_row_pks, tilde_decode, to_css_class, validate_sql_select, @@ -678,6 +680,43 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis +async def _inserted_row_url(datasette, db, analysis, cursor): + if cursor.rowcount != 1: + return None + lastrowid = getattr(cursor, "lastrowid", None) + if lastrowid is None: + return None + direct_inserts = [ + access + for access in analysis.table_accesses + if access.operation == "insert" + and access.source is None + and access.database == db.name + ] + if len(direct_inserts) != 1: + return None + table = direct_inserts[0].table + pks = await db.primary_keys(table) + use_rowid = not pks + select = ( + "rowid" + if use_rowid + else ", ".join(escape_sqlite(primary_key) for primary_key in pks) + ) + try: + result = await db.execute( + "select {} from {} where rowid = ?".format(select, escape_sqlite(table)), + [lastrowid], + ) + except sqlite3.DatabaseError: + return None + row = result.first() + if row is None: + return None + row_path = path_from_row_pks(row, pks, use_rowid) + return datasette.urls.row(db.name, table, row_path) + + def _apply_query_data_types(data): typed = dict(data) for key in ("hide_sql", "is_published"): @@ -824,10 +863,12 @@ class ExecuteWriteView(BaseView): analysis=None, analysis_error=None, execution_message=None, + execution_links=None, execution_ok=None, status=200, ): parameter_values = parameter_values or {} + execution_links = execution_links or [] parameter_names = [] analysis_rows = [] table_columns = await _table_columns(self.ds, db.name) @@ -869,6 +910,7 @@ class ExecuteWriteView(BaseView): row for row in analysis_rows if row["operation"] != "read" ], "execution_message": execution_message, + "execution_links": execution_links, "execution_ok": execution_ok, "execute_disabled": bool( (not sql) @@ -964,6 +1006,12 @@ class ExecuteWriteView(BaseView): ) ) + inserted_row_url = await _inserted_row_url(self.ds, db, analysis, cursor) + execution_links = ( + [{"href": inserted_row_url, "label": "View row"}] + if inserted_row_url + else [] + ) return await self._render_form( request, db, @@ -971,6 +1019,7 @@ class ExecuteWriteView(BaseView): parameter_values={name: params.get(name, "") for name in parameter_names}, analysis=analysis, execution_message=message, + execution_links=execution_links, execution_ok=True, ) diff --git a/tests/test_queries.py b/tests/test_queries.py index 684454fc..ed981ee7 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -849,6 +849,48 @@ async def test_execute_write_post_requires_database_and_table_permissions(): assert (await db.execute("select name from dogs")).first()[0] == "Cleo" +@pytest.mark.asyncio +async def test_execute_write_insert_links_to_inserted_row(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_insert_link", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await db.execute_write("create table log (id integer primary key, message text)") + await db.execute_write("insert into log (message) values ('existing')") + await db.execute_write(""" + create trigger dogs_after_insert after insert on dogs begin + insert into log (message) values (new.name); + end + """) + await ds.invoke_startup() + + insert_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "insert into dogs (name) values (:name)", + "name": "Cleo", + }, + ) + update_response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "update dogs set name = :name where id = :id", + "name": "Cleo 2", + "id": "1", + }, + ) + + assert insert_response.status_code == 200 + assert "Query executed, 1 row affected" in insert_response.text + assert 'View row' in insert_response.text + assert "/data/log/2" not in insert_response.text + assert update_response.status_code == 200 + assert "Query executed, 1 row affected" in update_response.text + assert "View row" not in update_response.text + + @pytest.mark.asyncio async def test_execute_write_post_rejects_read_only_sql(): ds = Datasette(memory=True, default_deny=True) From 66bbbbc947bd4d7305761a627dc2f1949949c0a5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 11:35:09 -0700 Subject: [PATCH 112/156] Support multi-line parameters on /db/-/execute-write Refs https://github.com/simonw/datasette/issues/2742#issuecomment-4536317049 Each paramater input now has an expand/collapse button toggle to turn into a textarea. If you paste text that includes at least one newline it toggles automatically. --- datasette/templates/execute_write.html | 94 +++++++++++++++++++++++++- tests/test_queries.py | 1 + 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 705181d8..a560e920 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -74,6 +74,25 @@ color: #b00020; font-weight: 700; } +form.sql .execute-write-parameter-row textarea[data-parameter-control] { + border: 1px solid #ccc; + border-radius: 3px; + box-sizing: content-box; + display: inline-block; + font-family: Helvetica, sans-serif; + font-size: 1em; + min-height: 7rem; + padding: 9px 4px; + vertical-align: top; + width: 60%; +} +form.sql.core button.execute-write-parameter-toggle[type=button] { + font-size: 0.72rem; + height: 1.8rem; + line-height: 1; + margin-left: 0.35rem; + padding: 0.25rem 0.45rem; +} {% endblock %} @@ -118,7 +137,7 @@ {% if parameter_names %}

      Parameters

      {% for parameter in parameter_names %} -

      +

      {% endfor %} {% endif %} @@ -164,6 +183,79 @@ if (executeWriteSqlInput && !executeWriteSqlInput.value) { {% include "_codemirror_foot.html" %} + + {% if write_template_tables %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 2b3920f7..e4eaee30 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -680,6 +680,39 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis +async def _execute_write_analysis_data(datasette, db, sql, actor): + parameter_names = [] + analysis_rows = [] + analysis_error = None + if sql: + try: + parameter_names = _derived_query_parameters(sql) + params = {parameter: "" for parameter in parameter_names} + analysis = await db.analyze_sql(sql, params) + if _analysis_is_write(analysis): + analysis_rows = await _analysis_rows_with_permissions( + datasette, analysis, actor + ) + else: + analysis_error = ( + "Use /-/query for read-only SQL; " + "this endpoint only executes writes" + ) + except (QueryValidationError, sqlite3.DatabaseError) as ex: + analysis_error = getattr(ex, "message", str(ex)) + return { + "ok": analysis_error is None, + "parameters": parameter_names, + "analysis_error": analysis_error, + "analysis_rows": [row for row in analysis_rows if row["operation"] != "read"], + "execute_disabled": bool( + (not sql) + or analysis_error + or any(row["allowed"] is False for row in analysis_rows) + ), + } + + async def _inserted_row_url(datasette, db, analysis, cursor): if cursor.rowcount != 1: return None @@ -1024,6 +1057,45 @@ class ExecuteWriteView(BaseView): ) +class ExecuteWriteAnalyzeView(BaseView): + name = "execute-write-analyze" + has_json_alternate = False + + async def post(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing( + _error(["Permission denied: need execute-write-sql"], 403) + ) + + try: + data, _ = await _json_or_form_payload(request) + except QueryValidationError as ex: + return _block_framing(_error([ex.message], ex.status)) + if not isinstance(data, dict): + return _block_framing(_error(["JSON must be a dictionary"], 400)) + invalid_keys = set(data) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + sql = data.get("sql") or "" + if not isinstance(sql, str): + return _block_framing(_error(["sql must be a string"], 400)) + return _block_framing( + Response.json( + await _execute_write_analysis_data(self.ds, db, sql, request.actor) + ) + ) + + class QueryListView(BaseView): name = "query-list" diff --git a/docs/json_api.rst b/docs/json_api.rst index f44a39fe..2f581661 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -528,6 +528,7 @@ Creating saved queries ``POST //-/queries/-/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. .. _ExecuteWriteView: +.. _ExecuteWriteAnalyzeView: Executing write SQL ~~~~~~~~~~~~~~~~~~~ @@ -536,6 +537,8 @@ Executing write SQL ``POST //-/execute-write`` executes writable SQL. This requires ``execute-write-sql`` for the database plus the relevant table-level write permissions. +``POST //-/execute-write/-/analyze`` accepts ``{"sql": "..."}`` and returns the derived parameters plus the write operations that SQL would need in order to execute. + .. _QueryDefinitionView: Getting a saved query definition diff --git a/tests/test_queries.py b/tests/test_queries.py index a6080958..6d2c0b25 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -719,7 +719,9 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'data-sql-template="insert"' in response.text assert 'data-sql-template="update"' in response.text assert 'data-sql-template="delete"' in response.text + assert 'data-analyze-url="/data/-/execute-write/-/analyze"' in response.text assert 'addEventListener("paste"' in response.text + assert "refreshExecuteWriteAnalysis" in response.text assert '
      Required permissioninsertupdateread
      ' in response.text assert '' in response.text assert "" in response.text @@ -737,6 +739,53 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'executeWriteSqlInput.value = "\\n\\n\\n";' in empty_response.text +@pytest.mark.asyncio +async def test_execute_write_analyze_endpoint_uses_sql_only(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_analyze", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.post( + "/data/-/execute-write/-/analyze", + actor={"id": "root"}, + json={"sql": "insert into dogs (name) values (:name)"}, + ) + read_only_response = await ds.client.post( + "/data/-/execute-write/-/analyze", + actor={"id": "root"}, + json={"sql": "select * from dogs where name = :name"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["ok"] is True + assert data["parameters"] == ["name"] + assert data["analysis_error"] is None + assert data["execute_disabled"] is False + assert data["analysis_rows"] == [ + { + "operation": "insert", + "database": "data", + "table": "dogs", + "required_permission": "insert-row", + "source": None, + "allowed": True, + } + ] + assert "params" not in data + + assert read_only_response.status_code == 200 + read_only_data = read_only_response.json() + assert read_only_data["ok"] is False + assert read_only_data["parameters"] == ["name"] + assert read_only_data["analysis_error"] == ( + "Use /-/query for read-only SQL; this endpoint only executes writes" + ) + assert read_only_data["execute_disabled"] is True + + @pytest.mark.asyncio async def test_database_action_menu_links_to_execute_write_for_permitted_actor(): ds = Datasette( From de55a76d402a6326c60a5f4cd1a03c7476613f0b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 12:33:57 -0700 Subject: [PATCH 115/156] Fix 500 error when accessing query page without ?sql= parameter (#2744) Closes #2743 --- datasette/templates/query.html | 4 ++-- datasette/views/database.py | 43 ++++++++++++++++++---------------- docs/changelog.rst | 7 ++++++ tests/plugins/my_plugin.py | 4 ++-- tests/test_html.py | 16 +++++++++++++ 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 8b405da5..5f85ac6b 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -46,14 +46,14 @@ {% if not hide_sql %} {% if editable and allow_execute_sql %}

      + >{% if query and query.sql %}{{ query.sql }}{% elif tables %}select * from {{ tables[0].name|escape_sqlite }}{% endif %}

      {% else %} 
      {% if query %}{{ query.sql }}{% endif %}
      {% endif %} {% else %} {% if not canned_query %} {% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 0cf93832..8e4ea85a 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -577,7 +577,7 @@ class QueryView(View): named_parameters = [] if canned_query and canned_query.get("params"): named_parameters = canned_query["params"] - if not named_parameters: + if not named_parameters and sql: named_parameters = derive_named_parameters(sql) named_parameter_values = { named_parameter: params.get(named_parameter) or "" @@ -602,7 +602,7 @@ class QueryView(View): params_for_query = params - if not canned_query_write: + if sql and not canned_query_write: try: if not canned_query: # For regular queries we only allow SELECT, plus other rules @@ -646,6 +646,8 @@ class QueryView(View): # Handle formats from plugins if format_ == "csv": + if not sql: + raise DatasetteError("?sql= is required", status=400) async def fetch_data_for_csv(request, _next=None): results = await db.execute(sql, params, truncate=True) @@ -771,25 +773,26 @@ class QueryView(View): # - No magic parameters, so no :_ in the SQL string edit_sql_url = None is_validated_sql = False - try: - validate_sql_select(sql) - is_validated_sql = True - except InvalidSql: - pass - if allow_execute_sql and is_validated_sql and ":_" not in sql: - edit_sql_url = ( - datasette.urls.database(database) - + "/-/query" - + "?" - + urlencode( - { - **{ - "sql": sql, - }, - **named_parameter_values, - } + if sql: + try: + validate_sql_select(sql) + is_validated_sql = True + except InvalidSql: + pass + if allow_execute_sql and is_validated_sql and ":_" not in sql: + edit_sql_url = ( + datasette.urls.database(database) + + "/-/query" + + "?" + + urlencode( + { + **{ + "sql": sql, + }, + **named_parameter_values, + } + ) ) - ) async def query_actions(): query_actions = [] diff --git a/docs/changelog.rst b/docs/changelog.rst index 329b4769..dfb2a736 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v1_0_unreleased: + +Unreleased +---------- + +- Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) + .. _v1_0_a30: 1.0a30 (2026-05-24) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 4e401c07..f682e8b9 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -387,8 +387,8 @@ def view_actions(datasette, database, view, actor): @hookimpl def query_actions(datasette, database, query_name, sql): - # Don't explain an explain - if sql.lower().startswith("explain"): + # Don't explain an explain (or a missing query) + if not sql or sql.lower().startswith("explain"): return return [ { diff --git a/tests/test_html.py b/tests/test_html.py index efc1040d..d20796c9 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -241,6 +241,22 @@ def test_query_page_truncates(): ] +@pytest.mark.asyncio +async def test_query_page_with_no_sql(ds_client): + # https://github.com/simonw/datasette/issues/2743 + response = await ds_client.get("/fixtures/-/query") + assert response.status_code == 200 + assert '

      +

      + {% set parameter_names = [] %} + {% set parameter_values = {} %} + {% set sql_parameters_allow_expand = false %} + {% include "_sql_parameters.html" %}

      @@ -90,5 +95,11 @@ {% endif %} {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} + {% endblock %} diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 5037d006..9b522f66 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -75,61 +75,8 @@ color: #b00020; font-weight: 700; } -form.sql .execute-write-parameter-row textarea[data-parameter-control] { - border: 1px solid #ccc; - border-radius: 3px; - box-sizing: border-box; - display: block; - font-family: Helvetica, sans-serif; - font-size: 1em; - min-height: 7rem; - padding: 9px 4px; - width: 100%; -} -form.sql .execute-write-parameter-row { - align-items: start; - column-gap: 0.6rem; - display: grid; - grid-template-columns: minmax(8rem, 11rem) minmax(16rem, 1fr) auto; - margin: 0 0 0.65rem; - max-width: 52rem; -} -form.sql .execute-write-parameter-row label { - overflow-wrap: anywhere; - padding-top: 0.55rem; - width: auto; -} -form.sql .execute-write-parameter-row input[data-parameter-control] { - box-sizing: border-box; - width: 100%; -} -form.sql.core button.execute-write-parameter-toggle[type=button] { - font-size: 0.72rem; - height: 1.8rem; - line-height: 1; - margin: 0.25rem 0 0; - padding: 0.25rem 0.45rem; -} -@media (max-width: 480px) { - form.sql .execute-write-parameter-row { - grid-template-columns: 1fr; - row-gap: 0.25rem; - } - form.sql .execute-write-parameter-row label { - padding-top: 0; - } - form.sql.core button.execute-write-parameter-toggle[type=button] { - justify-self: start; - margin-top: 0; - } -} -form.sql .execute-write-editor { - max-width: 52rem; -} -form.sql .execute-write-editor textarea#sql-editor { - width: 100%; -} +{% include "_sql_parameter_styles.html" %} {% endblock %} {% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %} @@ -168,16 +115,11 @@ form.sql .execute-write-editor textarea#sql-editor { {% endif %} -

      +

      -
      - {% if parameter_names %} -

      Parameters

      - {% for parameter in parameter_names %} -

      - {% endfor %} - {% endif %} -
      + {% set sql_parameters_section_id = "execute-write-parameters-section" %} + {% set sql_parameters_allow_expand = true %} + {% include "_sql_parameters.html" %}

      Query operations

      @@ -222,128 +164,15 @@ if (executeWriteSqlInput && !executeWriteSqlInput.value) { {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 7c251e2c..3bcc7178 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -14,6 +14,7 @@ {% endif %} {% include "_codemirror.html" %} +{% include "_sql_parameter_styles.html" %} {% endblock %} {% block body_class %}query db-{{ database|to_css_class }}{% if canned_query %} query-{{ canned_query|to_css_class }}{% endif %}{% endblock %} @@ -36,7 +37,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

      Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

      @@ -45,7 +46,7 @@ {% endif %} {% if not hide_sql %} {% if editable and allow_execute_sql %} -

      {% else %} 
      {% if query %}{{ query.sql }}{% endif %}
      @@ -57,12 +58,10 @@ > {% endif %} {% endif %} - {% if named_parameter_values %} -

      Query parameters

      - {% for name, value in named_parameter_values.items() %} -

      - {% endfor %} - {% endif %} + {% set parameter_names = named_parameter_values.keys()|list %} + {% set parameter_values = named_parameter_values %} + {% set sql_parameters_allow_expand = false %} + {% include "_sql_parameters.html" %}

      {% if not hide_sql %}{% endif %} @@ -97,5 +96,11 @@ {% endif %} {% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} + {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index e4eaee30..278f7e8c 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1061,7 +1061,7 @@ class ExecuteWriteAnalyzeView(BaseView): name = "execute-write-analyze" has_json_alternate = False - async def post(self, request): + async def get(self, request): db = await self.ds.resolve_database(request) if not await self.ds.allowed( action="execute-write-sql", @@ -1072,13 +1072,7 @@ class ExecuteWriteAnalyzeView(BaseView): _error(["Permission denied: need execute-write-sql"], 403) ) - try: - data, _ = await _json_or_form_payload(request) - except QueryValidationError as ex: - return _block_framing(_error([ex.message], ex.status)) - if not isinstance(data, dict): - return _block_framing(_error(["JSON must be a dictionary"], 400)) - invalid_keys = set(data) - {"sql"} + invalid_keys = set(request.args) - {"sql"} if invalid_keys: return _block_framing( _error( @@ -1086,9 +1080,7 @@ class ExecuteWriteAnalyzeView(BaseView): 400, ) ) - sql = data.get("sql") or "" - if not isinstance(sql, str): - return _block_framing(_error(["sql must be a string"], 400)) + sql = request.args.get("sql") or "" return _block_framing( Response.json( await _execute_write_analysis_data(self.ds, db, sql, request.actor) @@ -1096,6 +1088,34 @@ class ExecuteWriteAnalyzeView(BaseView): ) +class QueryParametersView(BaseView): + name = "query-parameters" + has_json_alternate = False + + async def get(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + + invalid_keys = set(request.args) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + try: + parameters = _derived_query_parameters(request.args.get("sql") or "") + except QueryValidationError as ex: + return _block_framing(_error([ex.message], ex.status)) + return _block_framing(Response.json({"ok": True, "parameters": parameters})) + + class QueryListView(BaseView): name = "query-list" diff --git a/docs/json_api.rst b/docs/json_api.rst index 2f581661..91ed5306 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -527,17 +527,20 @@ Creating saved queries ``POST //-/queries/-/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. +.. _QueryParametersView: .. _ExecuteWriteView: .. _ExecuteWriteAnalyzeView: Executing write SQL ~~~~~~~~~~~~~~~~~~~ +``GET //-/query/-/parameters?sql=...`` returns the named parameters used by a SQL query. This requires ``execute-sql`` for the database. + ``GET //-/execute-write`` displays a form for executing writable SQL. A ``?sql=`` query string pre-populates the form without executing it. ``POST //-/execute-write`` executes writable SQL. This requires ``execute-write-sql`` for the database plus the relevant table-level write permissions. -``POST //-/execute-write/-/analyze`` accepts ``{"sql": "..."}`` and returns the derived parameters plus the write operations that SQL would need in order to execute. +``GET //-/execute-write/-/analyze?sql=...`` returns the derived parameters plus the write operations that SQL would need in order to execute. .. _QueryDefinitionView: diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py index a9d22036..ae2c74e0 100644 --- a/tests/test_canned_queries.py +++ b/tests/test_canned_queries.py @@ -200,7 +200,10 @@ def test_error_in_on_success_message_sql(canned_write_client): def test_custom_params(canned_write_client): response = canned_write_client.get("/data/update_name?extra=foo") - assert '' in response.text + assert ( + '' + in response.text + ) def test_canned_query_pages_no_vary_header(canned_write_client): diff --git a/tests/test_html.py b/tests/test_html.py index e5f00e17..b49391a6 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -326,17 +326,29 @@ async def test_query_parameter_form_fields(ds_client): response = await ds_client.get("/fixtures/-/query?sql=select+:name") assert response.status_code == 200 assert ( - ' ' + ' ' in response.text ) + assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'id="sql-parameters-section"' in response.text + assert "setupSqlParameterRefresh" in response.text response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") assert response2.status_code == 200 assert ( - ' ' + ' ' in response2.text ) +@pytest.mark.asyncio +async def test_database_page_sql_parameter_refresh_markup(ds_client): + response = await ds_client.get("/fixtures") + assert response.status_code == 200 + assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'id="sql-parameters-section"' in response.text + assert "setupSqlParameterRefresh" in response.text + + @pytest.mark.asyncio async def test_row_html_simple_primary_key(ds_client): response = await ds_client.get("/fixtures/simple_primary_key/1") diff --git a/tests/test_queries.py b/tests/test_queries.py index 6d2c0b25..23820cf3 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -721,7 +721,7 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'data-sql-template="delete"' in response.text assert 'data-analyze-url="/data/-/execute-write/-/analyze"' in response.text assert 'addEventListener("paste"' in response.text - assert "refreshExecuteWriteAnalysis" in response.text + assert "setupSqlParameterRefresh" in response.text assert '

      Required permissioninsert
      ' in response.text assert '' in response.text assert "" in response.text @@ -747,15 +747,15 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): await db.execute_write("create table dogs (id integer primary key, name text)") await ds.invoke_startup() - response = await ds.client.post( + response = await ds.client.get( "/data/-/execute-write/-/analyze", actor={"id": "root"}, - json={"sql": "insert into dogs (name) values (:name)"}, + params={"sql": "insert into dogs (name) values (:name)"}, ) - read_only_response = await ds.client.post( + read_only_response = await ds.client.get( "/data/-/execute-write/-/analyze", actor={"id": "root"}, - json={"sql": "select * from dogs where name = :name"}, + params={"sql": "select * from dogs where name = :name"}, ) assert response.status_code == 200 @@ -786,6 +786,44 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): assert read_only_data["execute_disabled"] is True +@pytest.mark.asyncio +async def test_query_parameters_endpoint_uses_get_sql_only(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_parameters", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + + response = await ds.client.get( + "/data/-/query/-/parameters", + actor={"id": "root"}, + params={ + "sql": "select * from dogs where name = :name and id = :id", + }, + ) + permission_denied_response = await ds.client.get( + "/data/-/query/-/parameters", + actor={"id": "not-root"}, + params={"sql": "select * from dogs where name = :name"}, + ) + magic_parameter_response = await ds.client.get( + "/data/-/query/-/parameters", + actor={"id": "root"}, + params={"sql": "select :_actor_id"}, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": True, "parameters": ["name", "id"]} + assert permission_denied_response.status_code == 403 + assert permission_denied_response.json()["errors"] == [ + "Permission denied: need execute-sql" + ] + assert magic_parameter_response.status_code == 400 + assert magic_parameter_response.json()["errors"] == [ + "Magic parameters are not allowed" + ] + + @pytest.mark.asyncio async def test_database_action_menu_links_to_execute_write_for_permitted_actor(): ds = Datasette( From 4208ded249b28f8b0918ce80d289bfc88f9e8921 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 12:46:21 -0700 Subject: [PATCH 117/156] No execute-write on immutable databases Refs https://github.com/simonw/datasette/issues/2742#issuecomment-4536690161 --- datasette/default_database_actions.py | 2 ++ datasette/views/database.py | 7 ++++ tests/test_queries.py | 46 +++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/datasette/default_database_actions.py b/datasette/default_database_actions.py index 78055392..e0cb3cdf 100644 --- a/datasette/default_database_actions.py +++ b/datasette/default_database_actions.py @@ -5,6 +5,8 @@ from datasette.resources import DatabaseResource @hookimpl def database_actions(datasette, actor, database, request): async def inner(): + if not datasette.get_database(database).is_mutable: + return [] if not await datasette.allowed( action="execute-write-sql", resource=DatabaseResource(database), diff --git a/datasette/views/database.py b/datasette/views/database.py index 278f7e8c..de02cd0f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -964,6 +964,13 @@ class ExecuteWriteView(BaseView): resource=DatabaseResource(db.name), actor=request.actor, ) + if not db.is_mutable: + return _block_framing( + _error( + ["Cannot execute write SQL because this database is immutable."], + 403, + ) + ) return await self._render_form( request, db, diff --git a/tests/test_queries.py b/tests/test_queries.py index 23820cf3..c31d7205 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -858,6 +858,52 @@ async def test_database_action_menu_links_to_execute_write_for_permitted_actor() assert "Execute write SQL" in writer_response.text +@pytest.mark.asyncio +async def test_database_action_menu_hides_execute_write_for_immutable_database(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "writer"}, + "execute-write-sql": {"id": "writer"}, + } + } + } + }, + ) + db = ds.add_memory_database("execute_write_menu_immutable", name="data") + db.is_mutable = False + await ds.invoke_startup() + + response = await ds.client.get("/data", actor={"id": "writer"}) + + assert response.status_code == 200 + assert "Execute write SQL" not in response.text + assert 'href="/data/-/execute-write"' not in response.text + + +@pytest.mark.asyncio +async def test_execute_write_get_rejects_immutable_database(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_get_immutable", name="data") + db.is_mutable = False + await ds.invoke_startup() + + response = await ds.client.get( + "/data/-/execute-write?sql=insert+into+dogs+(name)+values+('Cleo')", + actor={"id": "root"}, + ) + + assert response.status_code == 403 + assert response.json()["errors"] == [ + "Cannot execute write SQL because this database is immutable." + ] + + @pytest.mark.asyncio async def test_execute_write_post_requires_database_and_table_permissions(): ds = Datasette( From 8ab8999ba97e0ec1d113ee8d3954d6431f39fa28 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 12:55:36 -0700 Subject: [PATCH 118/156] Big visual improvement to /-/queries pages Including /db/-/queries Refs https://github.com/simonw/datasette/issues/2735#issuecomment-4536860239 --- datasette/templates/query_list.html | 226 ++++++++++++++++++++++++---- datasette/views/database.py | 12 +- tests/test_queries.py | 25 ++- 3 files changed, 229 insertions(+), 34 deletions(-) diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index af974550..dbd607ab 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -2,6 +2,155 @@ {% block title %}{% if database %}{{ database }}: {% endif %}queries{% endblock %} +{% block extra_head %} +{{- super() -}} + +{% endblock %} + {% block body_class %}query-list{% if database %} db-{{ database|to_css_class }}{% endif %}{% endblock %} {% block crumbs %} @@ -10,49 +159,66 @@ {% block content %} -

      Queries

      +
      - -

      +

      Queries

      + + +

      -

      - - - - -

      +
      +
      + Mode + + + +
      +
      + Publication + + + +
      +
      {% if queries %} -
        - {% for query in queries %} -
      • - {% if show_database %} - {{ query.database }}: - {% endif %} - {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} - {% if query.is_write %}Writable{% endif %} - {% if query.is_published %}Published{% endif %} -
        • - {% endfor %} -
      +
      Required permissioninsert
      + + + {% if show_database %}{% endif %} + + + + + + + {% for query in queries %} + + {% if show_database %} + + {% endif %} + + + + + {% endfor %} + +
      DatabaseQueryModePublication
      {{ query.database }} + {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} + {% if query.description %}

      {{ query.description }}

      {% endif %} +       		{% if query.is_write %}Writable{% else %}Read-only{% endif %}{% if query.is_published %}Published{% else %}Unpublished{% endif %}
      {% else %}

      No queries found.

      {% endif %} {% if next_url %} -

      Next page

      + {% endif %} + + {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index de02cd0f..3c660bc7 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -487,9 +487,9 @@ def _as_optional_bool(value, name): raise QueryValidationError("{} must be 0 or 1".format(name)) -def _query_list_limit(value): +def _query_list_limit(value, default=50): if value in (None, ""): - return 50 + return default try: return min(max(1, int(value)), 1000) except ValueError as ex: @@ -1136,7 +1136,10 @@ class QueryListView(BaseView): database = await self.database_name(request) format_ = request.url_vars.get("format") or "html" try: - limit = _query_list_limit(request.args.get("_size")) + limit = _query_list_limit( + request.args.get("_size"), + default=20 if format_ == "html" else 50, + ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") is_published = _as_optional_bool( request.args.get("is_published"), "is_published" @@ -1175,6 +1178,9 @@ class QueryListView(BaseView): data = { "ok": True, "database": database, + "database_color": ( + self.ds.get_database(database).color if database is not None else None + ), "queries": page["queries"], "next": page["next"], "next_url": next_url, diff --git a/tests/test_queries.py b/tests/test_queries.py index c31d7205..b7416ac7 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -451,12 +451,34 @@ async def test_query_list_search_filter_and_html(): assert html_response.status_code == 200 assert "Demo query 02" in html_response.text assert "Demo query 01" not in html_response.text + assert 'class="query-list-results"' in html_response.text + assert "Mode" in html_response.text + assert 'type="radio" name="is_published" value="1"' in html_response.text assert json_response.json()["queries"][0]["name"] == "demo_query_02" assert [query["name"] for query in filtered_response.json()["queries"]] == [ "private_query" ] +@pytest.mark.asyncio +async def test_query_list_html_defaults_to_twenty_and_shows_pagination(): + ds = Datasette(memory=True) + ds.root_enabled = True + ds.add_memory_database("query_list_html_pagination", name="data") + await ds.invoke_startup() + await add_numbered_queries(ds, "data", 25) + + response = await ds.client.get("/data/-/queries", actor={"id": "root"}) + json_response = await ds.client.get("/data/-/queries.json", actor={"id": "root"}) + + assert response.status_code == 200 + assert response.text.count('aria-label="Query pagination"') == 1 + assert "Demo query 20" in response.text + assert "Demo query 21" not in response.text + assert 'href="/data/-/queries?_next=' in response.text + assert len(json_response.json()["queries"]) == 25 + + @pytest.mark.asyncio async def test_global_query_list_api_and_html(): ds = Datasette(memory=True) @@ -519,7 +541,8 @@ async def test_global_query_list_api_and_html(): ("beta", "beta_first"), ] assert html_response.status_code == 200 - assert 'href="/beta">beta:' in html_response.text + assert 'Database' in html_response.text + assert 'class="query-list-database" href="/beta">beta' in html_response.text assert "Beta first" in html_response.text assert "Alpha first" not in html_response.text From f1dd86ebfb01644fead19f9f007b9b76f863d72e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 25 May 2026 14:05:26 -0700 Subject: [PATCH 119/156] Tweak URL designs of new endpoints --- datasette/app.py | 6 +++--- datasette/templates/database.html | 2 +- datasette/templates/execute_write.html | 2 +- datasette/templates/query.html | 2 +- datasette/templates/query_create.html | 2 +- docs/json_api.rst | 6 +++--- queries-plan.md | 4 ++-- tests/test_html.py | 4 ++-- tests/test_queries.py | 22 +++++++++++----------- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 90e41521..232aa0cf 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2745,11 +2745,11 @@ class Datasette: ) add_route( QueryInsertView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/-/insert$", + r"/(?P[^\/\.]+)/-/queries/insert$", ) add_route( ExecuteWriteAnalyzeView.as_view(self), - r"/(?P[^\/\.]+)/-/execute-write/-/analyze$", + r"/(?P[^\/\.]+)/-/execute-write/analyze$", ) add_route( ExecuteWriteView.as_view(self), @@ -2761,7 +2761,7 @@ class Datasette: ) add_route( QueryParametersView.as_view(self), - r"/(?P[^\/\.]+)/-/query/-/parameters$", + r"/(?P[^\/\.]+)/-/query/parameters$", ) add_route( wrap_view(QueryView, self), diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 0c9ec94c..62f9c620 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -26,7 +26,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} {% if allow_execute_sql %} -
      +

      Custom SQL query

      {% set parameter_names = [] %} diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 9b522f66..46f58c3b 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -95,7 +95,7 @@

      {{ execution_message }}{% for link in execution_links %} {{ link.label }}{% endfor %}

      {% endif %} - + {% if write_template_tables %}
      diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 3bcc7178..f74d21f1 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -37,7 +37,7 @@ {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

      Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

      diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index fb2599d2..3c027def 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -17,7 +17,7 @@

      Create query

      - +


      diff --git a/docs/json_api.rst b/docs/json_api.rst index 91ed5306..dd54c459 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -525,7 +525,7 @@ Creating saved queries in the UI Creating saved queries ~~~~~~~~~~~~~~~~~~~~~~ -``POST //-/queries/-/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. +``POST //-/queries/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. .. _QueryParametersView: .. _ExecuteWriteView: @@ -534,13 +534,13 @@ Creating saved queries Executing write SQL ~~~~~~~~~~~~~~~~~~~ -``GET //-/query/-/parameters?sql=...`` returns the named parameters used by a SQL query. This requires ``execute-sql`` for the database. +``GET //-/query/parameters?sql=...`` returns the named parameters used by a SQL query. This requires ``execute-sql`` for the database. ``GET //-/execute-write`` displays a form for executing writable SQL. A ``?sql=`` query string pre-populates the form without executing it. ``POST //-/execute-write`` executes writable SQL. This requires ``execute-write-sql`` for the database plus the relevant table-level write permissions. -``GET //-/execute-write/-/analyze?sql=...`` returns the derived parameters plus the write operations that SQL would need in order to execute. +``GET //-/execute-write/analyze?sql=...`` returns the derived parameters plus the write operations that SQL would need in order to execute. .. _QueryDefinitionView: diff --git a/queries-plan.md b/queries-plan.md index a708e887..72427df2 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -211,7 +211,7 @@ JSON endpoints should follow Datasette's existing write API style: use `POST` pl Endpoints: - `GET /-/queries` and `GET /{database}/-/queries` show searchable HTML query browsers. `GET /-/queries.json` lists query definitions across every database the actor can view; `GET /{database}/-/queries.json` scopes that list to one database. Both JSON endpoints use cursor pagination with `_next` and `_size`. -- `POST /{database}/-/queries/-/insert` creates a query. +- `POST /{database}/-/queries/insert` creates a query. - `GET /{database}/{query}/-/definition` returns one query definition without executing it. - `POST /{database}/{query}/-/update` updates one query. - `POST /{database}/{query}/-/delete` deletes one query. @@ -388,7 +388,7 @@ The read methods should reconstruct the existing dictionary shape used by query On `/{database}/-/query`, if the actor has both `execute-sql` and `insert-query`, show a save control for valid read-only SQL. That page already executes read-only arbitrary SQL, so the first UI can stay read-only even though the JSON API can accept writable SQL after `Database.analyze_sql()` validation. -The save form should call `POST /{database}/-/queries/-/insert` and default to `is_published=false`. +The save form should call `POST /{database}/-/queries/insert` and default to `is_published=false`. If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. diff --git a/tests/test_html.py b/tests/test_html.py index b49391a6..8cda6dba 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -329,7 +329,7 @@ async def test_query_parameter_form_fields(ds_client): ' ' in response.text ) - assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text assert 'id="sql-parameters-section"' in response.text assert "setupSqlParameterRefresh" in response.text response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") @@ -344,7 +344,7 @@ async def test_query_parameter_form_fields(ds_client): async def test_database_page_sql_parameter_refresh_markup(ds_client): response = await ds_client.get("/fixtures") assert response.status_code == 200 - assert 'data-parameters-url="/fixtures/-/query/-/parameters"' in response.text + assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text assert 'id="sql-parameters-section"' in response.text assert "setupSqlParameterRefresh" in response.text diff --git a/tests/test_queries.py b/tests/test_queries.py index b7416ac7..57920584 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -356,7 +356,7 @@ async def test_query_insert_api_creates_read_only_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={ "query": { @@ -568,7 +568,7 @@ async def test_query_insert_api_publish_requires_publish_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "writer"}, json={"query": {"name": "public", "sql": "select 1", "is_published": True}}, ) @@ -586,7 +586,7 @@ async def test_query_insert_api_creates_writable_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={ "query": { @@ -603,7 +603,7 @@ async def test_query_insert_api_creates_writable_query(): assert query["parameters"] == ["name"] bad_response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={ "query": { @@ -671,7 +671,7 @@ async def test_query_insert_api_rejects_magic_parameters(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/-/insert", + "/data/-/queries/insert", actor={"id": "root"}, json={"query": {"name": "magic", "sql": "select :_actor_id"}}, ) @@ -742,7 +742,7 @@ async def test_execute_write_get_prepopulates_without_executing(): assert 'data-sql-template="insert"' in response.text assert 'data-sql-template="update"' in response.text assert 'data-sql-template="delete"' in response.text - assert 'data-analyze-url="/data/-/execute-write/-/analyze"' in response.text + assert 'data-analyze-url="/data/-/execute-write/analyze"' in response.text assert 'addEventListener("paste"' in response.text assert "setupSqlParameterRefresh" in response.text assert '' in response.text @@ -771,12 +771,12 @@ async def test_execute_write_analyze_endpoint_uses_sql_only(): await ds.invoke_startup() response = await ds.client.get( - "/data/-/execute-write/-/analyze", + "/data/-/execute-write/analyze", actor={"id": "root"}, params={"sql": "insert into dogs (name) values (:name)"}, ) read_only_response = await ds.client.get( - "/data/-/execute-write/-/analyze", + "/data/-/execute-write/analyze", actor={"id": "root"}, params={"sql": "select * from dogs where name = :name"}, ) @@ -818,19 +818,19 @@ async def test_query_parameters_endpoint_uses_get_sql_only(): await ds.invoke_startup() response = await ds.client.get( - "/data/-/query/-/parameters", + "/data/-/query/parameters", actor={"id": "root"}, params={ "sql": "select * from dogs where name = :name and id = :id", }, ) permission_denied_response = await ds.client.get( - "/data/-/query/-/parameters", + "/data/-/query/parameters", actor={"id": "not-root"}, params={"sql": "select * from dogs where name = :name"}, ) magic_parameter_response = await ds.client.get( - "/data/-/query/-/parameters", + "/data/-/query/parameters", actor={"id": "root"}, params={"sql": "select :_actor_id"}, ) From 4a1a4d7807fb99203b9053b6d270b265df61f0af Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 11:59:49 -0700 Subject: [PATCH 120/156] Query is_trusted and is_private properties Refs https://github.com/simonw/datasette/issues/2735#issuecomment-4547270516 Diff explanation: https://gist.github.com/simonw/1e4de6c4b041a51968eb273ee96dec1f --- datasette/app.py | 39 ++-- datasette/default_actions.py | 7 - datasette/default_permissions/defaults.py | 100 +++++---- datasette/templates/query_create.html | 4 +- datasette/templates/query_list.html | 65 +++++- datasette/utils/internal_db.py | 3 +- datasette/views/database.py | 79 ++++--- docs/authentication.rst | 10 - docs/internals.rst | 3 +- queries-plan.md | 84 ++++---- tests/test_queries.py | 245 ++++++++++++++++++---- 11 files changed, 421 insertions(+), 218 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 232aa0cf..3329ee7e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -618,7 +618,8 @@ class Datasette: fragment=query_config.get("fragment"), parameters=query_config.get("params"), is_write=bool(query_config.get("write")), - is_published=bool(query_config.get("is_published")), + is_private=bool(query_config.get("is_private")), + is_trusted=bool(query_config.get("is_trusted", True)), source="config", on_success_message=query_config.get("on_success_message"), on_success_message_sql=query_config.get("on_success_message_sql"), @@ -1084,7 +1085,8 @@ class Datasette: "parameters": parameters, "is_write": is_write, "write": is_write, - "is_published": bool(row["is_published"]), + "is_private": bool(row["is_private"]), + "is_trusted": bool(row["is_trusted"]), "source": row["source"], "owner_id": row["owner_id"], "on_success_message": options.get("on_success_message"), @@ -1119,7 +1121,8 @@ class Datasette: fragment=None, parameters=None, is_write=False, - is_published=False, + is_private=False, + is_trusted=False, source="plugin", owner_id=None, on_success_message=None, @@ -1144,8 +1147,8 @@ class Datasette: sql_statement = """ INSERT INTO queries ( database_name, name, sql, title, description, description_html, - options, parameters, is_write, is_published, source, owner_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + options, parameters, is_write, is_private, is_trusted, source, owner_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ if replace: sql_statement += """ @@ -1157,7 +1160,8 @@ class Datasette: options = excluded.options, parameters = excluded.parameters, is_write = excluded.is_write, - is_published = excluded.is_published, + is_private = excluded.is_private, + is_trusted = excluded.is_trusted, source = excluded.source, owner_id = excluded.owner_id, updated_at = CURRENT_TIMESTAMP @@ -1174,7 +1178,8 @@ class Datasette: options_json, parameters_json, int(bool(is_write)), - int(bool(is_published)), + int(bool(is_private)), + int(bool(is_trusted)), source, owner_id, ], @@ -1193,7 +1198,8 @@ class Datasette: fragment=UNCHANGED, parameters=UNCHANGED, is_write=UNCHANGED, - is_published=UNCHANGED, + is_private=UNCHANGED, + is_trusted=UNCHANGED, source=UNCHANGED, owner_id=UNCHANGED, on_success_message=UNCHANGED, @@ -1209,7 +1215,8 @@ class Datasette: "description_html": description_html, "parameters": parameters, "is_write": is_write, - "is_published": is_published, + "is_private": is_private, + "is_trusted": is_trusted, "source": source, "owner_id": owner_id, } @@ -1227,7 +1234,7 @@ class Datasette: for field, value in fields.items(): if value is UNCHANGED: continue - if field in {"is_write", "is_published"}: + if field in {"is_write", "is_private", "is_trusted"}: value = int(bool(value)) elif field == "parameters": value = json.dumps(list(value or [])) @@ -1300,7 +1307,8 @@ class Datasette: cursor=None, q=None, is_write=None, - is_published=None, + is_private=None, + is_trusted=None, source=None, owner_id=None, include_private=False, @@ -1372,9 +1380,12 @@ class Datasette: if is_write is not None: where_clauses.append("q.is_write = :query_is_write") params["query_is_write"] = int(bool(is_write)) - if is_published is not None: - where_clauses.append("q.is_published = :query_is_published") - params["query_is_published"] = int(bool(is_published)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) if source is not None: where_clauses.append("q.source = :query_source") params["query_source"] = source diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 6787b80e..6a1f77b8 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -68,13 +68,6 @@ def register_actions(): resource_class=DatabaseResource, also_requires="execute-sql", ), - Action( - name="publish-query", - abbr="pq", - description="Publish saved queries for actors without execute-sql", - resource_class=DatabaseResource, - also_requires="insert-query", - ), # Table-level actions (child-level) Action( name="view-table", diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 58deea01..dfd8d3e9 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -26,6 +26,32 @@ DEFAULT_ALLOW_ACTIONS = frozenset( ) +def _configured_query_restriction_selects(datasette: "Datasette") -> tuple[list[str], dict]: + selects = [] + params = {} + for index, (database_name, db_config) in enumerate( + ((datasette.config or {}).get("databases") or {}).items() + ): + for query_name, query_config in (db_config.get("queries") or {}).items(): + if isinstance(query_config, dict) and query_config.get("is_private"): + continue + parent_param = f"query_config_parent_{index}_{len(selects)}" + child_param = f"query_config_child_{index}_{len(selects)}" + selects.append( + f""" + SELECT :{parent_param} AS parent, :{child_param} AS child + WHERE NOT EXISTS ( + SELECT 1 FROM queries + WHERE database_name = :{parent_param} + AND name = :{child_param} + ) + """ + ) + params[parent_param] = database_name + params[child_param] = query_name + return selects, params + + @hookimpl(specname="permission_resources_sql") async def default_allow_sql_check( datasette: "Datasette", @@ -93,61 +119,45 @@ async def default_query_permissions_sql( if action != "view-query": return None - execute_sql = await datasette.allowed_resources_sql( - action="execute-sql", actor=actor - ) - sql = execute_sql.sql - params = {} - for key, value in execute_sql.params.items(): - new_key = f"query_execute_sql_{key}" - sql = sql.replace(f":{key}", f":{new_key}") - params[new_key] = value - - trusted_writable_sql = "" + params = {"query_owner_id": actor_id} + rule_sqls = [] if not datasette.default_deny: - trusted_writable_sql = """ - UNION ALL + rule_sqls.append( + """ SELECT database_name AS parent, name AS child, 1 AS allow, - 'trusted writable query' AS reason + 'non-private query' AS reason FROM queries - WHERE is_write = 1 - AND source IN ('config', 'plugin') - """ + WHERE is_private = 0 + """ + ) - user_writable_sql = "" if actor_id is not None: - params["query_owner_id"] = actor_id - user_writable_sql = """ - UNION ALL + rule_sqls.append( + """ SELECT database_name AS parent, name AS child, 1 AS allow, 'query owner' AS reason FROM queries - WHERE is_write = 1 - AND source = 'user' - AND owner_id = :query_owner_id + WHERE owner_id = :query_owner_id + """ + ) + + config_restriction_selects, config_restriction_params = ( + _configured_query_restriction_selects(datasette) + ) + + restriction_sqls = [ """ + SELECT database_name AS parent, name AS child + FROM queries + WHERE is_private = 0 + OR owner_id = :query_owner_id + """ + ] + restriction_sqls.extend(config_restriction_selects) + params.update(config_restriction_params) return PermissionSQL( - sql=f""" - WITH execute_sql_allowed AS ( - {sql} - ) - SELECT database_name AS parent, name AS child, 1 AS allow, - 'published query' AS reason - FROM queries - WHERE is_write = 0 - AND is_published = 1 - UNION ALL - SELECT q.database_name AS parent, q.name AS child, 1 AS allow, - 'execute-sql allows query' AS reason - FROM queries q - JOIN execute_sql_allowed es - ON es.parent = q.database_name - AND es.child IS NULL - WHERE q.is_write = 0 - AND q.is_published = 0 - {trusted_writable_sql} - {user_writable_sql} - """, + sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, + restriction_sql="\nUNION ALL\n".join(restriction_sqls), params=params, ) diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index 3c027def..686d971e 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -27,9 +27,7 @@

      - {% if can_publish %} -

      - {% endif %} +

      {% if sql and analysis_is_write %}

      Execute write SQL

      {% endif %} diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index dbd607ab..25259b3d 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -73,7 +73,7 @@ border-collapse: collapse; font-size: 0.9rem; margin: 0.25rem 0 1rem; - min-width: 36rem; + min-width: 42rem; width: 100%; } .query-list-results th, @@ -100,6 +100,16 @@ font-size: 0.78rem; margin: 0.15rem 0 0; } +.query-list-owner { + color: #39445a; + font-family: var(--font-monospace, monospace); + white-space: nowrap; +} +.query-list-flags { + display: flex; + flex-wrap: wrap; + gap: 0.3rem; +} .query-list-pill { background-color: #eef1f5; border: 1px solid #d7dde5; @@ -116,15 +126,36 @@ background-color: #fff4db; border-color: #e2b64e; } -.query-list-pill-published { +.query-list-pill-public { background-color: #e7f5ec; border-color: #9ecfab; color: #267a3e; } -.query-list-pill-unpublished { +.query-list-pill-private { background-color: #f7edf0; border-color: #dbb8c1; } +.query-list-pill-trusted { + background-color: #e7f5ec; + border-color: #9ecfab; + color: #267a3e; +} +.query-list-empty { + color: #6b7280; +} +.query-list-footnotes { + border-top: 1px solid #d7dde5; + color: #4f5b6d; + font-size: 0.82rem; + margin: 0.35rem 0 1rem; + padding-top: 0.55rem; +} +.query-list-footnotes p { + margin: 0.25rem 0; +} +.query-list-footnotes .query-list-pill { + margin-right: 0.35rem; +} .query-list-pagination a { border: 1px solid #007bff; border-radius: 0.25rem; @@ -177,10 +208,10 @@
      - Publication - - - + Visibility + + +
      @@ -191,8 +222,8 @@       {% if show_database %}{% endif %} - - + + @@ -205,12 +236,24 @@ {{ query.title or query.name }}{% if query.private %} 🔒{% endif %} {% if query.description %}

      {{ query.description }}

      {% endif %} - - + + {% endfor %}
      DatabaseQueryModePublicationOwnerFlags
      {% if query.is_write %}Writable{% else %}Read-only{% endif %}{% if query.is_published %}Published{% else %}Unpublished{% endif %}{% if query.owner_id is not none %}{{ query.owner_id }}{% else %}-{% endif %} + + {% if query.is_write %}Writable{% else %}Read-only{% endif %} + {% if query.is_private %}Private{% endif %} + {% if query.is_trusted %}Trusted{% endif %} + +
      + {% if show_private_note or show_trusted_note %} +
      + {% if show_private_note %}

      PrivateOnly the owning actor can view this query.

      {% endif %} + {% if show_trusted_note %}

      TrustedExecution skips the usual SQL and write permission checks after view-query allows access.

      {% endif %} +
      + {% endif %} {% else %}

      No queries found.

      {% endif %} diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index 9c693b0a..bf172667 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -123,7 +123,8 @@ async def initialize_metadata_tables(db): options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/datasette/views/database.py b/datasette/views/database.py index 3c660bc7..91e9c350 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -428,7 +428,7 @@ _query_fields = { "fragment", "parameters", "params", - "is_published", + "is_private", "on_success_message", "on_success_message_sql", "on_success_redirect", @@ -571,7 +571,7 @@ async def _check_query_name(db, name, *, existing=False): raise QueryValidationError("Query name conflicts with a table or view") -async def _analyze_user_query(datasette, db, sql, *, actor, is_published): +async def _analyze_user_query(datasette, db, sql, *, actor): if not sql or not isinstance(sql, str): raise QueryValidationError("SQL is required") derived = _derived_query_parameters(sql) @@ -583,8 +583,6 @@ async def _analyze_user_query(datasette, db, sql, *, actor, is_published): is_write = _analysis_is_write(analysis) if is_write: - if is_published: - raise QueryValidationError("Writable queries cannot be published") try: await datasette.ensure_query_write_permissions( db.name, sql, actor=actor, analysis=analysis @@ -680,6 +678,26 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis +async def _ensure_stored_query_execution_permissions(datasette, db, query, actor): + if query.get("is_trusted"): + return + if query.get("write"): + await datasette.ensure_permission( + action="execute-write-sql", + resource=DatabaseResource(db.name), + actor=actor, + ) + await datasette.ensure_query_write_permissions( + db.name, query["sql"], actor=actor + ) + else: + await datasette.ensure_permission( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=actor, + ) + + async def _execute_write_analysis_data(datasette, db, sql, actor): parameter_names = [] analysis_rows = [] @@ -752,7 +770,7 @@ async def _inserted_row_url(datasette, db, analysis, cursor): def _apply_query_data_types(data): typed = dict(data) - for key in ("hide_sql", "is_published"): + for key in ("hide_sql", "is_private"): if key in typed: typed[key] = _as_bool(typed[key]) return typed @@ -769,20 +787,12 @@ async def _prepare_query_create(datasette, request, db, data): if await datasette.get_query(db.name, name) is not None: raise QueryValidationError("Query already exists") - is_published = _as_bool(data.get("is_published")) is_write, derived, analysis = await _analyze_user_query( datasette, db, data.get("sql"), actor=request.actor, - is_published=is_published, ) - if is_published and not await datasette.allowed( - action="publish-query", - resource=DatabaseResource(db.name), - actor=request.actor, - ): - raise QueryValidationError("Permission denied: need publish-query", status=403) if not is_write and any(data.get(field) for field in _query_write_fields): raise QueryValidationError("Writable query fields require writable SQL") @@ -800,7 +810,8 @@ async def _prepare_query_create(datasette, request, db, data): "fragment": data.get("fragment"), "parameters": parameters, "is_write": is_write, - "is_published": is_published, + "is_private": _as_bool(data.get("is_private", True)), + "is_trusted": False, "source": "user", "owner_id": _actor_id(request.actor), "on_success_message": data.get("on_success_message"), @@ -819,7 +830,6 @@ async def _prepare_query_update(datasette, request, db, existing, update): update = _apply_query_data_types(update) sql = update.get("sql", existing["sql"]) - is_published = update.get("is_published", existing["is_published"]) query_is_write = existing["is_write"] derived = _derived_query_parameters(sql) parameters = None @@ -830,19 +840,7 @@ async def _prepare_query_update(datasette, request, db, existing, update): db, sql, actor=request.actor, - is_published=is_published, ) - elif is_published and query_is_write: - raise QueryValidationError("Writable queries cannot be published") - if is_published and not existing["is_published"]: - if not await datasette.allowed( - action="publish-query", - resource=DatabaseResource(db.name), - actor=request.actor, - ): - raise QueryValidationError( - "Permission denied: need publish-query", status=403 - ) if "parameters" in update or "params" in update: parameters = _coerce_query_parameters( @@ -864,7 +862,7 @@ async def _prepare_query_update(datasette, request, db, existing, update): "fragment": update.get("fragment"), "parameters": parameters, "is_write": query_is_write, - "is_published": is_published, + "is_private": update.get("is_private"), "on_success_message": update.get("on_success_message"), "on_success_message_sql": update.get("on_success_message_sql"), "on_success_redirect": update.get("on_success_redirect"), @@ -1141,8 +1139,8 @@ class QueryListView(BaseView): default=20 if format_ == "html" else 50, ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") - is_published = _as_optional_bool( - request.args.get("is_published"), "is_published" + is_private = _as_optional_bool( + request.args.get("is_private"), "is_private" ) except QueryValidationError as ex: return _error([ex.message], ex.status) @@ -1154,7 +1152,7 @@ class QueryListView(BaseView): cursor=request.args.get("_next"), q=request.args.get("q") or None, is_write=is_write, - is_published=is_published, + is_private=is_private, source=request.args.get("source") or None, owner_id=request.args.get("owner_id") or None, include_private=True, @@ -1186,12 +1184,14 @@ class QueryListView(BaseView): "next_url": next_url, "has_more": page["has_more"], "limit": page["limit"], + "show_private_note": any(query["is_private"] for query in page["queries"]), + "show_trusted_note": any(query["is_trusted"] for query in page["queries"]), "query_list_path": query_list_path, "show_database": database is None, "filters": { "q": request.args.get("q") or "", "is_write": request.args.get("is_write") or "", - "is_published": request.args.get("is_published") or "", + "is_private": request.args.get("is_private") or "", "source": request.args.get("source") or "", "owner_id": request.args.get("owner_id") or "", }, @@ -1255,11 +1255,6 @@ class QueryCreateView(BaseView): "database_color": db.color, "sql": sql, "parameter_names": parameter_names, - "can_publish": await self.ds.allowed( - action="publish-query", - resource=DatabaseResource(db.name), - actor=request.actor, - ), "analysis_error": analysis_error, "analysis_rows": analysis_rows, "analysis_is_write": bool( @@ -1435,9 +1430,9 @@ class QueryView(View): ): raise Forbidden("You do not have permission to view this query") - if canned_query.get("write") and canned_query.get("source") == "user": - await datasette.ensure_query_write_permissions( - db.name, canned_query["sql"], actor=request.actor + if canned_query.get("write"): + await _ensure_stored_query_execution_permissions( + datasette, db, canned_query, request.actor ) # If database is immutable, return an error @@ -1558,6 +1553,10 @@ class QueryView(View): ) if not visible: raise Forbidden("You do not have permission to view this query") + if not canned_query_write: + await _ensure_stored_query_execution_permissions( + datasette, db, canned_query, request.actor + ) else: await datasette.ensure_permission( diff --git a/docs/authentication.rst b/docs/authentication.rst index b6a4cb7e..6e835c8d 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1299,16 +1299,6 @@ insert-query Actor is allowed to create saved queries in a database. -``resource`` - ``datasette.resources.DatabaseResource(database)`` - ``database`` is the name of the database (string) - -.. _actions_publish_query: - -publish-query -------------- - -Actor is allowed to publish a saved read-only query so actors without ``execute-sql`` can run it. - ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) diff --git a/docs/internals.rst b/docs/internals.rst index b5da7cbf..c76de487 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2158,7 +2158,8 @@ The internal database schema is as follows: options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/queries-plan.md b/queries-plan.md index 72427df2..f4b8049c 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -13,9 +13,9 @@ Terminology change: these are now "queries", not "canned queries". Legacy code a - Internal table name: `queries`. - Query definitions should use real columns, not a JSON blob for all options. - Query parameter names live in a `parameters` text column as a JSON array. No default values for parameters in this pass. -- No `queries_database_is_published_idx` index. -- User-created queries require `execute-sql` and `insert-query` on the database. Writable queries additionally require matching table write permissions discovered by `Database.analyze_sql()`. -- `publish-query` is the permission for creating or updating a query so users without `execute-sql` can execute it. +- No separate index is needed for the privacy/trust flags yet. +- User-created queries require `execute-sql` and `insert-query` on the database. They default to private, and writable queries additionally require matching table write permissions discovered by `Database.analyze_sql()`. +- Configured queries default to trusted, which means actors who can view them can execute them without also holding `execute-sql` or the relevant write permissions. Config can opt out with `is_trusted: false`. - Add `update-query` and `delete-query`, so administrators can manage queries created by other users. - Remove the old `canned_queries()` hook from core. If we want compatibility later, build a separate `datasette-old-canned-queries` plugin. - Writable user-created queries can be supported using `Database.analyze_sql()`, provided we fail closed when analysis cannot prove the required permissions. @@ -45,7 +45,8 @@ CREATE TABLE IF NOT EXISTS queries ( options TEXT NOT NULL DEFAULT '{}', parameters TEXT NOT NULL DEFAULT '[]', is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_published INTEGER NOT NULL DEFAULT 0 CHECK (is_published IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), source TEXT NOT NULL DEFAULT 'user', owner_id TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -64,11 +65,12 @@ Column notes: - Less common presentation and writable-query behavior lives in `options`, stored as a JSON object. That covers `hide_sql`, `fragment`, `on_success_message`, `on_success_message_sql`, `on_success_redirect`, `on_error_message`, and `on_error_redirect`. - `parameters` is a JSON array of parameter names, stored as text. This preserves explicit parameter order, but does not support labels or default values. - Existing writable query behavior gets `is_write` as a column. Success/error messages, success/error redirects, and `on_success_message_sql` are stored in `options`. -- `is_published` only applies to read-only queries. A writable query can still be public through explicit `view-query` permissions, but the "publish for users without execute-sql" shortcut should be read-only. +- `is_private` means the query is only visible to its owning actor. This is enforced as a permission restriction, so broader `view-query` grants do not expose private rows. +- `is_trusted` means execution skips the usual `execute-sql` or write-permission checks after `view-query` has allowed access. - `source` distinguishes `user`, `config`, and `plugin` rows. - `owner_id` is the actor id for user-created rows. It is `NULL` for config/plugin rows. -No separate index is needed on `(database_name, name)` because the primary key already creates one. Do not add a `queries_database_is_published_idx` index for now. +No separate index is needed on `(database_name, name)` because the primary key already creates one. `QueryResource.resources_sql()` can become: @@ -104,7 +106,6 @@ Remove the old `canned_queries()` hookspec and all core calls to it. If compatib Add core actions: - `insert-query`, database-level, for creating queries in a database. -- `publish-query`, database-level, for marking read-only queries as executable by actors who lack `execute-sql`. - `update-query`, query-level, for modifying existing query definitions. - `delete-query`, query-level, for deleting existing query definitions. @@ -114,17 +115,11 @@ User-created query creation requires: - `insert-query` on `DatabaseResource(database)` - If analysis shows the query is writable, the table-level write permissions described in the writable query section. -Setting `is_published=1` requires: - -- `publish-query` on `DatabaseResource(database)` -- The query must be read-only according to `Database.analyze_sql()`. - Updating an existing query requires: - `update-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. - If the SQL changes, also require `execute-sql` on the database. - If the changed SQL is writable, also require the table-level write permissions described in the writable query section. -- If `is_published` changes from `0` to `1`, also require `publish-query` on the database. Deleting an existing query requires: @@ -133,18 +128,18 @@ Deleting an existing query requires: Default owner permissions: - For `source='user' AND owner_id = actor.id`, grant `update-query` and `delete-query`. -- Do not automatically grant execution if the user no longer has the execution permission described below. +- For `source='user' AND owner_id = actor.id`, grant `view-query`. If the query is private, restriction SQL ensures no other actor sees it through a broader grant. ## Executing queries Default execution rule for read-only queries: -- If `is_published=0`, the actor needs `execute-sql` on the database. -- If `is_published=1`, the actor can execute the query without `execute-sql`. +- If `is_trusted=0`, the actor needs `execute-sql` on the database. +- If `is_trusted=1`, the actor can execute the query without `execute-sql`, provided `view-query` allows access. Default execution rule for user-created writable queries: -- `is_published` must be `0`. +- `is_trusted` must be `0`. - The actor must have `view-query`. - The actor must currently have every write permission required by fresh `Database.analyze_sql()` results for the query SQL. @@ -152,14 +147,14 @@ Implementation: - Remove `view-query` from the broad `DEFAULT_ALLOW_ACTIONS` set. - Replace it with query-aware default `view-query` permission SQL. -- For `is_published=1 AND is_write=0`, emit a child-level `view-query` allow. -- For `is_published=0 AND is_write=0`, emit child-level `view-query` allows for queries whose parent database is in the actor's `execute-sql` allowed resources. -- For `is_write=1 AND source='user'`, emit `view-query` only for the owner or actors with explicit `view-query` permission, then have `QueryView` perform the fresh analysis/table-permission check before execution. -- For trusted writable queries, preserve current behavior by emitting child-level `view-query` allows for `is_write=1 AND source IN ('config', 'plugin')` when Datasette is not running with `--default-deny`. +- Emit default `view-query` allows for non-private rows when Datasette is not running with `--default-deny`. +- Emit default `view-query` allows for the owning actor. +- Use `restriction_sql` to limit private rows to their owner even when broader `view-query` permissions exist. +- Have `QueryView` perform the fresh `execute-sql` or table-permission check before execution unless the row has `is_trusted=1`. -For read-only queries this keeps `QueryView` simple: it checks `view-query` for the query resource, and the default permission hook encodes the relationship with `execute-sql`. User-created writable queries need one additional runtime permission check because their required table permissions are derived from fresh SQL analysis. +For read-only queries this keeps `QueryView` explicit: it checks `view-query` for the query resource, then checks `execute-sql` unless the row is trusted. User-created writable queries need one additional runtime permission check because their required table permissions are derived from fresh SQL analysis. -Explicit deny rules should still be able to block a published query. +Explicit deny rules should still be able to block a query, and `--default-deny` still blocks trusted queries unless something grants `view-query`. ## Writable queries @@ -180,7 +175,7 @@ Validation flow for user-created queries: 1. Derive named parameters from the SQL and pass harmless placeholder values into `db.analyze_sql()` so SQLite can prepare statements with bindings. 2. If analysis raises a SQLite error, reject the query. 3. If every table access is `read`, treat the query as read-only and require `execute-sql` plus `insert-query`/`update-query` as described above. -4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `is_published=0`. +4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `is_trusted=0`. 5. Reject writable user-created queries that access a database other than the database they are being saved against, until `analyze_sql()` can reliably map attached SQLite schemas back to Datasette database names. 6. For every write access returned by analysis, require the corresponding permission on `TableResource(access.database, access.table)`: - `insert` -> `insert-row` @@ -200,7 +195,7 @@ Fail closed cases for user-created writable queries: - Analysis reports any write operation that cannot be mapped to a Datasette table resource. - Analysis reports writes outside the target database. - The actor lacks any required table write permission. -- `is_published=1` is requested. +- `is_trusted=1` is requested through the user-facing API. This gives us writable user-created queries without letting `execute-sql` alone become a path to create arbitrary write endpoints. @@ -225,7 +220,7 @@ Create request: "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers", "description": "Highest revenue customers", - "is_published": false, + "is_private": true, "parameters": ["region"] } } @@ -242,7 +237,8 @@ Successful create returns `201` and the created query definition: "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers", "description": "Highest revenue customers", - "is_published": false, + "is_private": true, + "is_trusted": false, "parameters": ["region"] } } @@ -254,7 +250,7 @@ Update request, imitating `RowUpdateView`: { "update": { "title": "Top customers by revenue", - "is_published": true + "is_private": false }, "return": true } @@ -270,7 +266,8 @@ Successful update returns `{"ok": true}` by default. With `"return": true`, retu "name": "top_customers", "sql": "select * from customers order by revenue desc limit 20", "title": "Top customers by revenue", - "is_published": true + "is_private": false, + "is_trusted": false } } ``` @@ -317,7 +314,8 @@ await datasette.add_query( fragment=None, parameters=None, is_write=False, - is_published=False, + is_private=False, + is_trusted=False, source="plugin", owner_id=None, on_success_message=None, @@ -340,7 +338,8 @@ await datasette.update_query( fragment=UNCHANGED, parameters=UNCHANGED, is_write=UNCHANGED, - is_published=UNCHANGED, + is_private=UNCHANGED, + is_trusted=UNCHANGED, source=UNCHANGED, owner_id=UNCHANGED, on_success_message=UNCHANGED, @@ -360,7 +359,8 @@ await datasette.list_queries( cursor=None, q=None, is_write=None, - is_published=None, + is_private=None, + is_trusted=None, source=None, owner_id=None, ) @@ -382,15 +382,13 @@ For column-backed fields, `None` should write SQL `NULL`. For option fields, `No Implementation detail: build the `UPDATE` statement dynamically from fields whose value is not `UNCHANGED`, validate non-nullable fields before writing, and update `updated_at` whenever at least one field changes. -The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `is_published`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. +The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `is_private`, `is_trusted`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. ## Query page save UI On `/{database}/-/query`, if the actor has both `execute-sql` and `insert-query`, show a save control for valid read-only SQL. That page already executes read-only arbitrary SQL, so the first UI can stay read-only even though the JSON API can accept writable SQL after `Database.analyze_sql()` validation. -The save form should call `POST /{database}/-/queries/insert` and default to `is_published=false`. - -If the actor also has `publish-query`, include a publish control. The UI copy should make it clear that publishing allows people without arbitrary SQL permission to run this query. +The save form should call `POST /{database}/-/queries/insert` and default to `is_private=true`. On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. The global `/-/queries` page reuses the same interface and shows the database for each query. @@ -403,7 +401,7 @@ This page should require `execute-sql` and `insert-query` to access. It should p - Read-only - Writable -Read-only mode can share the same fields as the arbitrary SQL save flow: name, title, description, parameters, and optional published status if the actor has `publish-query`. +Read-only mode can share the same fields as the arbitrary SQL save flow: name, title, description, parameters, and privacy status. Writable mode should always run `Database.analyze_sql()` and show an analysis panel before saving: @@ -413,7 +411,7 @@ Writable mode should always run `Database.analyze_sql()` and show an analysis pa - whether the actor has that permission - source, when the operation comes from a trigger or view -The Save button should be disabled until analysis succeeds and every required table write permission is allowed. Writable mode should not show a publish control, because user-created writable queries cannot be published. +The Save button should be disabled until analysis succeeds and every required table write permission is allowed. The existing edit-SQL flow from query pages can continue to point back to arbitrary SQL. A later enhancement can add "update this query" when the actor owns it or has `update-query`. @@ -427,14 +425,16 @@ The existing edit-SQL flow from query pages can continue to point back to arbitr - `QueryResource.resources_sql()` returns rows from `queries`. - Database page and `/-/jump` list queries from the internal DB. - `view-query` is no longer globally default-allowed; default query permissions come from the query-aware hook. -- Unpublished read-only query requires `execute-sql` to execute. -- Published read-only query can be executed without `execute-sql`. -- Setting `is_published=true` requires `publish-query`. +- Private query is only visible to its owner, even when a broader `view-query` rule applies. +- Non-trusted read-only query requires `execute-sql` to execute. +- Trusted read-only query can be executed without `execute-sql` after `view-query` passes. +- Config queries default to trusted and can opt out with `is_trusted: false`. +- User API rejects client-supplied `is_trusted`. - User-created query requires both `execute-sql` and `insert-query`. - User-created writable query creation uses `Database.analyze_sql()` and requires matching `insert-row`, `update-row`, and/or `delete-row` permissions for every reported write access. - `/{database}/-/queries/-/create` provides the writable-query authoring UI with an analysis panel and disabled save until all required write permissions pass. - User-created writable query execution re-runs `Database.analyze_sql()` and re-checks table write permissions. -- User-created writable query cannot be published. +- User-created writable query cannot be trusted through the user API. - Query update uses `POST /{database}/{query}/-/update` with an `{"update": {...}}` body. - Query delete uses `POST /{database}/{query}/-/delete`. - There are no `PATCH` or HTTP `DELETE` routes for query management. diff --git a/tests/test_queries.py b/tests/test_queries.py index 57920584..c97b5733 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -15,7 +15,6 @@ async def add_numbered_queries(ds, database, count): "select {} as query_number".format(i), title="Demo query {:02d}".format(i), description="Seeded demo query number {:02d}".format(i), - is_published=True, source="user", owner_id="root", ) @@ -44,7 +43,8 @@ async def test_queries_internal_table_schema(): "options", "parameters", "is_write", - "is_published", + "is_private", + "is_trusted", "source", "owner_id", "created_at", @@ -67,7 +67,7 @@ async def test_add_get_and_remove_query(): hide_sql=True, fragment="chart", parameters=["region"], - is_published=True, + is_trusted=True, source="user", owner_id="alice", ) @@ -100,7 +100,8 @@ async def test_add_get_and_remove_query(): "parameters": ["region"], "is_write": False, "write": False, - "is_published": True, + "is_private": False, + "is_trusted": True, "source": "user", "owner_id": "alice", "on_success_message": None, @@ -161,7 +162,8 @@ async def test_update_query_only_updates_provided_fields(): assert query["params"] == [] assert query["on_success_redirect"] is None assert query["sql"] == "select 1" - assert query["is_published"] is False + assert query["is_private"] is False + assert query["is_trusted"] is False options_row = ( await ds.get_internal_database().execute( """ @@ -208,7 +210,8 @@ async def test_config_queries_imported_to_internal_table(): "parameters": ["name"], "is_write": False, "write": False, - "is_published": False, + "is_private": False, + "is_trusted": True, "source": "config", "owner_id": None, "on_success_message": None, @@ -232,30 +235,171 @@ async def test_query_resources_come_from_internal_table(): @pytest.mark.asyncio -async def test_unpublished_query_requires_execute_sql_but_published_does_not(): - ds = Datasette(memory=True, settings={"default_allow_sql": False}) +async def test_default_deny_blocks_view_query_even_for_trusted_query(): + ds = Datasette(memory=True, default_deny=True) ds.add_memory_database("query_permissions", name="data") await ds.invoke_startup() - await ds.add_query("data", "unpublished", "select 1", is_published=False) - await ds.add_query("data", "published", "select 1", is_published=True) + await ds.add_query("data", "trusted", "select 1", is_trusted=True) assert not await ds.allowed( - action="execute-sql", - resource=DatabaseResource("data"), + action="view-query", + resource=QueryResource("data", "trusted"), actor=None, ) + + +@pytest.mark.asyncio +async def test_private_query_restriction_blocks_broad_view_query_permission(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-query": {"id": "*"}, + } + } + } + }, + ) + ds.add_memory_database("private_query_permissions", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "shared_report", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "alice"}, + ) assert not await ds.allowed( action="view-query", - resource=QueryResource("data", "unpublished"), - actor=None, + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, ) assert await ds.allowed( action="view-query", - resource=QueryResource("data", "published"), - actor=None, + resource=QueryResource("data", "shared_report"), + actor={"id": "bob"}, ) +@pytest.mark.asyncio +async def test_config_query_restriction_does_not_override_private_internal_query(): + ds = Datasette(memory=True, default_deny=True) + ds.add_memory_database("private_query_with_config_name", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + ds.config = { + "databases": { + "data": { + "permissions": {"view-query": {"id": "*"}}, + "queries": {"private_report": {"sql": "select 2"}}, + } + } + } + + assert not await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, + ) + + +@pytest.mark.asyncio +async def test_untrusted_shared_query_execution_requires_execute_sql(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-database": {"id": "viewer"}, + "view-query": {"id": "viewer"}, + } + } + } + }, + ) + ds.add_memory_database("untrusted_query_execution", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "shared_report", + "select 1 as one", + is_private=False, + is_trusted=False, + source="user", + owner_id="alice", + ) + + denied = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) + assert denied.status_code == 403 + + ds.config["databases"]["data"]["permissions"]["execute-sql"] = {"id": "viewer"} + allowed = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) + assert allowed.status_code == 200 + assert allowed.json()["rows"] == [{"one": 1}] + + +@pytest.mark.asyncio +async def test_config_queries_are_trusted_by_default_but_can_opt_out(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "view-query": {"id": "viewer"}, + }, + "queries": { + "trusted_report": {"sql": "select 1 as one"}, + "untrusted_report": { + "sql": "select 2 as two", + "is_trusted": False, + }, + }, + } + } + }, + ) + ds.add_memory_database("trusted_query_config", name="data") + await ds.invoke_startup() + + trusted = await ds.client.get("/data/trusted_report.json", actor={"id": "viewer"}) + untrusted = await ds.client.get( + "/data/untrusted_report.json", actor={"id": "viewer"} + ) + + assert trusted.status_code == 200 + assert trusted.json()["rows"] == [{"one": 1}] + assert untrusted.status_code == 403 + + @pytest.mark.asyncio async def test_database_page_query_preview_is_limited(): ds = Datasette(memory=True) @@ -281,7 +425,6 @@ async def test_query_actions_are_registered(): assert ds.get_action("execute-write-sql").resource_class is DatabaseResource assert ds.get_action("insert-query").resource_class is DatabaseResource - assert ds.get_action("publish-query").resource_class is DatabaseResource assert ds.get_action("update-query").resource_class is QueryResource assert ds.get_action("delete-query").resource_class is QueryResource @@ -430,21 +573,33 @@ async def test_query_list_search_filter_and_html(): "private_query", "select 'private'", title="Private query", - is_published=False, + is_private=True, source="user", owner_id="root", ) + await ds.add_query( + "data", + "trusted_query", + "select 'trusted'", + title="Trusted query", + is_trusted=True, + source="config", + ) html_response = await ds.client.get( "/data/-/queries?q=02", actor={"id": "root"}, ) + flags_response = await ds.client.get( + "/data/-/queries", + actor={"id": "root"}, + ) json_response = await ds.client.get( "/data/-/queries.json?q=02", actor={"id": "root"}, ) filtered_response = await ds.client.get( - "/data/-/queries.json?is_published=0", + "/data/-/queries.json?is_private=1", actor={"id": "root"}, ) @@ -453,7 +608,22 @@ async def test_query_list_search_filter_and_html(): assert "Demo query 01" not in html_response.text assert 'class="query-list-results"' in html_response.text assert "Mode" in html_response.text - assert 'type="radio" name="is_published" value="1"' in html_response.text + assert 'type="radio" name="is_private" value="1"' in html_response.text + assert "Only the owning actor can view this query." not in html_response.text + assert ( + "Execution skips the usual SQL and write permission checks" + not in html_response.text + ) + assert flags_response.status_code == 200 + assert 'Owner' in flags_response.text + assert 'Flags' in flags_response.text + assert 'Mode' not in flags_response.text + assert 'class="query-list-owner">root' in flags_response.text + assert 'class="query-list-pill">Read-only' in flags_response.text + assert 'class="query-list-pill query-list-pill-private">Private' in flags_response.text + assert 'class="query-list-pill query-list-pill-trusted">Trusted' in flags_response.text + assert "Only the owning actor can view this query." in flags_response.text + assert "Execution skips the usual SQL and write permission checks" in flags_response.text assert json_response.json()["queries"][0]["name"] == "demo_query_02" assert [query["name"] for query in filtered_response.json()["queries"]] == [ "private_query" @@ -491,7 +661,6 @@ async def test_global_query_list_api_and_html(): "alpha_first", "select 1", title="Alpha first", - is_published=True, source="user", owner_id="root", ) @@ -500,7 +669,6 @@ async def test_global_query_list_api_and_html(): "alpha_second", "select 2", title="Alpha second", - is_published=True, source="user", owner_id="root", ) @@ -509,7 +677,6 @@ async def test_global_query_list_api_and_html(): "beta_first", "select 3", title="Beta first", - is_published=True, source="user", owner_id="root", ) @@ -548,7 +715,7 @@ async def test_global_query_list_api_and_html(): @pytest.mark.asyncio -async def test_query_insert_api_publish_requires_publish_query(): +async def test_query_insert_api_rejects_is_trusted(): ds = Datasette( memory=True, default_deny=True, @@ -564,17 +731,17 @@ async def test_query_insert_api_publish_requires_publish_query(): } }, ) - ds.add_memory_database("query_publish_api", name="data") + ds.add_memory_database("query_trusted_api", name="data") await ds.invoke_startup() response = await ds.client.post( "/data/-/queries/insert", actor={"id": "writer"}, - json={"query": {"name": "public", "sql": "select 1", "is_published": True}}, + json={"query": {"name": "trusted", "sql": "select 1", "is_trusted": True}}, ) - assert response.status_code == 403 - assert response.json()["errors"] == ["Permission denied: need publish-query"] + assert response.status_code == 400 + assert response.json()["errors"] == ["Invalid keys: is_trusted"] @pytest.mark.asyncio @@ -599,24 +766,10 @@ async def test_query_insert_api_creates_writable_query(): assert response.status_code == 201 query = response.json()["query"] assert query["is_write"] is True - assert query["is_published"] is False + assert query["is_private"] is True + assert query["is_trusted"] is False assert query["parameters"] == ["name"] - bad_response = await ds.client.post( - "/data/-/queries/insert", - actor={"id": "root"}, - json={ - "query": { - "name": "published_insert", - "sql": "insert into dogs (name) values (:name)", - "is_published": True, - } - }, - ) - - assert bad_response.status_code == 400 - assert bad_response.json()["errors"] == ["Writable queries cannot be published"] - @pytest.mark.asyncio async def test_query_update_and_delete_api(): @@ -1103,6 +1256,10 @@ async def test_user_writable_query_execution_rechecks_table_permissions(): config={ "databases": { "data": { + "permissions": { + "view-database": {"id": ["alice", "bob"]}, + "execute-write-sql": {"id": ["alice", "bob"]}, + }, "tables": { "dogs": { "permissions": { From 1cd162e9da48b924c289ec9343e9d801b51a89f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 12:07:30 -0700 Subject: [PATCH 121/156] Removed some no-longer-necessary code, simplified view-query is back in the default allow actions now. We have other mechanisms that work for controlling visibility, and the fact that queries default to running with the permissions of the actor makes this safe. --- datasette/default_permissions/defaults.py | 55 +++-------------------- tests/test_permissions.py | 9 +++- tests/test_queries.py | 39 ++++++++++++++++ 3 files changed, 51 insertions(+), 52 deletions(-) diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index dfd8d3e9..ed0a6d66 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -21,37 +21,12 @@ DEFAULT_ALLOW_ACTIONS = frozenset( "view-database", "view-database-download", "view-table", + "view-query", "execute-sql", } ) -def _configured_query_restriction_selects(datasette: "Datasette") -> tuple[list[str], dict]: - selects = [] - params = {} - for index, (database_name, db_config) in enumerate( - ((datasette.config or {}).get("databases") or {}).items() - ): - for query_name, query_config in (db_config.get("queries") or {}).items(): - if isinstance(query_config, dict) and query_config.get("is_private"): - continue - parent_param = f"query_config_parent_{index}_{len(selects)}" - child_param = f"query_config_child_{index}_{len(selects)}" - selects.append( - f""" - SELECT :{parent_param} AS parent, :{child_param} AS child - WHERE NOT EXISTS ( - SELECT 1 FROM queries - WHERE database_name = :{parent_param} - AND name = :{child_param} - ) - """ - ) - params[parent_param] = database_name - params[child_param] = query_name - return selects, params - - @hookimpl(specname="permission_resources_sql") async def default_allow_sql_check( datasette: "Datasette", @@ -121,16 +96,6 @@ async def default_query_permissions_sql( params = {"query_owner_id": actor_id} rule_sqls = [] - if not datasette.default_deny: - rule_sqls.append( - """ - SELECT database_name AS parent, name AS child, 1 AS allow, - 'non-private query' AS reason - FROM queries - WHERE is_private = 0 - """ - ) - if actor_id is not None: rule_sqls.append( """ @@ -141,23 +106,13 @@ async def default_query_permissions_sql( """ ) - config_restriction_selects, config_restriction_params = ( - _configured_query_restriction_selects(datasette) - ) - - restriction_sqls = [ - """ + return PermissionSQL( + sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, + restriction_sql=""" SELECT database_name AS parent, name AS child FROM queries WHERE is_private = 0 OR owner_id = :query_owner_id - """ - ] - restriction_sqls.extend(config_restriction_selects) - params.update(config_restriction_params) - - return PermissionSQL( - sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, - restriction_sql="\nUNION ALL\n".join(restriction_sqls), + """, params=params, ) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 22f294bb..4f342d8f 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -937,16 +937,20 @@ async def test_permissions_in_config( updated_config = copy.deepcopy(previous_config) updated_config.update(config) perms_ds.config = updated_config + await perms_ds.apply_queries_config() try: # Convert old-style resource to Resource object - from datasette.resources import DatabaseResource, TableResource + from datasette.resources import DatabaseResource, QueryResource, TableResource resource_obj = None if resource: if isinstance(resource, str): resource_obj = DatabaseResource(database=resource) elif isinstance(resource, tuple) and len(resource) == 2: - resource_obj = TableResource(database=resource[0], table=resource[1]) + if action == "view-query": + resource_obj = QueryResource(database=resource[0], query=resource[1]) + else: + resource_obj = TableResource(database=resource[0], table=resource[1]) result = await perms_ds.allowed( action=action, resource=resource_obj, actor=actor @@ -956,6 +960,7 @@ async def test_permissions_in_config( assert result == expected_result finally: perms_ds.config = previous_config + await perms_ds.apply_queries_config() @pytest.mark.asyncio diff --git a/tests/test_queries.py b/tests/test_queries.py index c97b5733..dde57dea 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -248,6 +248,45 @@ async def test_default_deny_blocks_view_query_even_for_trusted_query(): ) +@pytest.mark.asyncio +async def test_view_query_default_allow_still_respects_private_restriction(): + ds = Datasette(memory=True) + ds.add_memory_database("default_view_query_permissions", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "private_report", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "shared_report", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "shared_report"), + actor=None, + ) + assert await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action="view-query", + resource=QueryResource("data", "private_report"), + actor={"id": "bob"}, + ) + + @pytest.mark.asyncio async def test_private_query_restriction_blocks_broad_view_query_permission(): ds = Datasette( From 1ac4265ffd295ea62008b13b3e37af96f5450be4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 12:12:59 -0700 Subject: [PATCH 122/156] Require permissions for untrusted stored query execution, refs #2735 --- datasette/views/database.py | 7 +++---- docs/authentication.rst | 2 +- queries-plan.md | 8 +++----- tests/test_queries.py | 12 ++++++++++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 91e9c350..bd939d87 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1430,10 +1430,9 @@ class QueryView(View): ): raise Forbidden("You do not have permission to view this query") - if canned_query.get("write"): - await _ensure_stored_query_execution_permissions( - datasette, db, canned_query, request.actor - ) + await _ensure_stored_query_execution_permissions( + datasette, db, canned_query, request.actor + ) # If database is immutable, return an error if not db.is_mutable: diff --git a/docs/authentication.rst b/docs/authentication.rst index 6e835c8d..453aaa19 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1285,7 +1285,7 @@ Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.i view-query ---------- -Actor is allowed to view (and execute) a saved query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size - this includes executing :ref:`canned_queries_writable`. +Actor is allowed to view a saved query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size. Executing an untrusted saved query also requires ``execute-sql`` or the relevant write permissions; trusted saved queries can execute with ``view-query`` alone. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) diff --git a/queries-plan.md b/queries-plan.md index f4b8049c..da6b7c92 100644 --- a/queries-plan.md +++ b/queries-plan.md @@ -25,7 +25,7 @@ Terminology change: these are now "queries", not "canned queries". Legacy code a - Query definitions currently come from `datasette.yaml` or the `canned_queries()` plugin hook. - `Datasette.get_canned_queries(database_name, actor)` calls that hook every time it needs query definitions. - `QueryResource.resources_sql()` currently enumerates databases and calls the hook for each one, because permissions and `/-/jump` need query resources. -- Query pages execute if the actor has `view-query` for `QueryResource(database, query)`. +- Query pages are visible if the actor has `view-query` for `QueryResource(database, query)`. Executing an untrusted stored query also checks `execute-sql` or the relevant write permissions. - Arbitrary SQL executes if the actor has `execute-sql` for `DatabaseResource(database)`. The main performance and architecture win is making query resource enumeration a direct SQL query against the internal database. @@ -145,9 +145,7 @@ Default execution rule for user-created writable queries: Implementation: -- Remove `view-query` from the broad `DEFAULT_ALLOW_ACTIONS` set. -- Replace it with query-aware default `view-query` permission SQL. -- Emit default `view-query` allows for non-private rows when Datasette is not running with `--default-deny`. +- Keep `view-query` in the broad `DEFAULT_ALLOW_ACTIONS` set, so saved queries remain visible by default in all-public Datasette. - Emit default `view-query` allows for the owning actor. - Use `restriction_sql` to limit private rows to their owner even when broader `view-query` permissions exist. - Have `QueryView` perform the fresh `execute-sql` or table-permission check before execution unless the row has `is_trusted=1`. @@ -424,7 +422,7 @@ The existing edit-SQL flow from query pages can continue to point back to arbitr - The old `canned_queries()` hook is no longer called by core. - `QueryResource.resources_sql()` returns rows from `queries`. - Database page and `/-/jump` list queries from the internal DB. -- `view-query` is no longer globally default-allowed; default query permissions come from the query-aware hook. +- `view-query` remains globally default-allowed, with `restriction_sql` narrowing private queries to their owner. - Private query is only visible to its owner, even when a broader `view-query` rule applies. - Non-trusted read-only query requires `execute-sql` to execute. - Trusted read-only query can be executed without `execute-sql` after `view-query` passes. diff --git a/tests/test_queries.py b/tests/test_queries.py index dde57dea..997f8b39 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -395,8 +395,16 @@ async def test_untrusted_shared_query_execution_requires_execute_sql(): owner_id="alice", ) - denied = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) - assert denied.status_code == 403 + denied_get = await ds.client.get( + "/data/shared_report.json", actor={"id": "viewer"} + ) + denied_post = await ds.client.post( + "/data/shared_report", + actor={"id": "viewer"}, + data={}, + ) + assert denied_get.status_code == 403 + assert denied_post.status_code == 403 ds.config["databases"]["data"]["permissions"]["execute-sql"] = {"id": "viewer"} allowed = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) From 866852eff603c219b8bf7d13f2a69b5ff032fa67 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 12:46:18 -0700 Subject: [PATCH 123/156] Clarifying comments --- datasette/default_permissions/defaults.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index ed0a6d66..32ad4ef1 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -80,6 +80,7 @@ async def default_query_permissions_sql( if action in {"update-query", "delete-query"}: if actor_id is None: return None + # Query owner can update/delete query return PermissionSQL( sql=""" SELECT database_name AS parent, name AS child, 1 AS allow, @@ -97,15 +98,15 @@ async def default_query_permissions_sql( params = {"query_owner_id": actor_id} rule_sqls = [] if actor_id is not None: - rule_sqls.append( - """ + # Query owner can view-query + rule_sqls.append(""" SELECT database_name AS parent, name AS child, 1 AS allow, 'query owner' AS reason FROM queries WHERE owner_id = :query_owner_id - """ - ) + """) + # restriction_sql enforces private queries ONLY visible to owner return PermissionSQL( sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, restriction_sql=""" From 71c76e38534378cbce8576771238a788feccf3ad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:08:19 -0700 Subject: [PATCH 124/156] Better faceting on /-/queries Ref https://github.com/simonw/datasette/pull/2741#issuecomment-4548321815 --- datasette/app.py | 69 +++++++++++++++++ datasette/templates/query_list.html | 94 +++++++++++++---------- datasette/views/database.py | 99 +++++++++++++++++++++++- tests/test_permissions.py | 8 +- tests/test_queries.py | 115 +++++++++++++++++++++++++--- 5 files changed, 330 insertions(+), 55 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 3329ee7e..1acdfcd8 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1298,6 +1298,75 @@ class Datasette: ) return self._query_row_to_dict(rows.first()) + async def count_queries( + self, + database=None, + *, + actor=None, + q=None, + is_write=None, + is_private=None, + is_trusted=None, + source=None, + owner_id=None, + ): + allowed_sql, allowed_params = await self.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + ) + params = dict(allowed_params) + where_clauses = [] + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + row = ( + await self.get_internal_database().execute( + """ + SELECT count(*) AS count + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + """.format( + allowed_sql=allowed_sql, + where=" AND ".join(where_clauses) or "1 = 1", + ), + params, + ) + ).first() + return row["count"] + async def list_queries( self, database=None, diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index 25259b3d..fa4859b1 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -9,7 +9,7 @@ max-width: 64rem; } .query-list-filters { - margin: 0.5rem 0 1rem; + margin: 0.5rem 0 0.75rem; } .query-list-search { align-items: center; @@ -32,43 +32,63 @@ line-height: 1.1; padding: 0.35rem 0.65rem; } -.query-list-filter-groups { +.query-list-facets { align-items: flex-start; display: flex; flex-wrap: wrap; - gap: 0.8rem 1.4rem; + gap: 1rem 1.6rem; + margin: 0 0 1rem; } -.query-list-filter-group { - border: 0; +.query-list-facet { + margin: 0; +} +.query-list-facet h2 { + font-size: 0.9rem; + line-height: 1.2; + margin: 0 0 0.35rem; +} +.query-list-facet ul { display: flex; flex-wrap: wrap; gap: 0.35rem; margin: 0; - min-width: 0; padding: 0; + list-style: none; } -.query-list-filter-group legend { - font-weight: 700; - margin: 0 0.45rem 0 0; - padding: 0; -} -.query-list-filter-group label { +.query-list-facet-link, +.query-list-facet-link:link, +.query-list-facet-link:visited, +.query-list-facet-link:hover, +.query-list-facet-link:focus, +.query-list-facet-link:active { align-items: center; border: 1px solid #c8d1dc; border-radius: 0.25rem; - cursor: pointer; + color: #39445a; display: inline-flex; font-size: 0.82rem; - gap: 0.3rem; + gap: 0.4rem; line-height: 1.1; padding: 0.35rem 0.55rem; + text-decoration: none; } -.query-list-filter-group input { - margin: 0; +.query-list-facet-link:hover { + border-color: #7ca5c8; + color: #1f5d85; } -.query-list-filter-group input:checked + span { +.query-list-facet-link-active { + background-color: #edf6fb; + border-color: #6d9fc0; font-weight: 700; } +.query-list-facet-disabled { + color: #7b8794; + cursor: default; +} +.query-list-facet-count { + color: #4f5b6d; + font-variant-numeric: tabular-nums; +} .query-list-results { border-collapse: collapse; font-size: 0.9rem; @@ -169,15 +189,6 @@ .query-list-search input[type=search] { max-width: none; } - .query-list-filter-group { - display: block; - } - .query-list-filter-group legend { - margin-bottom: 0.3rem; - } - .query-list-filter-group label { - margin: 0 0.25rem 0.35rem 0; - } } {% endblock %} @@ -198,24 +209,27 @@

      + {% if filters.is_write %}{% endif %} + {% if filters.is_private %}{% endif %} + {% if filters.source %}{% endif %} + {% if filters.owner_id %}{% endif %}

      -
      -
      - Mode - - - -
      -
      - Visibility - - - -
      -
      + + {% if queries %}
      diff --git a/datasette/views/database.py b/datasette/views/database.py index bd939d87..2e77d36b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1121,6 +1121,21 @@ class QueryParametersView(BaseView): return _block_framing(Response.json({"ok": True, "parameters": parameters})) +def _query_list_url(path, query_string, *, set_args=None, remove_args=None): + set_args = set_args or {} + remove_args = set(remove_args or ()) + skip = set(set_args) | remove_args | {"_next"} + pairs = [ + (key, value) + for key, value in parse_qsl(query_string, keep_blank_values=True) + if key not in skip + ] + for key, value in set_args.items(): + if value not in (None, ""): + pairs.append((key, value)) + return path + (("?" + urlencode(pairs)) if pairs else "") + + class QueryListView(BaseView): name = "query-list" @@ -1139,9 +1154,7 @@ class QueryListView(BaseView): default=20 if format_ == "html" else 50, ) is_write = _as_optional_bool(request.args.get("is_write"), "is_write") - is_private = _as_optional_bool( - request.args.get("is_private"), "is_private" - ) + is_private = _as_optional_bool(request.args.get("is_private"), "is_private") except QueryValidationError as ex: return _error([ex.message], ex.status) @@ -1173,6 +1186,80 @@ class QueryListView(BaseView): urlencode(pairs), ) + current_filters = { + "actor": request.actor, + "q": request.args.get("q") or None, + "is_write": is_write, + "is_private": is_private, + "source": request.args.get("source") or None, + "owner_id": request.args.get("owner_id") or None, + } + + async def facet_count(field, value): + if current_filters[field] is not None and current_filters[field] != value: + return 0 + filters = dict(current_filters) + filters[field] = value + return await self.ds.count_queries(database, **filters) + + def facet_href(field, value): + if current_filters[field] == value: + return _query_list_url( + query_list_path, + request.query_string, + remove_args=[field], + ) + if current_filters[field] is not None: + return None + return _query_list_url( + query_list_path, + request.query_string, + set_args={field: str(int(value))}, + ) + + async def facet_item(label, field, value): + count = await facet_count(field, value) + active = current_filters[field] == value + if not active and not count: + return None + return { + "label": label, + "count": count, + "href": facet_href(field, value) if active or count else None, + "active": active, + } + + async def facet_items(items): + return [ + item + for item in [ + await facet_item(label, field, value) + for label, field, value in items + ] + if item is not None + ] + + facets = [ + { + "title": "Mode", + "items": await facet_items( + [ + ("Read-only", "is_write", False), + ("Writable", "is_write", True), + ] + ), + }, + { + "title": "Visibility", + "items": await facet_items( + [ + ("Not private", "is_private", False), + ("Private", "is_private", True), + ] + ), + }, + ] + data = { "ok": True, "database": database, @@ -1188,6 +1275,7 @@ class QueryListView(BaseView): "show_trusted_note": any(query["is_trusted"] for query in page["queries"]), "query_list_path": query_list_path, "show_database": database is None, + "facets": facets, "filters": { "q": request.args.get("q") or "", "is_write": request.args.get("is_write") or "", @@ -1715,6 +1803,9 @@ class QueryView(View): } ) metadata = await datasette.get_database_metadata(database) + if canned_query: + metadata = dict(canned_query) + metadata.pop("source", None) renderers = {} for key, (_, can_render) in datasette.renderers.items(): @@ -1865,7 +1956,7 @@ class QueryView(View): ) ), show_hide_hidden=markupsafe.Markup(show_hide_hidden), - metadata=canned_query or metadata, + metadata=metadata, alternate_url_json=alternate_url_json, select_templates=[ f"{'*' if template_name == template.name else ''}{template_name}" diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 4f342d8f..eb6cee9f 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -948,9 +948,13 @@ async def test_permissions_in_config( resource_obj = DatabaseResource(database=resource) elif isinstance(resource, tuple) and len(resource) == 2: if action == "view-query": - resource_obj = QueryResource(database=resource[0], query=resource[1]) + resource_obj = QueryResource( + database=resource[0], query=resource[1] + ) else: - resource_obj = TableResource(database=resource[0], table=resource[1]) + resource_obj = TableResource( + database=resource[0], table=resource[1] + ) result = await perms_ds.allowed( action=action, resource=resource_obj, actor=actor diff --git a/tests/test_queries.py b/tests/test_queries.py index 997f8b39..36f7107a 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -395,9 +395,7 @@ async def test_untrusted_shared_query_execution_requires_execute_sql(): owner_id="alice", ) - denied_get = await ds.client.get( - "/data/shared_report.json", actor={"id": "viewer"} - ) + denied_get = await ds.client.get("/data/shared_report.json", actor={"id": "viewer"}) denied_post = await ds.client.post( "/data/shared_report", actor={"id": "viewer"}, @@ -608,6 +606,27 @@ async def test_query_list_and_definition_api(): assert definition_response.json()["query"]["title"] == "Demo query 01" +@pytest.mark.asyncio +async def test_query_page_does_not_show_internal_source(): + ds = Datasette(memory=True) + ds.add_memory_database("query_page_source", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "stored_report", + "select 1 as one", + title="Stored report", + source="user", + owner_id="root", + ) + + response = await ds.client.get("/data/stored_report", actor={"id": "root"}) + + assert response.status_code == 200 + assert "Stored report" in response.text + assert "Data source:" not in response.text + + @pytest.mark.asyncio async def test_query_list_search_filter_and_html(): ds = Datasette(memory=True) @@ -632,6 +651,15 @@ async def test_query_list_search_filter_and_html(): is_trusted=True, source="config", ) + await ds.add_query( + "data", + "writable_query", + "insert into dogs (name) values (:name)", + title="Writable query", + is_write=True, + source="user", + owner_id="root", + ) html_response = await ds.client.get( "/data/-/queries?q=02", @@ -649,13 +677,21 @@ async def test_query_list_search_filter_and_html(): "/data/-/queries.json?is_private=1", actor={"id": "root"}, ) + filtered_write_response = await ds.client.get( + "/data/-/queries?is_write=1", + actor={"id": "root"}, + ) + filtered_private_response = await ds.client.get( + "/data/-/queries?is_private=1", + actor={"id": "root"}, + ) assert html_response.status_code == 200 assert "Demo query 02" in html_response.text assert "Demo query 01" not in html_response.text assert 'class="query-list-results"' in html_response.text - assert "Mode" in html_response.text - assert 'type="radio" name="is_private" value="1"' in html_response.text + assert 'class="query-list-facets"' in html_response.text + assert 'type="radio"' not in html_response.text assert "Only the owning actor can view this query." not in html_response.text assert ( "Execution skips the usual SQL and write permission checks" @@ -667,14 +703,75 @@ async def test_query_list_search_filter_and_html(): assert '' not in flags_response.text assert 'class="query-list-owner">root' in flags_response.text assert 'class="query-list-pill">Read-only' in flags_response.text - assert 'class="query-list-pill query-list-pill-private">Private' in flags_response.text - assert 'class="query-list-pill query-list-pill-trusted">Trusted' in flags_response.text + assert ( + 'class="query-list-pill query-list-pill-write">Writable' + in flags_response.text + ) + assert ( + 'class="query-list-pill query-list-pill-private">Private' + in flags_response.text + ) + assert ( + 'class="query-list-pill query-list-pill-trusted">Trusted' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_write=0">Read-only5' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_write=1">Writable1' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_private=0">Not private5' + in flags_response.text + ) + assert ( + 'href="/data/-/queries?is_private=1">Private1' + in flags_response.text + ) assert "Only the owning actor can view this query." in flags_response.text - assert "Execution skips the usual SQL and write permission checks" in flags_response.text + assert ( + "Execution skips the usual SQL and write permission checks" + in flags_response.text + ) assert json_response.json()["queries"][0]["name"] == "demo_query_02" assert [query["name"] for query in filtered_response.json()["queries"]] == [ "private_query" ] + assert "Writable query" in filtered_write_response.text + assert "Demo query 01" not in filtered_write_response.text + assert ( + 'query-list-facet-link query-list-facet-link-active" href="/data/-/queries"' + in filtered_write_response.text + ) + assert ( + 'Read-only0' + not in filtered_write_response.text + ) + assert ( + 'href="/data/-/queries?is_write=1&is_private=0">Not private1' + in filtered_write_response.text + ) + assert ( + 'Private0' + not in filtered_write_response.text + ) + assert "Private query" in filtered_private_response.text + assert "Demo query 01" not in filtered_private_response.text + assert ( + 'href="/data/-/queries?is_private=1&is_write=0">Read-only1' + in filtered_private_response.text + ) + assert ( + 'Writable0' + not in filtered_private_response.text + ) + assert ( + 'Not private0' + not in filtered_private_response.text + ) @pytest.mark.asyncio @@ -1313,7 +1410,7 @@ async def test_user_writable_query_execution_rechecks_table_permissions(): "insert-row": {"id": "alice"}, } } - } + }, } } }, From 0fcaa5792ba73143661515af0088d7e5d968e96c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:12:07 -0700 Subject: [PATCH 125/156] Style query operations on create query Made it consistent with the SQL write page. --- .../_execute_write_analysis_styles.html | 37 +++++++++++++++++++ datasette/templates/execute_write.html | 36 +----------------- datasette/templates/query_create.html | 19 +++++----- tests/test_queries.py | 6 ++- 4 files changed, 52 insertions(+), 46 deletions(-) create mode 100644 datasette/templates/_execute_write_analysis_styles.html diff --git a/datasette/templates/_execute_write_analysis_styles.html b/datasette/templates/_execute_write_analysis_styles.html new file mode 100644 index 00000000..f20e67b2 --- /dev/null +++ b/datasette/templates/_execute_write_analysis_styles.html @@ -0,0 +1,37 @@ + diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 46f58c3b..414d4af7 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -40,42 +40,8 @@ border-radius: 0.25rem; min-width: 13rem; } -.execute-write-analysis { - border-collapse: collapse; - font-size: 0.9rem; - margin: 0.25rem 0 1rem; - min-width: 44rem; -} -.execute-write-analysis th, -.execute-write-analysis td { - border-bottom: 1px solid #d7dde5; - padding: 0.45rem 0.7rem; - text-align: left; - vertical-align: top; -} -.execute-write-analysis th { - background-color: #edf6fb; - border-top: 1px solid #d7dde5; - color: #39445a; - font-weight: 700; -} -.execute-write-analysis tbody tr:nth-child(even) { - background-color: rgba(39, 104, 144, 0.05); -} -.execute-write-analysis code { - background: transparent; - font-size: 0.9em; - white-space: nowrap; -} -.execute-write-analysis-allowed { - color: #267a3e; - font-weight: 700; -} -.execute-write-analysis-denied { - color: #b00020; - font-weight: 700; -} +{% include "_execute_write_analysis_styles.html" %} {% include "_sql_parameter_styles.html" %} {% endblock %} diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index 686d971e..2d8a9122 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -5,6 +5,7 @@ {% block extra_head %} {{- super() -}} {% include "_codemirror.html" %} +{% include "_execute_write_analysis_styles.html" %} {% endblock %} {% block body_class %}query-create db-{{ database|to_css_class }}{% endblock %} @@ -32,30 +33,28 @@

      Execute write SQL

      {% endif %} -

      Analysis

      +

      Query operations

      {% if analysis_error %}

      {{ analysis_error }}

      {% elif analysis_rows %} -
      Mode
      +
      - + - {% for row in analysis_rows %} - - - - - - + + + + + {% endfor %} diff --git a/tests/test_queries.py b/tests/test_queries.py index 36f7107a..c27c23da 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -998,7 +998,11 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): assert "Create query" in create_response.text assert "Read-only" in create_response.text assert "Writable" in create_response.text - assert "required permission" in create_response.text + assert "

      Query operations

      " in create_response.text + assert '
      Operation Database Tablerequired permissionRequired permission AllowedSource
      {{ row.operation }}{{ row.database }}{{ row.table }}{{ row.required_permission }}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}{{ row.source or "" }}{{ row.operation }}{{ row.database }}{{ row.table }}{% if row.required_permission %}{{ row.required_permission }}{% endif %}{% if row.allowed is none %}{% elif row.allowed %}yes{% else %}no{% endif %}
      ' in create_response.text + assert '' in create_response.text + assert '' not in create_response.text + assert "" in create_response.text assert query_response.status_code == 200 assert "Save query" in query_response.text assert "/data/-/queries/-/create?sql=select+%2A+from+dogs" in query_response.text From 70b23ff4a55528083512fab96aa50725f415cbe4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:47:24 -0700 Subject: [PATCH 126/156] Tweaked save query link --- datasette/templates/query.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index f74d21f1..1900bd31 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -66,7 +66,7 @@ {% if not hide_sql %}{% endif %} {{ show_hide_hidden }} - {% if save_query_url %}Save query{% endif %} + {% if save_query_url %}Save this query{% endif %} {% if canned_query and edit_sql_url %}Edit SQL{% endif %}

      From eb7c25c57cf914629c08eaa477d0709b0f41efeb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:48:40 -0700 Subject: [PATCH 127/156] Major redesign of create saved query UI https://github.com/simonw/datasette/pull/2741#issuecomment-4548707129 --- datasette/app.py | 6 +- datasette/static/app.css | 4 + .../_execute_write_analysis_scripts.html | 111 +++++++ .../_execute_write_analysis_styles.html | 4 + .../templates/_sql_parameter_scripts.html | 17 +- datasette/templates/execute_write.html | 88 +----- datasette/templates/query_create.html | 296 +++++++++++++++--- datasette/views/database.py | 181 ++++++++--- tests/test_queries.py | 170 +++++++++- 9 files changed, 705 insertions(+), 172 deletions(-) create mode 100644 datasette/templates/_execute_write_analysis_scripts.html diff --git a/datasette/app.py b/datasette/app.py index 1acdfcd8..8936b099 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -50,7 +50,7 @@ from .views.database import ( ExecuteWriteView, TableCreateView, QueryView, - QueryCreateView, + QueryCreateAnalyzeView, QueryDeleteView, QueryDefinitionView, GlobalQueryListView, @@ -2820,8 +2820,8 @@ class Datasette: r"/(?P[^\/\.]+)/-/queries(\.(?Pjson))?$", ) add_route( - QueryCreateView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/-/create$", + QueryCreateAnalyzeView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/analyze$", ) add_route( QueryInsertView.as_view(self), diff --git a/datasette/static/app.css b/datasette/static/app.css index c21d0dc4..4f4db133 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1414,6 +1414,10 @@ svg.dropdown-menu-icon { position: relative; top: 1px; } +.save-query { + display: inline-block; + margin-left: 0.45em; +} .blob-download { display: block; diff --git a/datasette/templates/_execute_write_analysis_scripts.html b/datasette/templates/_execute_write_analysis_scripts.html new file mode 100644 index 00000000..a19bae13 --- /dev/null +++ b/datasette/templates/_execute_write_analysis_scripts.html @@ -0,0 +1,111 @@ + diff --git a/datasette/templates/_execute_write_analysis_styles.html b/datasette/templates/_execute_write_analysis_styles.html index f20e67b2..165cfe9f 100644 --- a/datasette/templates/_execute_write_analysis_styles.html +++ b/datasette/templates/_execute_write_analysis_styles.html @@ -34,4 +34,8 @@ color: #b00020; font-weight: 700; } +.execute-write-analysis-na { + color: #687386; + font-style: italic; +} diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html index 68e46069..159a141c 100644 --- a/datasette/templates/_sql_parameter_scripts.html +++ b/datasette/templates/_sql_parameter_scripts.html @@ -215,9 +215,10 @@ window.datasetteSqlParameters = (() => { if (!form) { return null; } + const shouldRenderParameters = options.renderParameters !== false; const section = options.section || form.querySelector("[data-sql-parameters-section]"); - if (!section) { + if (shouldRenderParameters && !section) { return null; } const manager = { @@ -225,12 +226,16 @@ window.datasetteSqlParameters = (() => { section, allowExpand: options.allowExpand === undefined - ? section.dataset.allowExpand === "1" + ? section + ? section.dataset.allowExpand === "1" + : false : options.allowExpand, parameterState: new Map(), }; - bindParameterControls(manager); - syncParameterState(manager); + if (section) { + bindParameterControls(manager); + syncParameterState(manager); + } const url = options.url || form.dataset.parametersUrl; let refreshTimer = null; @@ -254,7 +259,9 @@ window.datasetteSqlParameters = (() => { if (!response.ok) { throw new Error((data.errors || [response.statusText]).join("; ")); } - renderParameters(manager, data.parameters || []); + if (shouldRenderParameters) { + renderParameters(manager, data.parameters || []); + } if (options.onData) { options.onData(data, manager); } diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 414d4af7..7a627a7a 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -131,6 +131,7 @@ if (executeWriteSqlInput && !executeWriteSqlInput.value) { {% include "_codemirror_foot.html" %} {% include "_sql_parameter_scripts.html" %} +{% include "_execute_write_analysis_scripts.html" %} + + {% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index 2e77d36b..aafcf40b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -551,6 +551,17 @@ def _wants_json(request, is_json, data): ) +def _query_create_form_error_message(message): + return { + "Query name is required": "URL is required", + "Invalid query name": "Invalid URL", + "Query name conflicts with a table or view": ( + "URL conflicts with an existing table or view" + ), + "Query already exists": "A query already exists at that URL", + }.get(message, message) + + async def _json_or_form_payload(request): content_type = request.headers.get("content-type", "") if content_type.startswith("application/json"): @@ -731,6 +742,54 @@ async def _execute_write_analysis_data(datasette, db, sql, actor): } +async def _query_create_analysis_data(datasette, db, sql, actor): + has_sql = bool(sql and sql.strip()) + parameter_names = [] + analysis_rows = [] + analysis_error = None + if has_sql: + try: + parameter_names = _derived_query_parameters(sql) + params = {parameter: "" for parameter in parameter_names} + analysis = await db.analyze_sql(sql, params) + analysis_rows = await _analysis_rows_with_permissions( + datasette, analysis, actor + ) + except (QueryValidationError, sqlite3.DatabaseError) as ex: + analysis_error = getattr(ex, "message", str(ex)) + return { + "ok": analysis_error is None, + "parameters": parameter_names, + "analysis_error": analysis_error, + "analysis_rows": analysis_rows, + "has_sql": has_sql, + "analysis_is_write": bool( + analysis_rows and any(row["required_permission"] for row in analysis_rows) + ), + "save_disabled": bool( + (not has_sql) + or analysis_error + or any(row["allowed"] is False for row in analysis_rows) + ), + } + + +async def _query_create_form_context( + datasette, request, db, *, sql="", name="", title="", description="", is_private=True +): + analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor) + return { + "database": db.name, + "database_color": db.color, + "sql": sql, + "name": name, + "title": title, + "description": description, + "is_private": is_private, + **analysis_data, + } + + async def _inserted_row_url(datasette, db, analysis, cursor): if cursor.rowcount != 1: return None @@ -1307,6 +1366,35 @@ class QueryCreateView(BaseView): name = "query-create" has_json_alternate = False + async def _render_form( + self, + request, + db, + *, + sql="", + name="", + title="", + description="", + is_private=True, + status=200, + ): + response = await self.render( + ["query_create.html"], + request, + await _query_create_form_context( + self.ds, + request, + db, + sql=sql, + name=name, + title=title, + description=description, + is_private=is_private, + ), + ) + response.status = status + return response + async def get(self, request): db = await self.ds.resolve_database(request) await self.ds.ensure_permission( @@ -1320,46 +1408,61 @@ class QueryCreateView(BaseView): actor=request.actor, ) - sql = request.args.get("sql") or "" - analysis_error = None - analysis_rows = [] - parameter_names = [] - if sql: - try: - parameter_names = _derived_query_parameters(sql) - params = {parameter: "" for parameter in parameter_names} - analysis = await db.analyze_sql(sql, params) - analysis_rows = await _analysis_rows_with_permissions( - self.ds, analysis, request.actor - ) - except (QueryValidationError, sqlite3.DatabaseError) as ex: - analysis_error = getattr(ex, "message", str(ex)) + return await self._render_form(request, db, sql=request.args.get("sql") or "") - return await self.render( - ["query_create.html"], - request, - { - "database": db.name, - "database_color": db.color, - "sql": sql, - "parameter_names": parameter_names, - "analysis_error": analysis_error, - "analysis_rows": analysis_rows, - "analysis_is_write": bool( - analysis_rows - and any(row["required_permission"] for row in analysis_rows) - ), - "save_disabled": bool( - analysis_error - or any(row["allowed"] is False for row in analysis_rows) - ), - }, + +class QueryCreateAnalyzeView(BaseView): + name = "query-create-analyze" + has_json_alternate = False + + async def get(self, request): + db = await self.ds.resolve_database(request) + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need execute-sql"], 403)) + if not await self.ds.allowed( + action="insert-query", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + return _block_framing(_error(["Permission denied: need insert-query"], 403)) + + invalid_keys = set(request.args) - {"sql"} + if invalid_keys: + return _block_framing( + _error( + ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))], + 400, + ) + ) + sql = request.args.get("sql") or "" + return _block_framing( + Response.json( + await _query_create_analysis_data(self.ds, db, sql, request.actor) + ) ) -class QueryInsertView(BaseView): +class QueryInsertView(QueryCreateView): name = "query-insert" + async def _error_response(self, request, db, query_data, message, status): + message = _query_create_form_error_message(message) + self.ds.add_message(request, message, self.ds.ERROR) + return await self._render_form( + request, + db, + sql=query_data.get("sql") or "", + name=query_data.get("name") or "", + title=query_data.get("title") or "", + description=query_data.get("description") or "", + is_private=_as_bool(query_data.get("is_private", True)), + status=status, + ) + async def post(self, request): db = await self.ds.resolve_database(request) if not await self.ds.allowed( @@ -1375,6 +1478,8 @@ class QueryInsertView(BaseView): ): return _error(["Permission denied: need insert-query"], 403) + is_json = False + query_data = {} try: data, is_json = await _json_or_form_payload(request) if not isinstance(data, dict): @@ -1384,6 +1489,10 @@ class QueryInsertView(BaseView): raise QueryValidationError("JSON must contain a query dictionary") prepared = await _prepare_query_create(self.ds, request, db, query_data) except QueryValidationError as ex: + if not is_json and isinstance(query_data, dict): + return await self._error_response( + request, db, query_data, ex.message, ex.status + ) return _error([ex.message], ex.status) prepared.pop("analysis") @@ -1391,6 +1500,8 @@ class QueryInsertView(BaseView): try: await self.ds.add_query(db.name, name, replace=False, **prepared) except sqlite3.IntegrityError as ex: + if not is_json and isinstance(query_data, dict): + return await self._error_response(request, db, query_data, str(ex), 400) return _error([str(ex)], 400) query = await self.ds.get_query(db.name, name) @@ -1896,7 +2007,7 @@ class QueryView(View): ): save_query_url = ( datasette.urls.database(database) - + "/-/queries/-/create?" + + "/-/queries/insert?" + urlencode({"sql": sql}) ) diff --git a/tests/test_queries.py b/tests/test_queries.py index c27c23da..32cdfae3 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -986,6 +986,14 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): await ds.invoke_startup() create_response = await ds.client.get( + "/data/-/queries/insert?sql=select+*+from+dogs", + actor={"id": "root"}, + ) + blank_create_response = await ds.client.get( + "/data/-/queries/insert", + actor={"id": "root"}, + ) + old_create_response = await ds.client.get( "/data/-/queries/-/create?sql=select+*+from+dogs", actor={"id": "root"}, ) @@ -996,16 +1004,171 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): assert create_response.status_code == 200 assert "Create query" in create_response.text - assert "Read-only" in create_response.text assert "Writable" in create_response.text + assert 'type="radio"' not in create_response.text + assert 'name="parameters"' not in create_response.text + assert 'id="query-parameters"' not in create_response.text + assert 'class="query-create-field"' in create_response.text + assert '' not in create_response.text + assert '' in create_response.text + assert '' in create_response.text + assert '/data/' in create_response.text + assert ( + '' + in create_response.text + ) + assert 'function slugify(value)' in create_response.text + assert 'data-analyze-url="/data/-/queries/analyze"' in create_response.text + assert "setupSqlParameterRefresh" in create_response.text + assert "renderParameters: false" in create_response.text + assert "datasetteSqlAnalysis.renderAnalysis" in create_response.text + assert "data-query-create-submit" in create_response.text + assert "data-query-create-writable" in create_response.text + assert ( + "Queries marked private can only be seen by you, their creator." + in create_response.text + ) assert "

      Query operations

      " in create_response.text assert '
      Required permissionSourceread
      ' in create_response.text assert '' in create_response.text assert '' not in create_response.text assert "" in create_response.text + assert ( + create_response.text.count( + '' + ) + == 2 + ) + assert create_response.text.index('value="Save query"') < create_response.text.index( + "

      Query operations

      " + ) + assert blank_create_response.status_code == 200 + assert ( + '
      Required permissionSourcereadn/a
      ' in response.text assert '' in response.text assert "" in response.text From 5dca2dc9beea96c52e6a9c806df66c9a1f2f7874 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 13:54:47 -0700 Subject: [PATCH 128/156] Show query count on database page --- datasette/templates/database.html | 2 +- datasette/views/database.py | 18 +++++++++++++++++- tests/test_queries.py | 11 ++++++----- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/datasette/templates/database.html b/datasette/templates/database.html index 62f9c620..371f6a22 100644 --- a/datasette/templates/database.html +++ b/datasette/templates/database.html @@ -59,7 +59,7 @@ {% endfor %} {% if queries_more %} -

      View all queries

      +

      View {{ "{:,}".format(queries_count) }} quer{% if queries_count == 1 %}y{% else %}ies{% endif %}

      {% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index feb38619..d40d69d1 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -102,6 +102,11 @@ class DatabaseView(View): ) canned_queries = queries_page["queries"] queries_more = queries_page["has_more"] + queries_count = ( + await datasette.count_queries(database, actor=request.actor) + if queries_more + else len(canned_queries) + ) async def database_actions(): links = [] @@ -134,6 +139,7 @@ class DatabaseView(View): "views": sql_views, "queries": canned_queries, "queries_more": queries_more, + "queries_count": queries_count, "allow_execute_sql": allow_execute_sql, "table_columns": ( await _table_columns(datasette, database) if allow_execute_sql else {} @@ -168,6 +174,7 @@ class DatabaseView(View): views=sql_views, queries=canned_queries, queries_more=queries_more, + queries_count=queries_count, allow_execute_sql=allow_execute_sql, table_columns=( await _table_columns(datasette, database) @@ -219,6 +226,7 @@ class DatabaseContext(Context): queries_more: bool = field( metadata={"help": "Boolean indicating if more saved queries are available"} ) + queries_count: int = field(metadata={"help": "Count of visible saved queries"}) allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) @@ -775,7 +783,15 @@ async def _query_create_analysis_data(datasette, db, sql, actor): async def _query_create_form_context( - datasette, request, db, *, sql="", name="", title="", description="", is_private=True + datasette, + request, + db, + *, + sql="", + name="", + title="", + description="", + is_private=True, ): analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor) return { diff --git a/tests/test_queries.py b/tests/test_queries.py index 32cdfae3..09b41645 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -458,9 +458,10 @@ async def test_database_page_query_preview_is_limited(): assert html_response.status_code == 200 assert "Demo query 05" in html_response.text assert "Demo query 06" not in html_response.text - assert 'href="/data/-/queries"' in html_response.text + assert 'View 25 queries' in html_response.text assert len(json_response.json()["queries"]) == 5 assert json_response.json()["queries_more"] is True + assert json_response.json()["queries_count"] == 25 @pytest.mark.asyncio @@ -1017,7 +1018,7 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): '' in create_response.text ) - assert 'function slugify(value)' in create_response.text + assert "function slugify(value)" in create_response.text assert 'data-analyze-url="/data/-/queries/analyze"' in create_response.text assert "setupSqlParameterRefresh" in create_response.text assert "renderParameters: false" in create_response.text @@ -1039,9 +1040,9 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): ) == 2 ) - assert create_response.text.index('value="Save query"') < create_response.text.index( - "

      Query operations

      " - ) + assert create_response.text.index( + 'value="Save query"' + ) < create_response.text.index("

      Query operations

      ") assert blank_create_response.status_code == 200 assert ( '
      Required permissioninsert
      ' in create_response.text assert '' in create_response.text @@ -1053,6 +1067,12 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): "

      Analysis will show each affected table and required permission.

      " not in blank_create_response.text ) + assert "Enter SQL to analyze this query." in blank_create_response.text + assert write_create_response.status_code == 200 + assert ( + 'This query updates data in the database.' + in write_create_response.text + ) assert query_response.status_code == 200 assert "Save this query" in query_response.text assert "/data/-/queries/insert?sql=select+%2A+from+dogs" in query_response.text From 024b9117725bbed17396a5a4b3f48663c23337f5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:09:53 -0700 Subject: [PATCH 130/156] Clarifying comment https://github.com/simonw/datasette/pull/2741/changes#r3306856046 --- datasette/default_permissions/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/default_permissions/__init__.py b/datasette/default_permissions/__init__.py index a9f2d8bd..6cd46f04 100644 --- a/datasette/default_permissions/__init__.py +++ b/datasette/default_permissions/__init__.py @@ -26,6 +26,7 @@ from .restrictions import ( from .root import root_user_permissions_sql as root_user_permissions_sql from .config import config_permissions_sql as config_permissions_sql from .defaults import ( + # Avoid "datasette.default_permissions" does not explicitly export attribute default_allow_sql_check as default_allow_sql_check, default_action_permissions_sql as default_action_permissions_sql, default_query_permissions_sql as default_query_permissions_sql, From ac6ee097dd06050188d44c6d4b17a98a12c7b481 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:10:48 -0700 Subject: [PATCH 131/156] Disallow update/delete of private queries If a user does not own a private query they cannot update or delete it either, even if they have global update-query. https://github.com/simonw/datasette/pull/2741/changes#r3306417463 --- datasette/default_permissions/defaults.py | 33 ++++----- tests/test_queries.py | 81 +++++++++++++++++++++++ 2 files changed, 95 insertions(+), 19 deletions(-) diff --git a/datasette/default_permissions/defaults.py b/datasette/default_permissions/defaults.py index 32ad4ef1..5bc74425 100644 --- a/datasette/default_permissions/defaults.py +++ b/datasette/default_permissions/defaults.py @@ -77,36 +77,31 @@ async def default_query_permissions_sql( ) -> Optional[PermissionSQL]: actor_id = actor.get("id") if isinstance(actor, dict) else None - if action in {"update-query", "delete-query"}: - if actor_id is None: - return None - # Query owner can update/delete query - return PermissionSQL( - sql=""" - SELECT database_name AS parent, name AS child, 1 AS allow, - 'query owner' AS reason - FROM queries - WHERE source = 'user' - AND owner_id = :query_owner_id - """, - params={"query_owner_id": actor_id}, - ) - - if action != "view-query": + if action not in {"view-query", "update-query", "delete-query"}: return None params = {"query_owner_id": actor_id} rule_sqls = [] if actor_id is not None: - # Query owner can view-query - rule_sqls.append(""" + if action in {"update-query", "delete-query"}: + # Query owner can update/delete query + rule_sqls.append(""" + SELECT database_name AS parent, name AS child, 1 AS allow, + 'query owner' AS reason + FROM queries + WHERE source = 'user' + AND owner_id = :query_owner_id + """) + else: + # Query owner can view-query + rule_sqls.append(""" SELECT database_name AS parent, name AS child, 1 AS allow, 'query owner' AS reason FROM queries WHERE owner_id = :query_owner_id """) - # restriction_sql enforces private queries ONLY visible to owner + # restriction_sql enforces private queries ONLY visible/mutable by owner return PermissionSQL( sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None, restriction_sql=""" diff --git a/tests/test_queries.py b/tests/test_queries.py index f888dda0..26a0748c 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1581,6 +1581,87 @@ async def test_query_owner_gets_update_delete_and_writable_view_defaults(): ) +@pytest.mark.asyncio +async def test_private_query_restricts_broad_update_delete_permissions(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "update-query": {"id": "bob"}, + "delete-query": {"id": "bob"}, + }, + }, + }, + }, + ) + ds.add_memory_database("query_broad_update_delete", name="data") + await ds.invoke_startup() + await ds.add_query( + "data", + "alice_private", + "select 1", + is_private=True, + source="user", + owner_id="alice", + ) + await ds.add_query( + "data", + "alice_public", + "select 2", + is_private=False, + source="user", + owner_id="alice", + ) + + for action in ("update-query", "delete-query"): + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), + actor={"id": "alice"}, + ) + assert not await ds.allowed( + action=action, + resource=QueryResource("data", "alice_private"), + actor={"id": "bob"}, + ) + assert await ds.allowed( + action=action, + resource=QueryResource("data", "alice_public"), + actor={"id": "bob"}, + ) + + private_update_response = await ds.client.post( + "/data/alice_private/-/update", + actor={"id": "bob"}, + json={"update": {"title": "Nope"}}, + ) + private_delete_response = await ds.client.post( + "/data/alice_private/-/delete", + actor={"id": "bob"}, + json={}, + ) + public_update_response = await ds.client.post( + "/data/alice_public/-/update", + actor={"id": "bob"}, + json={"update": {"title": "Bob can edit public queries"}}, + ) + public_delete_response = await ds.client.post( + "/data/alice_public/-/delete", + actor={"id": "bob"}, + json={}, + ) + + assert private_update_response.status_code == 403 + assert private_delete_response.status_code == 403 + assert public_update_response.status_code == 200 + assert public_delete_response.status_code == 200 + assert await ds.get_query("data", "alice_private") is not None + assert await ds.get_query("data", "alice_public") is None + + @pytest.mark.asyncio async def test_user_writable_query_execution_rechecks_table_permissions(): ds = Datasette( From 180a6a86fd77ac43f6cf3bfb7d7f9150003da419 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:16:10 -0700 Subject: [PATCH 132/156] Remove queries-plan.md We do not need this any more. It can live forever in Git history. --- queries-plan.md | 446 ------------------------------------------------ 1 file changed, 446 deletions(-) delete mode 100644 queries-plan.md diff --git a/queries-plan.md b/queries-plan.md deleted file mode 100644 index da6b7c92..00000000 --- a/queries-plan.md +++ /dev/null @@ -1,446 +0,0 @@ -# Queries in the internal database - -Plan for . - -## Goal - -Move named query definitions into Datasette's internal database, so hundreds or thousands of queries can be listed, searched, permission-filtered, managed, and executed efficiently. - -Terminology change: these are now "queries", not "canned queries". Legacy code and documentation can mention the old name only when describing compatibility or migration. - -## Decisions so far - -- Internal table name: `queries`. -- Query definitions should use real columns, not a JSON blob for all options. -- Query parameter names live in a `parameters` text column as a JSON array. No default values for parameters in this pass. -- No separate index is needed for the privacy/trust flags yet. -- User-created queries require `execute-sql` and `insert-query` on the database. They default to private, and writable queries additionally require matching table write permissions discovered by `Database.analyze_sql()`. -- Configured queries default to trusted, which means actors who can view them can execute them without also holding `execute-sql` or the relevant write permissions. Config can opt out with `is_trusted: false`. -- Add `update-query` and `delete-query`, so administrators can manage queries created by other users. -- Remove the old `canned_queries()` hook from core. If we want compatibility later, build a separate `datasette-old-canned-queries` plugin. -- Writable user-created queries can be supported using `Database.analyze_sql()`, provided we fail closed when analysis cannot prove the required permissions. - -## Current shape - -- Query definitions currently come from `datasette.yaml` or the `canned_queries()` plugin hook. -- `Datasette.get_canned_queries(database_name, actor)` calls that hook every time it needs query definitions. -- `QueryResource.resources_sql()` currently enumerates databases and calls the hook for each one, because permissions and `/-/jump` need query resources. -- Query pages are visible if the actor has `view-query` for `QueryResource(database, query)`. Executing an untrusted stored query also checks `execute-sql` or the relevant write permissions. -- Arbitrary SQL executes if the actor has `execute-sql` for `DatabaseResource(database)`. - -The main performance and architecture win is making query resource enumeration a direct SQL query against the internal database. - -## Proposed internal schema - -Start with one `queries` table. - -```sql -CREATE TABLE IF NOT EXISTS queries ( - database_name TEXT NOT NULL, - name TEXT NOT NULL, - sql TEXT NOT NULL, - title TEXT, - description TEXT, - description_html TEXT, - options TEXT NOT NULL DEFAULT '{}', - parameters TEXT NOT NULL DEFAULT '[]', - is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), - is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), - source TEXT NOT NULL DEFAULT 'user', - owner_id TEXT, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (database_name, name) -); - -CREATE INDEX IF NOT EXISTS queries_owner_idx - ON queries(owner_id); -``` - -Column notes: - -- `database_name`, `name`, and `sql` are the routing and execution core. -- Display fields become columns: `title`, `description`, and `description_html`. -- Less common presentation and writable-query behavior lives in `options`, stored as a JSON object. That covers `hide_sql`, `fragment`, `on_success_message`, `on_success_message_sql`, `on_success_redirect`, `on_error_message`, and `on_error_redirect`. -- `parameters` is a JSON array of parameter names, stored as text. This preserves explicit parameter order, but does not support labels or default values. -- Existing writable query behavior gets `is_write` as a column. Success/error messages, success/error redirects, and `on_success_message_sql` are stored in `options`. -- `is_private` means the query is only visible to its owning actor. This is enforced as a permission restriction, so broader `view-query` grants do not expose private rows. -- `is_trusted` means execution skips the usual `execute-sql` or write-permission checks after `view-query` has allowed access. -- `source` distinguishes `user`, `config`, and `plugin` rows. -- `owner_id` is the actor id for user-created rows. It is `NULL` for config/plugin rows. - -No separate index is needed on `(database_name, name)` because the primary key already creates one. - -`QueryResource.resources_sql()` can become: - -```sql -SELECT q.database_name AS parent, q.name AS child -FROM queries q -JOIN catalog_databases cd ON cd.database_name = q.database_name -``` - -The join keeps persisted queries for detached databases from appearing as live resources. - -## Config and plugin migration - -`datasette.yaml` can continue to support `databases: {db}: queries:` blocks, but core should import them directly into the internal `queries` tables at startup: - -1. Ensure the internal schema exists. -2. Delete previous `source='config'` rows. -3. Read configured query blocks for each live database. -4. Normalize string definitions to `{"sql": ...}`. -5. Insert rows into `queries`, storing explicit `params` as JSON in `parameters`. - -Plugins should move to: - -```python -await datasette.add_query(...) -await datasette.remove_query(...) -``` - -Remove the old `canned_queries()` hookspec and all core calls to it. If compatibility is needed, build `datasette-old-canned-queries` later as a plugin that restores the hook and imports old hook results using `datasette.add_query()`. - -## Permission model - -Add core actions: - -- `insert-query`, database-level, for creating queries in a database. -- `update-query`, query-level, for modifying existing query definitions. -- `delete-query`, query-level, for deleting existing query definitions. - -User-created query creation requires: - -- `execute-sql` on `DatabaseResource(database)` -- `insert-query` on `DatabaseResource(database)` -- If analysis shows the query is writable, the table-level write permissions described in the writable query section. - -Updating an existing query requires: - -- `update-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. -- If the SQL changes, also require `execute-sql` on the database. -- If the changed SQL is writable, also require the table-level write permissions described in the writable query section. - -Deleting an existing query requires: - -- `delete-query` on `QueryResource(database, query)` or default owner permission for a user-owned row. - -Default owner permissions: - -- For `source='user' AND owner_id = actor.id`, grant `update-query` and `delete-query`. -- For `source='user' AND owner_id = actor.id`, grant `view-query`. If the query is private, restriction SQL ensures no other actor sees it through a broader grant. - -## Executing queries - -Default execution rule for read-only queries: - -- If `is_trusted=0`, the actor needs `execute-sql` on the database. -- If `is_trusted=1`, the actor can execute the query without `execute-sql`, provided `view-query` allows access. - -Default execution rule for user-created writable queries: - -- `is_trusted` must be `0`. -- The actor must have `view-query`. -- The actor must currently have every write permission required by fresh `Database.analyze_sql()` results for the query SQL. - -Implementation: - -- Keep `view-query` in the broad `DEFAULT_ALLOW_ACTIONS` set, so saved queries remain visible by default in all-public Datasette. -- Emit default `view-query` allows for the owning actor. -- Use `restriction_sql` to limit private rows to their owner even when broader `view-query` permissions exist. -- Have `QueryView` perform the fresh `execute-sql` or table-permission check before execution unless the row has `is_trusted=1`. - -For read-only queries this keeps `QueryView` explicit: it checks `view-query` for the query resource, then checks `execute-sql` unless the row is trusted. User-created writable queries need one additional runtime permission check because their required table permissions are derived from fresh SQL analysis. - -Explicit deny rules should still be able to block a query, and `--default-deny` still blocks trusted queries unless something grants `view-query`. - -## Writable queries - -Writable user-created queries should be in scope, guarded by `Database.analyze_sql()`. - -The secure rule: a user can create, update, or execute a writable user-created query only if they currently have the corresponding write permissions for every table the SQL can affect. - -`Database.analyze_sql(sql, params=None)` runs the SQL through SQLite's authorizer on an isolated connection and returns a `SQLAnalysis` object containing `SQLTableAccess` rows: - -- `operation`: `read`, `insert`, `update`, or `delete` -- `database`: Datasette database name for `main`, or SQLite schema name where no Datasette mapping exists -- `table`: affected table or view -- `columns`: read/updated columns where SQLite reports them -- `source`: trigger/view/CTE source when SQLite reports one - -Validation flow for user-created queries: - -1. Derive named parameters from the SQL and pass harmless placeholder values into `db.analyze_sql()` so SQLite can prepare statements with bindings. -2. If analysis raises a SQLite error, reject the query. -3. If every table access is `read`, treat the query as read-only and require `execute-sql` plus `insert-query`/`update-query` as described above. -4. If any table access is `insert`, `update`, or `delete`, treat the query as writable and force `is_trusted=0`. -5. Reject writable user-created queries that access a database other than the database they are being saved against, until `analyze_sql()` can reliably map attached SQLite schemas back to Datasette database names. -6. For every write access returned by analysis, require the corresponding permission on `TableResource(access.database, access.table)`: - - `insert` -> `insert-row` - - `update` -> `update-row` - - `delete` -> `delete-row` -7. Include write accesses reported from triggers and views, since those are real side effects. -8. Re-run the same analysis and permission checks when SQL changes through `update_query()` or `POST .../-/update`. -9. Re-run analysis before executing user-created writable queries, so schema or trigger changes cannot leave a previously saved query with stale permission assumptions. - -The user-facing API should not trust a submitted `is_write` value. It should derive `is_write` from analysis. - -Trusted configuration and plugin code can still call `datasette.add_query(..., is_write=True, ...)`. Those are treated as deployment/admin-authored queries. They keep the existing execution model: they require `view-query`, and the default `view-query` hook should preserve current default-open behavior for trusted writable queries while still respecting `--default-deny`. - -Fail closed cases for user-created writable queries: - -- Analysis fails. -- Analysis reports any write operation that cannot be mapped to a Datasette table resource. -- Analysis reports writes outside the target database. -- The actor lacks any required table write permission. -- `is_trusted=1` is requested through the user-facing API. - -This gives us writable user-created queries without letting `execute-sql` alone become a path to create arbitrary write endpoints. - -## HTTP API sketch - -JSON endpoints should follow Datasette's existing write API style: use `POST` plus action paths such as `/-/insert`, `/-/update`, and `/-/delete`, not HTTP `PATCH` or `DELETE`. - -Endpoints: - -- `GET /-/queries` and `GET /{database}/-/queries` show searchable HTML query browsers. `GET /-/queries.json` lists query definitions across every database the actor can view; `GET /{database}/-/queries.json` scopes that list to one database. Both JSON endpoints use cursor pagination with `_next` and `_size`. -- `POST /{database}/-/queries/insert` creates a query. -- `GET /{database}/{query}/-/definition` returns one query definition without executing it. -- `POST /{database}/{query}/-/update` updates one query. -- `POST /{database}/{query}/-/delete` deletes one query. - -Create request: - -```json -{ - "query": { - "name": "top_customers", - "sql": "select * from customers order by revenue desc limit 20", - "title": "Top customers", - "description": "Highest revenue customers", - "is_private": true, - "parameters": ["region"] - } -} -``` - -Successful create returns `201` and the created query definition: - -```json -{ - "ok": true, - "query": { - "database": "fixtures", - "name": "top_customers", - "sql": "select * from customers order by revenue desc limit 20", - "title": "Top customers", - "description": "Highest revenue customers", - "is_private": true, - "is_trusted": false, - "parameters": ["region"] - } -} -``` - -Update request, imitating `RowUpdateView`: - -```json -{ - "update": { - "title": "Top customers by revenue", - "is_private": false - }, - "return": true -} -``` - -Successful update returns `{"ok": true}` by default. With `"return": true`, return the updated query definition: - -```json -{ - "ok": true, - "query": { - "database": "fixtures", - "name": "top_customers", - "sql": "select * from customers order by revenue desc limit 20", - "title": "Top customers by revenue", - "is_private": false, - "is_trusted": false - } -} -``` - -Delete request: - -```http -POST /{database}/{query}/-/delete -Content-Type: application/json -``` - -Successful delete returns: - -```json -{ - "ok": true -} -``` - -Validation: - -- Update bodies must be dictionaries containing an `update` dictionary, with optional `return`; invalid keys return `{"ok": false, "errors": [...]}`. -- Validate route-safe query names. -- Reject names that collide with a table or view in the same database, since table routes currently win over query routes. -- Analyze user-created SQL with `Database.analyze_sql()`. -- Use `validate_sql_select(sql)` as the read-only fast path when analysis shows only reads, but do not require it for writable queries that pass analysis and permission checks. -- Reject magic parameters such as `:_actor_id`, `:_cookie_*`, and `:_header_*` for user-created queries. -- Reject client-supplied `is_write`; derive it from analysis. -- Reject writable-only success/error fields for read-only queries. - -## Python API sketch - -Add methods on `Datasette`: - -```python -await datasette.add_query( - database, - name, - sql, - title=None, - description=None, - description_html=None, - hide_sql=False, - fragment=None, - parameters=None, - is_write=False, - is_private=False, - is_trusted=False, - source="plugin", - owner_id=None, - on_success_message=None, - on_success_message_sql=None, - on_success_redirect=None, - on_error_message=None, - on_error_redirect=None, - replace=True, -) - -await datasette.update_query( - database, - name, - *, - sql=UNCHANGED, - title=UNCHANGED, - description=UNCHANGED, - description_html=UNCHANGED, - hide_sql=UNCHANGED, - fragment=UNCHANGED, - parameters=UNCHANGED, - is_write=UNCHANGED, - is_private=UNCHANGED, - is_trusted=UNCHANGED, - source=UNCHANGED, - owner_id=UNCHANGED, - on_success_message=UNCHANGED, - on_success_message_sql=UNCHANGED, - on_success_redirect=UNCHANGED, - on_error_message=UNCHANGED, - on_error_redirect=UNCHANGED, -) - -await datasette.remove_query(database, name, source=None) - -await datasette.get_query(database, name) -await datasette.list_queries( - database, - actor=None, - limit=50, - cursor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, -) -``` - -`list_queries()` should return a bounded page shaped like `{"queries": [...], "next": "...", "has_more": true, "limit": 50}`. The `next` value is an opaque cursor token, not an offset. Passing `database=None` lists visible queries across all live databases, still filtered through `view-query` permission SQL. - -`update_query()` should use an internal sentinel default such as `UNCHANGED = object()` so callers can distinguish "leave this column alone" from "set this column to `NULL`": - -```python -await datasette.update_query( - "fixtures", - "top_customers", - on_success_redirect=None, -) -``` - -For column-backed fields, `None` should write SQL `NULL`. For option fields, `None` should remove that key from the JSON object so `get_query()` returns `None`; omitting the field should leave the existing option unchanged. - -Implementation detail: build the `UPDATE` statement dynamically from fields whose value is not `UNCHANGED`, validate non-nullable fields before writing, and update `updated_at` whenever at least one field changes. - -The read methods should reconstruct the existing dictionary shape used by query execution and templates, with `name`, `sql`, display fields, write fields, `params`, `is_private`, `is_trusted`, `owner_id`, and `source`. `parameters` should be returned as the decoded JSON array and exposed as `params` where existing query execution code expects that key. Option values should be unpacked from the `options` JSON object and returned as the same top-level keys accepted by `add_query()` and `update_query()`. - -## Query page save UI - -On `/{database}/-/query`, if the actor has both `execute-sql` and `insert-query`, show a save control for valid read-only SQL. That page already executes read-only arbitrary SQL, so the first UI can stay read-only even though the JSON API can accept writable SQL after `Database.analyze_sql()` validation. - -The save form should call `POST /{database}/-/queries/insert` and default to `is_private=true`. - -On `/{database}`, show a preview of the first 5 visible queries using `list_queries(..., limit=5)`. If the page has `has_more`, show a link to `/{database}/-/queries` rather than rendering hundreds or thousands of query links inline. The full `/{database}/-/queries` page provides search, filters, and cursor pagination. The global `/-/queries` page reuses the same interface and shows the database for each query. - -## Dedicated create query UI - -Add `/{database}/-/queries/-/create` for the fuller query authoring flow, including writable queries. - -This page should require `execute-sql` and `insert-query` to access. It should provide a SQL editor and a mode control: - -- Read-only -- Writable - -Read-only mode can share the same fields as the arbitrary SQL save flow: name, title, description, parameters, and privacy status. - -Writable mode should always run `Database.analyze_sql()` and show an analysis panel before saving: - -- detected operation -- database and table -- required permission -- whether the actor has that permission -- source, when the operation comes from a trigger or view - -The Save button should be disabled until analysis succeeds and every required table write permission is allowed. - -The existing edit-SQL flow from query pages can continue to point back to arbitrary SQL. A later enhancement can add "update this query" when the actor owns it or has `update-query`. - -## Test plan - -- Internal schema creates `queries`. -- Query parameters are stored in the `queries.parameters` text column as a JSON array of names. -- Config `queries:` blocks import into internal tables. -- Legacy string query definitions normalize to SQL rows. -- The old `canned_queries()` hook is no longer called by core. -- `QueryResource.resources_sql()` returns rows from `queries`. -- Database page and `/-/jump` list queries from the internal DB. -- `view-query` remains globally default-allowed, with `restriction_sql` narrowing private queries to their owner. -- Private query is only visible to its owner, even when a broader `view-query` rule applies. -- Non-trusted read-only query requires `execute-sql` to execute. -- Trusted read-only query can be executed without `execute-sql` after `view-query` passes. -- Config queries default to trusted and can opt out with `is_trusted: false`. -- User API rejects client-supplied `is_trusted`. -- User-created query requires both `execute-sql` and `insert-query`. -- User-created writable query creation uses `Database.analyze_sql()` and requires matching `insert-row`, `update-row`, and/or `delete-row` permissions for every reported write access. -- `/{database}/-/queries/-/create` provides the writable-query authoring UI with an analysis panel and disabled save until all required write permissions pass. -- User-created writable query execution re-runs `Database.analyze_sql()` and re-checks table write permissions. -- User-created writable query cannot be trusted through the user API. -- Query update uses `POST /{database}/{query}/-/update` with an `{"update": {...}}` body. -- Query delete uses `POST /{database}/{query}/-/delete`. -- There are no `PATCH` or HTTP `DELETE` routes for query management. -- `datasette.update_query(..., field=None)` writes `NULL` for column-backed fields and removes JSON keys for option fields, while omitted fields are left unchanged. -- Owner gets default `update-query` and `delete-query` for their own user-created rows. -- Admin can manage other users' queries with `update-query` and `delete-query`. -- User API rejects magic parameters. -- User API rejects writable queries if analysis fails, reports writes outside the target database, or reports writes the actor is not allowed to perform. -- Trusted config/plugin writable queries still execute through `view-query`. -- Trusted config/plugin writable queries are not default-allowed under `--default-deny`. -- Persisted internal DB does not expose queries for detached databases. From 24887004cffd52fe801ecd73da78e13b246ddede Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:51:57 -0700 Subject: [PATCH 133/156] Rename insert-query to store-query Also queries/insert to queries/store Refs https://github.com/simonw/datasette/pull/2741#issuecomment-4549103663 --- datasette/app.py | 6 ++--- datasette/default_actions.py | 6 ++--- datasette/templates/query_create.html | 2 +- datasette/views/database.py | 22 +++++++-------- docs/authentication.rst | 7 ++--- docs/json_api.rst | 5 ++-- tests/test_queries.py | 39 +++++++++++++++------------ 7 files changed, 47 insertions(+), 40 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8936b099..42a2d27d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -54,9 +54,9 @@ from .views.database import ( QueryDeleteView, QueryDefinitionView, GlobalQueryListView, - QueryInsertView, QueryListView, QueryParametersView, + QueryStoreView, QueryUpdateView, ) from .views.index import IndexView @@ -2824,8 +2824,8 @@ class Datasette: r"/(?P[^\/\.]+)/-/queries/analyze$", ) add_route( - QueryInsertView.as_view(self), - r"/(?P[^\/\.]+)/-/queries/insert$", + QueryStoreView.as_view(self), + r"/(?P[^\/\.]+)/-/queries/store$", ) add_route( ExecuteWriteAnalyzeView.as_view(self), diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 6a1f77b8..0f4c25fa 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -62,9 +62,9 @@ def register_actions(): resource_class=DatabaseResource, ), Action( - name="insert-query", - abbr="iq", - description="Create saved queries", + name="store-query", + abbr="sq", + description="Create stored queries", resource_class=DatabaseResource, also_requires="execute-sql", ), diff --git a/datasette/templates/query_create.html b/datasette/templates/query_create.html index cb14ada4..f5dadbff 100644 --- a/datasette/templates/query_create.html +++ b/datasette/templates/query_create.html @@ -156,7 +156,7 @@ form.sql .query-create-sql textarea#sql-editor {

      Create query

      -
      +

      {{ urls.database(database) }}/

      diff --git a/datasette/views/database.py b/datasette/views/database.py index d40d69d1..900b94ba 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -1419,7 +1419,7 @@ class QueryCreateView(BaseView): actor=request.actor, ) await self.ds.ensure_permission( - action="insert-query", + action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ) @@ -1440,11 +1440,11 @@ class QueryCreateAnalyzeView(BaseView): ): return _block_framing(_error(["Permission denied: need execute-sql"], 403)) if not await self.ds.allowed( - action="insert-query", + action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return _block_framing(_error(["Permission denied: need insert-query"], 403)) + return _block_framing(_error(["Permission denied: need store-query"], 403)) invalid_keys = set(request.args) - {"sql"} if invalid_keys: @@ -1462,8 +1462,8 @@ class QueryCreateAnalyzeView(BaseView): ) -class QueryInsertView(QueryCreateView): - name = "query-insert" +class QueryStoreView(QueryCreateView): + name = "query-store" async def _error_response(self, request, db, query_data, message, status): message = _query_create_form_error_message(message) @@ -1488,11 +1488,11 @@ class QueryInsertView(QueryCreateView): ): return _error(["Permission denied: need execute-sql"], 403) if not await self.ds.allowed( - action="insert-query", + action="store-query", resource=DatabaseResource(db.name), actor=request.actor, ): - return _error(["Permission denied: need insert-query"], 403) + return _error(["Permission denied: need store-query"], 403) is_json = False query_data = {} @@ -1961,8 +1961,8 @@ class QueryView(View): resource=DatabaseResource(database=database), actor=request.actor, ) - allow_insert_query = await datasette.allowed( - action="insert-query", + allow_store_query = await datasette.allowed( + action="store-query", resource=DatabaseResource(database=database), actor=request.actor, ) @@ -2020,13 +2020,13 @@ class QueryView(View): if ( not canned_query and allow_execute_sql - and allow_insert_query + and allow_store_query and is_validated_sql and ":_" not in sql ): save_query_url = ( datasette.urls.database(database) - + "/-/queries/insert?" + + "/-/queries/store?" + urlencode({"sql": sql}) ) diff --git a/docs/authentication.rst b/docs/authentication.rst index 453aaa19..184fec5e 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1293,11 +1293,12 @@ Actor is allowed to view a saved query page, e.g. https://latest.datasette.io/fi ``query`` is the name of the query (string) .. _actions_insert_query: +.. _actions_store_query: -insert-query ------------- +store-query +----------- -Actor is allowed to create saved queries in a database. +Actor is allowed to create stored queries in a database. ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) diff --git a/docs/json_api.rst b/docs/json_api.rst index dd54c459..1a6c7021 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -518,14 +518,15 @@ Listing saved queries Creating saved queries in the UI ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``GET //-/queries/-/create`` provides a form for creating saved queries. +``GET //-/queries/store`` provides a form for creating stored queries. +.. _QueryStoreView: .. _QueryInsertView: Creating saved queries ~~~~~~~~~~~~~~~~~~~~~~ -``POST //-/queries/insert`` creates a saved query. This requires ``execute-sql`` and ``insert-query`` for the database. +``POST //-/queries/store`` creates a stored query. This requires ``execute-sql`` and ``store-query`` for the database. .. _QueryParametersView: .. _ExecuteWriteView: diff --git a/tests/test_queries.py b/tests/test_queries.py index 26a0748c..5d4da9bb 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -470,7 +470,7 @@ async def test_query_actions_are_registered(): await ds.invoke_startup() assert ds.get_action("execute-write-sql").resource_class is DatabaseResource - assert ds.get_action("insert-query").resource_class is DatabaseResource + assert ds.get_action("store-query").resource_class is DatabaseResource assert ds.get_action("update-query").resource_class is QueryResource assert ds.get_action("delete-query").resource_class is QueryResource @@ -537,15 +537,15 @@ async def test_analyze_write_query_rejects_writes_to_attached_databases(): @pytest.mark.asyncio -async def test_query_insert_api_creates_read_only_query(): +async def test_query_store_api_creates_read_only_query(): ds = Datasette(memory=True, default_deny=True) ds.root_enabled = True - db = ds.add_memory_database("query_insert_api", name="data") + db = ds.add_memory_database("query_store_api", name="data") await db.execute_write("create table dogs (id integer primary key, name text)") await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, json={ "query": { @@ -860,7 +860,7 @@ async def test_global_query_list_api_and_html(): @pytest.mark.asyncio -async def test_query_insert_api_rejects_is_trusted(): +async def test_query_store_api_rejects_is_trusted(): ds = Datasette( memory=True, default_deny=True, @@ -870,7 +870,7 @@ async def test_query_insert_api_rejects_is_trusted(): "permissions": { "view-database": {"id": "writer"}, "execute-sql": {"id": "writer"}, - "insert-query": {"id": "writer"}, + "store-query": {"id": "writer"}, } } } @@ -880,7 +880,7 @@ async def test_query_insert_api_rejects_is_trusted(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "writer"}, json={"query": {"name": "trusted", "sql": "select 1", "is_trusted": True}}, ) @@ -890,7 +890,7 @@ async def test_query_insert_api_rejects_is_trusted(): @pytest.mark.asyncio -async def test_query_insert_api_creates_writable_query(): +async def test_query_store_api_creates_writable_query(): ds = Datasette(memory=True, default_deny=True) ds.root_enabled = True db = ds.add_memory_database("query_write_api", name="data") @@ -898,7 +898,7 @@ async def test_query_insert_api_creates_writable_query(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, json={ "query": { @@ -962,14 +962,14 @@ async def test_query_update_and_delete_api(): @pytest.mark.asyncio -async def test_query_insert_api_rejects_magic_parameters(): +async def test_query_store_api_rejects_magic_parameters(): ds = Datasette(memory=True, default_deny=True) ds.root_enabled = True ds.add_memory_database("query_magic_api", name="data") await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, json={"query": {"name": "magic", "sql": "select :_actor_id"}}, ) @@ -987,15 +987,19 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): await ds.invoke_startup() create_response = await ds.client.get( - "/data/-/queries/insert?sql=select+*+from+dogs", + "/data/-/queries/store?sql=select+*+from+dogs", actor={"id": "root"}, ) write_create_response = await ds.client.get( - "/data/-/queries/insert?sql=insert+into+dogs+(name)+values+('Cleo')", + "/data/-/queries/store?sql=insert+into+dogs+(name)+values+('Cleo')", actor={"id": "root"}, ) blank_create_response = await ds.client.get( - "/data/-/queries/insert", + "/data/-/queries/store", + actor={"id": "root"}, + ) + old_insert_response = await ds.client.get( + "/data/-/queries/insert?sql=select+*+from+dogs", actor={"id": "root"}, ) old_create_response = await ds.client.get( @@ -1075,7 +1079,8 @@ async def test_create_query_ui_and_arbitrary_sql_save_link(): ) assert query_response.status_code == 200 assert "Save this query" in query_response.text - assert "/data/-/queries/insert?sql=select+%2A+from+dogs" in query_response.text + assert "/data/-/queries/store?sql=select+%2A+from+dogs" in query_response.text + assert old_insert_response.status_code == 404 assert old_create_response.status_code == 404 @@ -1153,7 +1158,7 @@ async def test_create_query_form_error_redisplays_form_with_values(): await ds.invoke_startup() response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, data={ "name": "dogs", @@ -1176,7 +1181,7 @@ async def test_create_query_form_error_redisplays_form_with_values(): assert 'name="is_private" value="1" checked' in response.text public_response = await ds.client.post( - "/data/-/queries/insert", + "/data/-/queries/store", actor={"id": "root"}, data={ "name": "dogs", From 0cadd071871ef0b33e4ce3a23e316a104b3137c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:53:31 -0700 Subject: [PATCH 134/156] No need to document QueryCreateAnalyzeView --- tests/test_docs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index 396ba1a2..0d0ef1e1 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -66,7 +66,14 @@ def documented_views(): if first_word.endswith("View"): view_labels.add(first_word) # We deliberately don't document these: - view_labels.update(("PatternPortfolioView", "AuthTokenView", "ApiExplorerView")) + view_labels.update( + ( + "PatternPortfolioView", + "AuthTokenView", + "ApiExplorerView", + "QueryCreateAnalyzeView", + ) + ) return view_labels From 4bf1c4b065fef64676abf5eabd04ff35e07188c5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 14:54:35 -0700 Subject: [PATCH 135/156] Rename canned queries to queries/stored queries in docs --- datasette/default_actions.py | 4 +- datasette/hookspecs.py | 4 +- datasette/resources.py | 2 +- datasette/views/database.py | 24 ++++----- datasette/views/table.py | 4 +- docs/authentication.rst | 16 +++--- docs/configuration.rst | 10 ++-- docs/custom_templates.rst | 8 +-- docs/internals.rst | 12 ++--- docs/introspection.rst | 2 +- docs/json_api.rst | 32 ++++++------ docs/pages.rst | 4 +- docs/plugin_hooks.rst | 16 +++--- docs/spatialite.rst | 2 +- docs/sql_queries.rst | 95 ++++++++++++++++++++++++++---------- tests/test_html.py | 6 +-- tests/test_permissions.py | 4 +- 17 files changed, 144 insertions(+), 101 deletions(-) diff --git a/datasette/default_actions.py b/datasette/default_actions.py index 0f4c25fa..2f78570b 100644 --- a/datasette/default_actions.py +++ b/datasette/default_actions.py @@ -121,13 +121,13 @@ def register_actions(): Action( name="update-query", abbr="uq", - description="Update saved queries", + description="Update stored queries", resource_class=QueryResource, ), Action( name="delete-query", abbr="dq", - description="Delete saved queries", + description="Delete stored queries", resource_class=QueryResource, ), ) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index a4067eaa..22da02a4 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -174,7 +174,7 @@ def view_actions(datasette, actor, database, view, request): @hookspec def query_actions(datasette, actor, database, query_name, request, sql, params): - """Links for the query and canned query actions menu""" + """Links for the query and stored query actions menu""" @hookspec @@ -229,7 +229,7 @@ def top_query(datasette, request, database, sql): @hookspec def top_canned_query(datasette, request, database, query_name): - """HTML to include at the top of the canned query page""" + """HTML to include at the top of the stored query page""" @hookspec diff --git a/datasette/resources.py b/datasette/resources.py index 91a46d36..ee2e6d98 100644 --- a/datasette/resources.py +++ b/datasette/resources.py @@ -41,7 +41,7 @@ class TableResource(Resource): class QueryResource(Resource): - """A saved query in a database.""" + """A stored query in a database.""" name = "query" parent_class = DatabaseResource diff --git a/datasette/views/database.py b/datasette/views/database.py index 900b94ba..f30d3815 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -222,11 +222,11 @@ class DatabaseContext(Context): tables: list = field(metadata={"help": "List of table objects in the database"}) hidden_count: int = field(metadata={"help": "Count of hidden tables"}) views: list = field(metadata={"help": "List of view objects in the database"}) - queries: list = field(metadata={"help": "List of canned query objects"}) + queries: list = field(metadata={"help": "List of stored query objects"}) queries_more: bool = field( - metadata={"help": "Boolean indicating if more saved queries are available"} + metadata={"help": "Boolean indicating if more stored queries are available"} ) - queries_count: int = field(metadata={"help": "Count of visible saved queries"}) + queries_count: int = field(metadata={"help": "Count of visible stored queries"}) allow_execute_sql: bool = field( metadata={"help": "Boolean indicating if custom SQL can be executed"} ) @@ -272,7 +272,7 @@ class QueryContext(Context): metadata={"help": "The SQL query object containing the `sql` string"} ) canned_query: str = field( - metadata={"help": "The name of the canned query if this is a canned query"} + metadata={"help": "The name of the stored query if this is a stored query"} ) private: bool = field( metadata={"help": "Boolean indicating if this is a private database"} @@ -282,11 +282,11 @@ class QueryContext(Context): # ) canned_query_write: bool = field( metadata={ - "help": "Boolean indicating if this is a canned query that allows writes" + "help": "Boolean indicating if this is a stored query that allows writes" } ) metadata: dict = field( - metadata={"help": "Metadata about the database or the canned query"} + metadata={"help": "Metadata about the database or the stored query"} ) db_is_immutable: bool = field( metadata={"help": "Boolean indicating if this database is immutable"} @@ -315,7 +315,7 @@ class QueryContext(Context): metadata={"help": "Dictionary of parameter names/values"} ) edit_sql_url: str = field( - metadata={"help": "URL to edit the SQL for a canned query"} + metadata={"help": "URL to edit the SQL for a stored query"} ) display_rows: list = field(metadata={"help": "List of result rows to display"}) columns: list = field(metadata={"help": "List of column names"}) @@ -1623,7 +1623,7 @@ class QueryView(View): db = await datasette.resolve_database(request) - # We must be a canned query + # We must be a stored query table_found = False try: await datasette.resolve_table(request) @@ -1742,14 +1742,14 @@ class QueryView(View): # Create lookup dict for quick access allowed_dict = {r.child: r for r in allowed_tables_page.resources} - # Are we a canned query? + # Are we a stored query? canned_query = None canned_query_write = False if "table" in request.url_vars: try: await datasette.resolve_table(request) except TableNotFound as table_not_found: - # Was this actually a canned query? + # Was this actually a stored query? canned_query = await datasette.get_canned_query( table_not_found.database_name, table_not_found.table, request.actor ) @@ -1759,7 +1759,7 @@ class QueryView(View): private = False if canned_query: - # Respect canned query permissions + # Respect stored query permissions visible, private = await datasette.check_visibility( request.actor, action="view-query", @@ -1823,7 +1823,7 @@ class QueryView(View): # For regular queries we only allow SELECT, plus other rules validate_sql_select(sql) else: - # Canned queries can run magic parameters + # Stored queries can run magic parameters params_for_query = MagicParameters(sql, params, request, datasette) await params_for_query.execute_params() results = await datasette.execute( diff --git a/datasette/views/table.py b/datasette/views/table.py index 7027bb10..7b1a5a82 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -963,11 +963,11 @@ async def table_view_traced(datasette, request): try: resolved = await datasette.resolve_table(request) except TableNotFound as not_found: - # Was this actually a canned query? + # Was this actually a stored query? canned_query = await datasette.get_canned_query( not_found.database_name, not_found.table, request.actor ) - # If this is a canned query, not a table, then dispatch to QueryView instead + # If this is a stored query, not a table, then dispatch to QueryView instead if canned_query: return await QueryView()(request, datasette) else: diff --git a/docs/authentication.rst b/docs/authentication.rst index 184fec5e..22db41d8 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -468,7 +468,7 @@ You can control the following: * Access to the entire Datasette instance * Access to specific databases * Access to specific tables and views -* Access to specific :ref:`canned_queries` +* Access to specific :ref:`queries ` If a user has permission to view a table they will be able to view that table, independent of if they have permission to view the database or instance that the table exists within. @@ -641,12 +641,12 @@ This works for SQL views as well - you can list their names in the ``"tables"`` .. _authentication_permissions_query: -Access to specific canned queries ---------------------------------- +Access to specific queries +-------------------------- -:ref:`canned_queries` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. +:ref:`Queries ` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. -To limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user`: +To limit access to the ``add_name`` query in your ``dogs.db`` database to just the :ref:`root user`: .. [[[cog config_example(cog, """ @@ -1285,7 +1285,7 @@ Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.i view-query ---------- -Actor is allowed to view a saved query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size. Executing an untrusted saved query also requires ``execute-sql`` or the relevant write permissions; trusted saved queries can execute with ``view-query`` alone. +Actor is allowed to view a stored query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size. Executing an untrusted stored query also requires ``execute-sql`` or the relevant write permissions; :ref:`trusted stored queries ` can execute with ``view-query`` alone. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) @@ -1308,7 +1308,7 @@ Actor is allowed to create stored queries in a database. update-query ------------ -Actor is allowed to update a saved query. +Actor is allowed to update a stored query. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) @@ -1320,7 +1320,7 @@ Actor is allowed to update a saved query. delete-query ------------ -Actor is allowed to delete a saved query. +Actor is allowed to delete a stored query. ``resource`` - ``datasette.resources.QueryResource(database, query)`` ``database`` is the name of the database (string) diff --git a/docs/configuration.rst b/docs/configuration.rst index 8c8c8a67..cf9590b8 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -87,6 +87,7 @@ This is equivalent to a ``datasette.yaml`` file containing the following: } .. [[[end]]] + .. _configuration_reference: ``datasette.yaml`` reference @@ -435,10 +436,10 @@ Here is a simple example: .. _configuration_reference_canned_queries: -Canned queries configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Queries configuration +~~~~~~~~~~~~~~~~~~~~~ -:ref:`Canned queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: +:ref:`Queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: .. [[[cog from metadata_doc import config_example, config_example @@ -483,7 +484,7 @@ Canned queries configuration } .. [[[end]]] -See the :ref:`canned queries documentation ` for more, including how to configure :ref:`writable canned queries `. +See the :ref:`queries documentation ` for more, including how to configure :ref:`writable queries `. .. _configuration_reference_css_js: @@ -1211,4 +1212,3 @@ For column types that accept additional configuration, use an object with ``type } } .. [[[end]]] - diff --git a/docs/custom_templates.rst b/docs/custom_templates.rst index 8cc40f0f..c324fb79 100644 --- a/docs/custom_templates.rst +++ b/docs/custom_templates.rst @@ -29,7 +29,7 @@ The custom SQL template (``/dbname?sql=...``) gets this: -A canned query template (``/dbname/queryname``) gets this: +A stored query template (``/dbname/queryname``) gets this: .. code-block:: html @@ -193,8 +193,8 @@ The lookup rules Datasette uses are as follows:: query-mydatabase.html query.html - Canned query page (/mydatabase/canned-query): - query-mydatabase-canned-query.html + Stored query page (/mydatabase/query-name): + query-mydatabase-query-name.html query-mydatabase.html query.html @@ -230,7 +230,7 @@ will look something like this:: -This example is from the canned query page for a query called "tz" in the +This example is from the stored query page for a query called "tz" in the database called "mydb". The asterisk shows which template was selected - so in this case, Datasette found a template file called ``query-mydb-tz.html`` and used that - but if that template had not been found, it would have tried for diff --git a/docs/internals.rst b/docs/internals.rst index c76de487..084922f8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -725,7 +725,7 @@ The builder methods are: - ``allow_all(action)`` - allow an action across all databases and resources - ``allow_database(database, action)`` - allow an action on a specific database -- ``allow_resource(database, resource, action)`` - allow an action on a specific resource (table, SQL view or :ref:`canned query `) within a database +- ``allow_resource(database, resource, action)`` - allow an action on a specific resource (table, SQL view or :ref:`stored query `) within a database Each method returns the ``TokenRestrictions`` instance so calls can be chained. @@ -837,10 +837,10 @@ await .get_resource_metadata(self, database_name, resource_name) ``database_name`` - string The name of the database to query. ``resource_name`` - string - The name of the resource (table, view, or canned query) inside ``database_name`` to query. + The name of the resource (table, view, or stored query) inside ``database_name`` to query. Returns metadata keys and values for the specified "resource" as a dictionary. -A "resource" in this context can be a table, view, or canned query. +A "resource" in this context can be a table, view, or stored query. Internally queries the ``metadata_resources`` table inside the :ref:`internal database `. .. _datasette_get_column_metadata: @@ -851,7 +851,7 @@ await .get_column_metadata(self, database_name, resource_name, column_name) ``database_name`` - string The name of the database to query. ``resource_name`` - string - The name of the resource (table, view, or canned query) inside ``database_name`` to query. + The name of the resource (table, view, or stored query) inside ``database_name`` to query. ``column_name`` - string The name of the column inside ``resource_name`` to query. @@ -897,7 +897,7 @@ await .set_resource_metadata(self, database_name, resource_name, key, value) ``database_name`` - string The database the metadata entry belongs to. ``resource_name`` - string - The resource (table, view, or canned query) the metadata entry belongs to. + The resource (table, view, or stored query) the metadata entry belongs to. ``key`` - string The metadata entry key to insert (ex ``title``, ``description``, etc.) ``value`` - string @@ -915,7 +915,7 @@ await .set_column_metadata(self, database_name, resource_name, column_name, key, ``database_name`` - string The database the metadata entry belongs to. ``resource_name`` - string - The resource (table, view, or canned query) the metadata entry belongs to. + The resource (table, view, or stored query) the metadata entry belongs to. ``column-name`` - string The column the metadata entry belongs to. ``key`` - string diff --git a/docs/introspection.rst b/docs/introspection.rst index d2eb8efd..7702a4b5 100644 --- a/docs/introspection.rst +++ b/docs/introspection.rst @@ -149,7 +149,7 @@ Shows currently attached databases. `Databases example /-/queries.json`` returns saved query definitions for a specific database. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. +``GET /-/queries.json`` returns stored query definitions across every database that the actor can view. ``GET //-/queries.json`` returns stored query definitions for a specific database. Use ``?_size=50`` to set the page size and ``?_next=...`` with the cursor returned by the previous page to fetch the next page. .. _QueryCreateView: -Creating saved queries in the UI -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Creating stored queries in the UI +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``GET //-/queries/store`` provides a form for creating stored queries. .. _QueryStoreView: .. _QueryInsertView: -Creating saved queries -~~~~~~~~~~~~~~~~~~~~~~ +Creating stored queries +~~~~~~~~~~~~~~~~~~~~~~~ ``POST //-/queries/store`` creates a stored query. This requires ``execute-sql`` and ``store-query`` for the database. @@ -545,24 +545,24 @@ Executing write SQL .. _QueryDefinitionView: -Getting a saved query definition -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Getting a stored query definition +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``GET ///-/definition`` returns a saved query definition without executing it. +``GET ///-/definition`` returns a stored query definition without executing it. .. _QueryUpdateView: -Updating saved queries -~~~~~~~~~~~~~~~~~~~~~~ +Updating stored queries +~~~~~~~~~~~~~~~~~~~~~~~ -``POST ///-/update`` updates a saved query using a JSON body with an ``"update"`` object. +``POST ///-/update`` updates a stored query using a JSON body with an ``"update"`` object. .. _QueryDeleteView: -Deleting saved queries -~~~~~~~~~~~~~~~~~~~~~~ +Deleting stored queries +~~~~~~~~~~~~~~~~~~~~~~~ -``POST ///-/delete`` deletes a saved query. +``POST ///-/delete`` deletes a stored query. .. _TableInsertView: diff --git a/docs/pages.rst b/docs/pages.rst index 34c851a5..e57c15e6 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -28,7 +28,7 @@ The index page can also be accessed at ``/-/``, useful for if the default index Database ======== -Each database has a page listing the tables, views and canned queries available for that database. If the :ref:`actions_execute_sql` permission is enabled (it's on by default) there will also be an interface for executing arbitrary SQL select queries against the data. +Each database has a page listing the tables, views and stored queries available for that database. If the :ref:`actions_execute_sql` permission is enabled (it's on by default) there will also be an interface for executing arbitrary SQL select queries against the data. Examples: @@ -68,7 +68,7 @@ This means you can link directly to a query by constructing the following URL: ``/database-name/-/query?sql=SELECT+*+FROM+table_name`` -Each configured :ref:`canned query ` has its own page, at ``/database-name/query-name``. Viewing this page will execute the query and display the results. +Each configured :ref:`stored query ` has its own page, at ``/database-name/query-name``. Viewing this page will execute the query and display the results. In both cases adding a ``.json`` extension to the URL will return the results as JSON. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index b2676b3e..264b473e 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -609,7 +609,7 @@ When a request is received, the ``"render"`` callback function is called with ze The SQL query that was executed. ``query_name`` - string or None - If this was the execution of a :ref:`canned query `, the name of that query. + If this was the execution of a :ref:`stored query `, the name of that query. ``database`` - string The name of the database. @@ -1212,7 +1212,7 @@ Examples: `datasette-saved-queries `__ @@ -1635,7 +1635,7 @@ register_magic_parameters(datasette) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. -:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`canned queries `. This plugin hook allows additional magic parameters to be defined by plugins. +:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`configured queries `. This plugin hook allows additional magic parameters to be defined by plugins. Magic parameters all take this format: ``_prefix_rest_of_parameter``. The prefix indicates which magic parameter function should be called - the rest of the parameter is passed as an argument to that function. @@ -1828,7 +1828,7 @@ jump_items_sql(datasette, actor, request) This hook allows plugins to add extra results to Datasette's ``/`` jump menu, which is powered by the ``/-/jump`` JSON endpoint. -Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be searched alongside Datasette's own databases, tables, views and canned query results. The hook can also be an ``async def`` function, or return an awaitable that resolves to one of these values. +Return a ``datasette.jump.JumpSQL`` object, or a list of ``JumpSQL`` objects. Each ``JumpSQL`` object wraps a SQL query to be searched alongside Datasette's own databases, tables, views and stored query results. The hook can also be an ``async def`` function, or return an awaitable that resolves to one of these values. ``JumpSQL`` queries run against Datasette's internal database by default. To run a query against another database, pass its name as the optional ``database=`` argument. For example, ``JumpSQL(database="content", sql="...")`` runs against the ``content`` database. @@ -2004,7 +2004,7 @@ query_actions(datasette, actor, database, query_name, request, sql, params) The name of the database. ``query_name`` - string or None - The name of the canned query, or ``None`` if this is an arbitrary SQL query. + The name of the stored query, or ``None`` if this is an arbitrary SQL query. ``request`` - :ref:`internals_request` The current HTTP request. @@ -2015,7 +2015,7 @@ query_actions(datasette, actor, database, query_name, request, sql, params) ``params`` - dictionary The parameters passed to the SQL query, if any. -Populates a "Query actions" menu on the canned query and arbitrary SQL query pages. +Populates a "Query actions" menu on the stored query and arbitrary SQL query pages. This example adds a new query action linking to a page for explaining a query: @@ -2294,9 +2294,9 @@ top_canned_query(datasette, request, database, query_name) The name of the database. ``query_name`` - string - The name of the canned query. + The name of the stored query. -Returns HTML to be displayed at the top of the canned query page. +Returns HTML to be displayed at the top of the stored query page. .. _plugin_event_tracking: diff --git a/docs/spatialite.rst b/docs/spatialite.rst index c93c1e00..1999ab78 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -30,7 +30,7 @@ Warning The following steps are recommended: - Disable arbitrary SQL queries by untrusted users. See :ref:`authentication_permissions_execute_sql` for ways to do this. The easiest is to start Datasette with the ``datasette --setting default_allow_sql off`` option. - - Define :ref:`canned_queries` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. + - Define :ref:`queries ` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. The `Datasette SpatiaLite tutorial `__ includes detailed instructions for running SpatiaLite safely using these techniques diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index 7c3cd4ac..d60656e3 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -68,10 +68,10 @@ You can also use the `sqlite-utils `__ tool .. _canned_queries: -Canned queries --------------- +Queries +------- -As an alternative to adding views to your database, you can define canned queries inside your ``datasette.yaml`` file. Here's an example: +As an alternative to adding views to your database, you can define named queries inside your ``datasette.yaml`` file. Here's an example: .. [[[cog from metadata_doc import config_example, config_example @@ -120,24 +120,67 @@ Then run Datasette like this:: datasette sf-trees.db -m metadata.json -Each canned query will be listed on the database index page, and will also get its own URL at:: +Each configured query will be listed on the database index page, and will also get its own URL at:: - /database-name/canned-query-name + /database-name/query-name For the above example, that URL would be:: /sf-trees/just_species -You can optionally include ``"title"`` and ``"description"`` keys to show a title and description on the canned query page. As with regular table metadata you can alternatively specify ``"description_html"`` to have your description rendered as HTML (rather than having HTML special characters escaped). +You can optionally include ``"title"`` and ``"description"`` keys to show a title and description on the query page. As with regular table metadata you can alternatively specify ``"description_html"`` to have your description rendered as HTML (rather than having HTML special characters escaped). + +.. _stored_queries: +.. _saved_queries: + +Stored queries +~~~~~~~~~~~~~~ + +Datasette stores both configured queries and user-created queries in the ``queries`` table in the :ref:`internal database `. Configured queries come from the ``queries`` section of ``datasette.yaml``. User-created stored queries can be created from the SQL query page by actors with the :ref:`actions_store_query` and :ref:`actions_execute_sql` permissions. Writable stored queries also require the permissions needed for the writes they perform. + +Stored queries created by users default to private. Private stored queries can only be viewed, updated or deleted by the actor that created them. Broad ``view-query``, ``update-query`` or ``delete-query`` permission grants still do not allow other actors to access another actor's private stored queries. + +Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. + +.. _trusted_stored_queries: +.. _trusted_saved_queries: + +Trusted stored queries +++++++++++++++++++++++ + +A trusted stored query can execute with ``view-query`` permission alone. It skips the additional ``execute-sql`` and write permission checks that are applied to untrusted stored queries. + +Trusted stored queries should only be used for SQL that has been reviewed by someone trusted to configure the Datasette instance. For that reason, trusted stored queries can only be added using configuration. Users cannot create trusted stored queries through the web interface or the stored query JSON API. + +Queries defined in ``datasette.yaml`` are trusted by default: + +.. code-block:: yaml + + databases: + mydatabase: + queries: + report: + sql: select * from report + +You can opt out of this behavior for a configured query using ``is_trusted: false``: + +.. code-block:: yaml + + databases: + mydatabase: + queries: + report: + sql: select * from report + is_trusted: false .. _canned_queries_named_parameters: -Canned query parameters -~~~~~~~~~~~~~~~~~~~~~~~ +Query parameters +~~~~~~~~~~~~~~~~ -Canned queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the canned query page or by adding them to the URL. This means canned queries can be used to create custom JSON APIs based on a carefully designed SQL statement. +Configured queries support named parameters, so if you include those in the SQL you will then be able to enter them using the form fields on the query page or by adding them to the URL. This means configured queries can be used to create custom JSON APIs based on a carefully designed SQL statement. -Here's an example of a canned query with a named parameter: +Here's an example of a configured query with a named parameter: .. code-block:: sql @@ -147,7 +190,7 @@ Here's an example of a canned query with a named parameter: where neighborhood like '%' || :text || '%' order by neighborhood; -In the canned query configuration looks like this: +The query configuration looks like this: .. [[[cog @@ -204,7 +247,7 @@ In the canned query configuration looks like this: Note that we are using SQLite string concatenation here - the ``||`` operator - to add wildcard ``%`` characters to the string provided by the user. -You can try this canned query out here: +You can try this query out here: https://latest.datasette.io/fixtures/neighborhood_search?text=town In this example the ``:text`` named parameter is automatically extracted from the query using a regular expression. @@ -272,15 +315,15 @@ You can alternatively provide an explicit list of named parameters using the ``" .. _canned_queries_options: -Additional canned query options -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Additional query options +~~~~~~~~~~~~~~~~~~~~~~~~ -Additional options can be specified for canned queries in the YAML or JSON configuration. +Additional options can be specified for configured queries in the YAML or JSON configuration. hide_sql ++++++++ -Canned queries default to displaying their SQL query at the top of the page. If the query is extremely long you may want to hide it by default, with a "show" link that can be used to make it visible. +Configured queries default to displaying their SQL query at the top of the page. If the query is extremely long you may want to hide it by default, with a "show" link that can be used to make it visible. Add the ``"hide_sql": true`` option to hide the SQL query by default. @@ -289,7 +332,7 @@ fragment Some plugins, such as `datasette-vega `__, can be configured by including additional data in the fragment hash of the URL - the bit that comes after a ``#`` symbol. -You can set a default fragment hash that will be included in the link to the canned query from the database index page using the ``"fragment"`` key. +You can set a default fragment hash that will be included in the link to the query from the database index page using the ``"fragment"`` key. This example demonstrates both ``fragment`` and ``hide_sql``: @@ -348,12 +391,12 @@ This example demonstrates both ``fragment`` and ``hide_sql``: .. _canned_queries_writable: -Writable canned queries -~~~~~~~~~~~~~~~~~~~~~~~ +Writable queries +~~~~~~~~~~~~~~~~ -Canned queries by default are read-only. You can use the ``"write": true`` key to indicate that a canned query can write to the database. +Configured queries are read-only by default. You can use the ``"write": true`` key to indicate that a query can write to the database. -See :ref:`authentication_permissions_query` for details on how to add permission checks to canned queries, using the ``"allow"`` key. +See :ref:`authentication_permissions_query` for details on how to add permission checks to queries, using the ``"allow"`` key. .. [[[cog config_example(cog, { @@ -488,7 +531,7 @@ Magic parameters Named parameters that start with an underscore are special: they can be used to automatically add values created by Datasette that are not contained in the incoming form fields or query string. -These magic parameters are only supported for canned queries: to avoid security issues (such as queries that extract the user's private cookies) they are not available to SQL that is executed by the user as a custom SQL query. +These magic parameters are only supported for configured queries: to avoid security issues (such as queries that extract the user's private cookies) they are not available to SQL that is executed by the user as a custom SQL query. Available magic parameters are: @@ -580,12 +623,12 @@ Additional custom magic parameters can be added by plugins using the :ref:`plugi .. _canned_queries_json_api: -JSON API for writable canned queries -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +JSON API for writable queries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Writable canned queries can also be accessed using a JSON API. You can POST data to them using JSON, and you can request that their response is returned to you as JSON. +Writable queries can also be accessed using a JSON API. You can POST data to them using JSON, and you can request that their response is returned to you as JSON. -To submit JSON to a writable canned query, encode key/value parameters as a JSON document:: +To submit JSON to a writable query, encode key/value parameters as a JSON document:: POST /mydatabase/add_message diff --git a/tests/test_html.py b/tests/test_html.py index 9e460da1..8edb9f6e 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -154,7 +154,7 @@ async def test_database_page(ds_client): ("/fixtures/simple_view", "simple_view"), ] == sorted([(a["href"], a.text) for a in views_ul.find_all("a")]) - # And a list of canned queries + # And a list of stored queries queries_ul = soup.find("h2", string="Queries").find_next_sibling("ul") assert queries_ul is not None assert [ @@ -701,7 +701,7 @@ async def test_show_hide_sql_query(ds_client): @pytest.mark.asyncio async def test_canned_query_with_hide_has_no_hidden_sql(ds_client): - # For a canned query the show/hide should NOT have a hidden SQL field + # For a stored query the show/hide should NOT have a hidden SQL field # https://github.com/simonw/datasette/issues/1411 response = await ds_client.get("/fixtures/pragma_cache_size?_hide_sql=1") soup = Soup(response.content, "html.parser") @@ -1106,7 +1106,7 @@ async def test_trace_correctly_escaped(ds_client): "/fixtures/-/query?sql=select+*+from+facetable", "http://localhost/fixtures/-/query.json?sql=select+*+from+facetable", ), - # Canned query page + # Stored query page ( "/fixtures/neighborhood_search?text=town", "http://localhost/fixtures/neighborhood_search.json?text=town", diff --git a/tests/test_permissions.py b/tests/test_permissions.py index eb6cee9f..0e38c876 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -890,7 +890,7 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "t1"), expected_result=True, ), - # view-query on canned query, wrong actor + # view-query on stored query, wrong actor PermConfigTestCase( config={ "databases": { @@ -909,7 +909,7 @@ PermConfigTestCase = collections.namedtuple( resource=("perms_ds_one", "q1"), expected_result=False, ), - # view-query on canned query, right actor + # view-query on stored query, right actor PermConfigTestCase( config={ "databases": { From b1029acc68626c2fddf7b678adc3339be0fce6e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:05:41 -0700 Subject: [PATCH 136/156] top_canned_query is now top_stored_query, closes #2747 --- datasette/hookspecs.py | 2 +- datasette/templates/query.html | 2 +- datasette/views/database.py | 8 ++++---- docs/changelog.rst | 1 + docs/plugin_hooks.rst | 4 ++-- tests/test_plugins.py | 10 ++++++---- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 22da02a4..dcd502af 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -228,7 +228,7 @@ def top_query(datasette, request, database, sql): @hookspec -def top_canned_query(datasette, request, database, query_name): +def top_stored_query(datasette, request, database, query_name): """HTML to include at the top of the stored query page""" diff --git a/datasette/templates/query.html b/datasette/templates/query.html index 785b05af..3f03424a 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -33,7 +33,7 @@ {% set action_links, action_title = query_actions(), "Query actions" %} {% include "_action_menu.html" %} -{% if canned_query %}{{ top_canned_query() }}{% else %}{{ top_query() }}{% endif %} +{% if canned_query %}{{ top_stored_query() }}{% else %}{{ top_query() }}{% endif %} {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} diff --git a/datasette/views/database.py b/datasette/views/database.py index f30d3815..def3c530 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -339,8 +339,8 @@ class QueryContext(Context): top_query: callable = field( metadata={"help": "Callable to render the top_query slot"} ) - top_canned_query: callable = field( - metadata={"help": "Callable to render the top_canned_query slot"} + top_stored_query: callable = field( + metadata={"help": "Callable to render the top_stored_query slot"} ) query_actions: callable = field( metadata={ @@ -2095,8 +2095,8 @@ class QueryView(View): top_query=make_slot_function( "top_query", datasette, request, database=database, sql=sql ), - top_canned_query=make_slot_function( - "top_canned_query", + top_stored_query=make_slot_function( + "top_stored_query", datasette, request, database=database, diff --git a/docs/changelog.rst b/docs/changelog.rst index dfb2a736..300ac02f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) +- The ``top_canned_query()`` plugin hook has been renamed to :ref:`top_stored_query() `. (:issue:`2747`) .. _v1_0_a30: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 264b473e..4737ca03 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -2279,9 +2279,9 @@ top_query(datasette, request, database, sql) Returns HTML to be displayed at the top of the query results page. -.. _plugin_hook_top_canned_query: +.. _plugin_hook_top_stored_query: -top_canned_query(datasette, request, database, query_name) +top_stored_query(datasette, request, database, query_name) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``datasette`` - :ref:`internals_datasette` diff --git a/tests/test_plugins.py b/tests/test_plugins.py index f7adbd66..32276437 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1486,8 +1486,10 @@ class SlotPlugin: return "Xtop_query:{}:{}:{}".format(database, sql, request.args["z"]) @hookimpl - def top_canned_query(self, request, database, query_name): - return "Xtop_query:{}:{}:{}".format(database, query_name, request.args["z"]) + def top_stored_query(self, request, database, query_name): + return "Xtop_stored_query:{}:{}:{}".format( + database, query_name, request.args["z"] + ) @pytest.mark.asyncio @@ -1548,12 +1550,12 @@ async def test_hook_top_query(ds_client): @pytest.mark.asyncio -async def test_hook_top_canned_query(ds_client): +async def test_hook_top_stored_query(ds_client): try: pm.register(SlotPlugin(), name="SlotPlugin") response = await ds_client.get("/fixtures/magic_parameters?z=xyz") assert response.status_code == 200 - assert "Xtop_query:fixtures:magic_parameters:xyz" in response.text + assert "Xtop_stored_query:fixtures:magic_parameters:xyz" in response.text finally: pm.unregister(name="SlotPlugin") From 2f73869c09962e320e5f40f4691df70618cd052e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:09:48 -0700 Subject: [PATCH 137/156] Document that canned_queries() has been removed --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 300ac02f..674ff5b3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) - The ``top_canned_query()`` plugin hook has been renamed to :ref:`top_stored_query() `. (:issue:`2747`) +- The ``canned_queries()`` plugin hook has been removed. Plugins can use the new ``datasette.add_query()``, ``datasette.update_query()`` and ``datasette.remove_query()`` methods to managed stored queries instead. .. _v1_0_a30: From 56b14f37d547e03ba902516ac9ae13ef52765f77 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:16:18 -0700 Subject: [PATCH 138/156] The stored queries do not live in that DB --- docs/authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index 22db41d8..86df7f04 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -1298,7 +1298,7 @@ Actor is allowed to view a stored query page, e.g. https://latest.datasette.io/f store-query ----------- -Actor is allowed to create stored queries in a database. +Actor is allowed to create stored queries against a database. ``resource`` - ``datasette.resources.DatabaseResource(database)`` ``database`` is the name of the database (string) From 02a1468f1b3c8c14fb80037686b43de856e49c1f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:17:51 -0700 Subject: [PATCH 139/156] Renamed canned queries to queries / stored queries in docs And a few renames in code and YAML as well. --- .github/workflows/deploy-latest.yml | 33 +- datasette/app.py | 7 - datasette/facets.py | 2 +- datasette/static/app.css | 2 +- datasette/templates/query.html | 18 +- datasette/views/database.py | 92 +++--- datasette/views/table.py | 6 +- docs/authentication.rst | 10 +- docs/changelog.rst | 23 +- docs/configuration.rst | 6 +- docs/plugin_hooks.rst | 12 +- docs/spatialite.rst | 2 +- docs/sql_queries.rst | 12 +- docs/upgrade-1.0a20.md | 6 +- tests/test_canned_queries.py | 473 ---------------------------- tests/test_html.py | 12 +- tests/test_jump.py | 4 +- 17 files changed, 115 insertions(+), 605 deletions(-) delete mode 100644 tests/test_canned_queries.py diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 7d8dd37d..166d33d0 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -57,7 +57,7 @@ jobs: db.route = "alternative-route" ' > plugins/alternative_route.py cp fixtures.db fixtures2.db - - name: And the counters writable canned query demo + - name: And the counters writable stored query demo run: | cat > plugins/counters.py <This query cannot be executed because the database is immutable.

      {% endif %} -

      {{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}

      +

      {{ metadata.title or database }}{% if stored_query and not metadata.title %}: {{ stored_query }}{% endif %}{% if private %} 🔒{% endif %}

      {% set action_links, action_title = query_actions(), "Query actions" %} {% include "_action_menu.html" %} -{% if canned_query %}{{ top_stored_query() }}{% else %}{{ top_query() }}{% endif %} +{% if stored_query %}{{ top_stored_query() }}{% else %}{{ top_query() }}{% endif %} {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %} - +

      Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %} ({{ show_hide_text }}) {% endif %}

      @@ -52,7 +52,7 @@ 
      {% if query %}{{ query.sql }}{% endif %}
      {% endif %} {% else %} - {% if not canned_query %} + {% if not stored_query %} @@ -64,10 +64,10 @@ {% include "_sql_parameters.html" %}

      {% if not hide_sql %}{% endif %} - + {{ show_hide_hidden }} {% if save_query_url %}Save this query{% endif %} - {% if canned_query and edit_sql_url %}Edit SQL{% endif %} + {% if stored_query and edit_sql_url %}Edit SQL{% endif %}

      @@ -90,7 +90,7 @@
      Required permission
      {% else %} - {% if not canned_query_write and not error %} + {% if not stored_query_write and not error %}

      0 results

      {% endif %} {% endif %} diff --git a/datasette/views/database.py b/datasette/views/database.py index def3c530..c36476f6 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -100,12 +100,12 @@ class DatabaseView(View): limit=5, include_private=True, ) - canned_queries = queries_page["queries"] + stored_queries = queries_page["queries"] queries_more = queries_page["has_more"] queries_count = ( await datasette.count_queries(database, actor=request.actor) if queries_more - else len(canned_queries) + else len(stored_queries) ) async def database_actions(): @@ -137,7 +137,7 @@ class DatabaseView(View): "tables": tables, "hidden_count": len([t for t in tables if t["hidden"]]), "views": sql_views, - "queries": canned_queries, + "queries": stored_queries, "queries_more": queries_more, "queries_count": queries_count, "allow_execute_sql": allow_execute_sql, @@ -172,7 +172,7 @@ class DatabaseView(View): tables=tables, hidden_count=len([t for t in tables if t["hidden"]]), views=sql_views, - queries=canned_queries, + queries=stored_queries, queries_more=queries_more, queries_count=queries_count, allow_execute_sql=allow_execute_sql, @@ -271,7 +271,7 @@ class QueryContext(Context): query: dict = field( metadata={"help": "The SQL query object containing the `sql` string"} ) - canned_query: str = field( + stored_query: str = field( metadata={"help": "The name of the stored query if this is a stored query"} ) private: bool = field( @@ -280,7 +280,7 @@ class QueryContext(Context): # urls: dict = field( # metadata={"help": "Object containing URL helpers like `database()`"} # ) - canned_query_write: bool = field( + stored_query_write: bool = field( metadata={ "help": "Boolean indicating if this is a stored query that allows writes" } @@ -1629,10 +1629,10 @@ class QueryView(View): await datasette.resolve_table(request) table_found = True except TableNotFound as table_not_found: - canned_query = await datasette.get_canned_query( - table_not_found.database_name, table_not_found.table, request.actor + stored_query = await datasette.get_query( + table_not_found.database_name, table_not_found.table ) - if canned_query is None: + if stored_query is None: raise if table_found: # That should not have happened @@ -1640,13 +1640,13 @@ class QueryView(View): if not await datasette.allowed( action="view-query", - resource=QueryResource(database=db.name, query=canned_query["name"]), + resource=QueryResource(database=db.name, query=stored_query["name"]), actor=request.actor, ): raise Forbidden("You do not have permission to view this query") await _ensure_stored_query_execution_permissions( - datasette, db, canned_query, request.actor + datasette, db, stored_query, request.actor ) # If database is immutable, return an error @@ -1674,19 +1674,19 @@ class QueryView(View): or params.get("_json") ) params_for_query = MagicParameters( - canned_query["sql"], params, request, datasette + stored_query["sql"], params, request, datasette ) await params_for_query.execute_params() ok = None redirect_url = None try: cursor = await db.execute_write( - canned_query["sql"], params_for_query, request=request + stored_query["sql"], params_for_query, request=request ) # success message can come from on_success_message or on_success_message_sql message = None message_type = datasette.INFO - on_success_message_sql = canned_query.get("on_success_message_sql") + on_success_message_sql = stored_query.get("on_success_message_sql") if on_success_message_sql: try: message_result = ( @@ -1698,18 +1698,18 @@ class QueryView(View): message = "Error running on_success_message_sql: {}".format(ex) message_type = datasette.ERROR if not message: - message = canned_query.get( + message = stored_query.get( "on_success_message" ) or "Query executed, {} row{} affected".format( cursor.rowcount, "" if cursor.rowcount == 1 else "s" ) - redirect_url = canned_query.get("on_success_redirect") + redirect_url = stored_query.get("on_success_redirect") ok = True except Exception as ex: - message = canned_query.get("on_error_message") or str(ex) + message = stored_query.get("on_error_message") or str(ex) message_type = datasette.ERROR - redirect_url = canned_query.get("on_error_redirect") + redirect_url = stored_query.get("on_error_redirect") ok = False if should_return_json: return Response.json( @@ -1743,33 +1743,33 @@ class QueryView(View): allowed_dict = {r.child: r for r in allowed_tables_page.resources} # Are we a stored query? - canned_query = None - canned_query_write = False + stored_query = None + stored_query_write = False if "table" in request.url_vars: try: await datasette.resolve_table(request) except TableNotFound as table_not_found: # Was this actually a stored query? - canned_query = await datasette.get_canned_query( - table_not_found.database_name, table_not_found.table, request.actor + stored_query = await datasette.get_query( + table_not_found.database_name, table_not_found.table ) - if canned_query is None: + if stored_query is None: raise - canned_query_write = bool(canned_query.get("write")) + stored_query_write = bool(stored_query.get("write")) private = False - if canned_query: + if stored_query: # Respect stored query permissions visible, private = await datasette.check_visibility( request.actor, action="view-query", - resource=QueryResource(database=database, query=canned_query["name"]), + resource=QueryResource(database=database, query=stored_query["name"]), ) if not visible: raise Forbidden("You do not have permission to view this query") - if not canned_query_write: + if not stored_query_write: await _ensure_stored_query_execution_permissions( - datasette, db, canned_query, request.actor + datasette, db, stored_query, request.actor ) else: @@ -1783,15 +1783,15 @@ class QueryView(View): params = {key: request.args.get(key) for key in request.args} sql = None - if canned_query: - sql = canned_query["sql"] + if stored_query: + sql = stored_query["sql"] elif "sql" in params: sql = params.pop("sql") # Extract any :named parameters named_parameters = [] - if canned_query and canned_query.get("params"): - named_parameters = canned_query["params"] + if stored_query and stored_query.get("params"): + named_parameters = stored_query["params"] if not named_parameters and sql: named_parameters = derive_named_parameters(sql) named_parameter_values = { @@ -1817,9 +1817,9 @@ class QueryView(View): params_for_query = params - if sql and not canned_query_write: + if sql and not stored_query_write: try: - if not canned_query: + if not stored_query: # For regular queries we only allow SELECT, plus other rules validate_sql_select(sql) else: @@ -1879,7 +1879,7 @@ class QueryView(View): columns=columns, rows=rows, sql=sql, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query["name"] if stored_query else None, database=database, table=None, request=request, @@ -1911,10 +1911,10 @@ class QueryView(View): elif format_ == "html": headers = {} templates = [f"query-{to_css_class(database)}.html", "query.html"] - if canned_query: + if stored_query: templates.insert( 0, - f"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html", + f"query-{to_css_class(database)}-{to_css_class(stored_query['name'])}.html", ) environment = datasette.get_jinja_environment(request) @@ -1932,8 +1932,8 @@ class QueryView(View): } ) metadata = await datasette.get_database_metadata(database) - if canned_query: - metadata = dict(canned_query) + if stored_query: + metadata = dict(stored_query) metadata.pop("source", None) renderers = {} @@ -1968,7 +1968,7 @@ class QueryView(View): ) show_hide_hidden = "" - if canned_query and canned_query.get("hide_sql"): + if stored_query and stored_query.get("hide_sql"): if bool(params.get("_show_sql")): show_hide_link = path_with_removed_args(request, {"_show_sql"}) show_hide_text = "hide" @@ -2018,7 +2018,7 @@ class QueryView(View): ) save_query_url = None if ( - not canned_query + not stored_query and allow_execute_sql and allow_store_query and is_validated_sql @@ -2036,7 +2036,7 @@ class QueryView(View): datasette=datasette, actor=request.actor, database=database, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query["name"] if stored_query else None, request=request, sql=sql, params=params, @@ -2056,15 +2056,15 @@ class QueryView(View): "sql": sql, "params": params, }, - canned_query=canned_query["name"] if canned_query else None, + stored_query=stored_query["name"] if stored_query else None, private=private, - canned_query_write=canned_query_write, + stored_query_write=stored_query_write, db_is_immutable=not db.is_mutable, error=query_error, hide_sql=hide_sql, show_hide_link=datasette.urls.path(show_hide_link), show_hide_text=show_hide_text, - editable=not canned_query, + editable=not stored_query, allow_execute_sql=allow_execute_sql, save_query_url=save_query_url, tables=await get_tables(datasette, request, db, allowed_dict), @@ -2100,7 +2100,7 @@ class QueryView(View): datasette, request, database=database, - query_name=canned_query["name"] if canned_query else None, + query_name=stored_query["name"] if stored_query else None, ), query_actions=query_actions, ), diff --git a/datasette/views/table.py b/datasette/views/table.py index 7b1a5a82..da69c6b5 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -964,11 +964,11 @@ async def table_view_traced(datasette, request): resolved = await datasette.resolve_table(request) except TableNotFound as not_found: # Was this actually a stored query? - canned_query = await datasette.get_canned_query( - not_found.database_name, not_found.table, request.actor + stored_query = await datasette.get_query( + not_found.database_name, not_found.table ) # If this is a stored query, not a table, then dispatch to QueryView instead - if canned_query: + if stored_query: return await QueryView()(request, datasette) else: raise diff --git a/docs/authentication.rst b/docs/authentication.rst index 86df7f04..cec47f97 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -121,7 +121,7 @@ This configuration will deny access to everyone except the user with ``id`` of ` How permissions are resolved ---------------------------- -Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. +Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified. @@ -468,7 +468,7 @@ You can control the following: * Access to the entire Datasette instance * Access to specific databases * Access to specific tables and views -* Access to specific :ref:`queries ` +* Access to specific :ref:`queries ` If a user has permission to view a table they will be able to view that table, independent of if they have permission to view the database or instance that the table exists within. @@ -496,7 +496,7 @@ Here's how to restrict access to your entire Datasette instance to just the ``"i title: My private Datasette instance allow: id: root - + .. tab:: datasette.json @@ -644,7 +644,7 @@ This works for SQL views as well - you can list their names in the ``"tables"`` Access to specific queries -------------------------- -:ref:`Queries ` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. +:ref:`Queries ` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important. To limit access to the ``add_name`` query in your ``dogs.db`` database to just the :ref:`root user`: @@ -1020,7 +1020,7 @@ You can also restrict permissions such that they can only be used within specifi The resulting token will only be able to insert rows, and only to tables in the ``mydatabase`` database. -Finally, you can restrict permissions to individual resources - tables, SQL views and :ref:`named queries ` - within a specific database:: +Finally, you can restrict permissions to individual resources - tables, SQL views and :ref:`named queries ` - within a specific database:: datasette create-token root --resource mydatabase mytable insert-row diff --git a/docs/changelog.rst b/docs/changelog.rst index 674ff5b3..d15dec50 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,7 +11,8 @@ Unreleased - Fixed a bug where visiting ``//-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`) - The ``top_canned_query()`` plugin hook has been renamed to :ref:`top_stored_query() `. (:issue:`2747`) -- The ``canned_queries()`` plugin hook has been removed. Plugins can use the new ``datasette.add_query()``, ``datasette.update_query()`` and ``datasette.remove_query()`` methods to managed stored queries instead. +- The ``canned_queries()`` plugin hook has been removed. Plugins can use the new ``datasette.add_query()``, ``datasette.update_query()`` and ``datasette.remove_query()`` methods to manage stored queries instead. +- The ``datasette.get_canned_query()`` and ``datasette.get_canned_queries()`` methods have been removed. Plugins can use ``datasette.get_query()`` and ``datasette.list_queries()`` instead. .. _v1_0_a30: @@ -658,7 +659,7 @@ For more information and workarounds, read `the security advisory `` in a ``
      `` element, to help with advanced CSS customization. (:issue:`1446`) - The :ref:`render_cell() ` plugin hook can now return an awaitable function. This means the hook can execute SQL queries. (:issue:`1425`) - :ref:`plugin_register_routes` plugin hook now accepts an optional ``datasette`` argument. (:issue:`1404`) -- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`) +- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`queries_options`. (:issue:`1422`) - New ``--cpu`` option for :ref:`datasette publish cloudrun `. (:issue:`1420`) - If `Rich `__ is installed in the same virtual environment as Datasette, it will be used to provide enhanced display of error tracebacks on the console. (:issue:`1416`) - ``datasette.utils`` :ref:`internals_utils_parse_metadata` function, used by the new `datasette-remote-metadata plugin `__, is now a documented API. (:issue:`1405`) @@ -1426,7 +1427,7 @@ See also `Datasette 0.50: The annotated release notes `__. -- Writable canned queries now expose a JSON API, see :ref:`canned_queries_json_api`. (:issue:`880`) +- Writable canned queries now expose a JSON API, see :ref:`queries_json_api`. (:issue:`880`) - New mechanism for defining page templates with custom path parameters - a template file called ``pages/about/{slug}.html`` will be used to render any requests to ``/about/something``. See :ref:`custom_pages_parameters`. (:issue:`944`) - ``register_output_renderer()`` render functions can now return a ``Response``. (:issue:`953`) - New ``--upgrade`` option for ``datasette install``. (:issue:`945`) @@ -1518,7 +1519,7 @@ Magic parameters for canned queries, a log out feature, improved plugin document Magic parameters for canned queries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Canned queries now support :ref:`canned_queries_magic_parameters`, which can be used to insert or select automatically generated values. For example:: +Canned queries now support :ref:`queries_magic_parameters`, which can be used to insert or select automatically generated values. For example:: insert into logs (user_id, timestamp) @@ -1549,7 +1550,7 @@ New plugin hooks - :ref:`plugin_hook_register_magic_parameters` can be used to define new types of magic canned query parameters. - :ref:`plugin_hook_startup` can run custom code when Datasette first starts up. `datasette-init `__ is a new plugin that uses this hook to create database tables and views on startup if they have not yet been created. (:issue:`834`) -- :ref:`plugin_hook_canned_queries` lets plugins provide additional canned queries beyond those defined in Datasette's metadata. See `datasette-saved-queries `__ for an example of this hook in action. (:issue:`852`) +- ``canned_queries()`` lets plugins provide additional canned queries beyond those defined in Datasette's metadata. See `datasette-saved-queries `__ for an example of this hook in action. (:issue:`852`) - :ref:`plugin_hook_forbidden` is a hook for customizing how Datasette responds to 403 forbidden errors. (:issue:`812`) Smaller changes @@ -1624,7 +1625,7 @@ A new debug page at ``/-/permissions`` shows recent permission checks, to help a Writable canned queries ~~~~~~~~~~~~~~~~~~~~~~~ -Datasette's :ref:`canned_queries` feature lets you define SQL queries in ``metadata.json`` which can then be executed by users visiting a specific URL. https://latest.datasette.io/fixtures/neighborhood_search for example. +Datasette's :ref:`queries` feature lets you define SQL queries in ``metadata.json`` which can then be executed by users visiting a specific URL. https://latest.datasette.io/fixtures/neighborhood_search for example. Canned queries were previously restricted to ``SELECT``, but Datasette 0.44 introduces the ability for canned queries to execute ``INSERT`` or ``UPDATE`` queries as well, using the new ``"write": true`` property (:issue:`800`): @@ -1643,7 +1644,7 @@ Canned queries were previously restricted to ``SELECT``, but Datasette 0.44 intr } } -See :ref:`canned_queries_writable` for more details. +See :ref:`queries_writable` for more details. Flash messages ~~~~~~~~~~~~~~ @@ -1698,7 +1699,7 @@ Smaller changes - New ``request.cookies`` property. - ``/-/plugins`` endpoint now shows a list of hooks implemented by each plugin, e.g. https://latest.datasette.io/-/plugins?all=1 - ``request.post_vars()`` method no longer discards empty values. -- New "params" canned query key for explicitly setting named parameters, see :ref:`canned_queries_named_parameters`. (:issue:`797`) +- New "params" canned query key for explicitly setting named parameters, see :ref:`queries_named_parameters`. (:issue:`797`) - ``request.args`` is now a :ref:`MultiParams ` object. - Fixed a bug with the ``datasette plugins`` command. (:issue:`802`) - Nicer pattern for using ``make_app_client()`` in tests. (:issue:`395`) @@ -1732,7 +1733,7 @@ The main focus of this release is a major upgrade to the :ref:`plugin_register_o * Visually distinguish float and integer columns - useful for figuring out why order-by-column might be returning unexpected results. (:issue:`729`) * The :ref:`internals_request`, which is passed to several plugin hooks, is now documented. (:issue:`706`) * New ``metadata.json`` option for setting a custom default page size for specific tables and views, see :ref:`table_configuration_size`. (:issue:`751`) -* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega `__, see :ref:`canned_queries_options`. (:issue:`706`) +* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega `__, see :ref:`queries_options`. (:issue:`706`) * Fixed a bug in ``datasette publish`` when running on operating systems where the ``/tmp`` directory lives in a different volume, using a backport of the Python 3.8 ``shutil.copytree()`` function. (:issue:`744`) * Every plugin hook is now covered by the unit tests, and a new unit test checks that each plugin hook has at least one corresponding test. (:issue:`771`, :issue:`773`) @@ -2249,7 +2250,7 @@ A number of small new features: - Documentation for :ref:`datasette publish and datasette package `, closes `#337 `_ - Fixed compatibility with Python 3.7 - ``datasette publish heroku`` now supports app names via the ``-n`` option, which can also be used to overwrite an existing application [Russ Garrett] -- Title and description metadata can now be set for :ref:`canned SQL queries `, closes `#342 `_ +- Title and description metadata can now be set for :ref:`canned SQL queries `, closes `#342 `_ - New ``force_https_on`` config option, fixes ``https://`` API URLs when deploying to Zeit Now - closes `#333 `_ - ``?_json_infinity=1`` query string argument for handling Infinity/-Infinity values in JSON, closes `#332 `_ - URLs displayed in the results of custom SQL queries are now URLified, closes `#298 `_ diff --git a/docs/configuration.rst b/docs/configuration.rst index cf9590b8..9213a640 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -434,12 +434,12 @@ Here is a simple example: :ref:`authentication_permissions_config` has the full details. -.. _configuration_reference_canned_queries: +.. _configuration_reference_queries: Queries configuration ~~~~~~~~~~~~~~~~~~~~~ -:ref:`Queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: +:ref:`Queries ` are named SQL queries that appear in the Datasette interface. They can be configured in ``datasette.yaml`` using the ``queries`` key at the database level: .. [[[cog from metadata_doc import config_example, config_example @@ -484,7 +484,7 @@ Queries configuration } .. [[[end]]] -See the :ref:`queries documentation ` for more, including how to configure :ref:`writable queries `. +See the :ref:`queries documentation ` for more, including how to configure :ref:`writable queries `. .. _configuration_reference_css_js: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4737ca03..2a0ddc93 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1207,16 +1207,6 @@ Potential use-cases: Examples: `datasette-saved-queries `__, `datasette-init `__ -.. _plugin_hook_canned_queries: - -canned_queries(datasette, database, actor) ------------------------------------------- - -This hook has been removed. Plugins that need to add stored queries should use -the :ref:`plugin_hook_startup` hook and call ``await datasette.add_query(...)``. - -Example: `datasette-saved-queries `__ - .. _plugin_hook_actor_from_request: actor_from_request(datasette, request) @@ -1635,7 +1625,7 @@ register_magic_parameters(datasette) ``datasette`` - :ref:`internals_datasette` You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``. -:ref:`canned_queries_magic_parameters` can be used to add automatic parameters to :ref:`configured queries `. This plugin hook allows additional magic parameters to be defined by plugins. +:ref:`queries_magic_parameters` can be used to add automatic parameters to :ref:`configured queries `. This plugin hook allows additional magic parameters to be defined by plugins. Magic parameters all take this format: ``_prefix_rest_of_parameter``. The prefix indicates which magic parameter function should be called - the rest of the parameter is passed as an argument to that function. diff --git a/docs/spatialite.rst b/docs/spatialite.rst index 1999ab78..bea679aa 100644 --- a/docs/spatialite.rst +++ b/docs/spatialite.rst @@ -30,7 +30,7 @@ Warning The following steps are recommended: - Disable arbitrary SQL queries by untrusted users. See :ref:`authentication_permissions_execute_sql` for ways to do this. The easiest is to start Datasette with the ``datasette --setting default_allow_sql off`` option. - - Define :ref:`queries ` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. + - Define :ref:`queries ` with the SQL queries that use SpatiaLite functions that you want people to be able to execute. The `Datasette SpatiaLite tutorial `__ includes detailed instructions for running SpatiaLite safely using these techniques diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index d60656e3..f593a534 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -66,7 +66,7 @@ You can also use the `sqlite-utils `__ tool sqlite-utils create-view sf-trees.db demo_view "select qSpecies from Street_Tree_List" -.. _canned_queries: +.. _queries: Queries ------- @@ -173,7 +173,7 @@ You can opt out of this behavior for a configured query using ``is_trusted: fals sql: select * from report is_trusted: false -.. _canned_queries_named_parameters: +.. _queries_named_parameters: Query parameters ~~~~~~~~~~~~~~~~ @@ -313,7 +313,7 @@ You can alternatively provide an explicit list of named parameters using the ``" } .. [[[end]]] -.. _canned_queries_options: +.. _queries_options: Additional query options ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -389,7 +389,7 @@ This example demonstrates both ``fragment`` and ``hide_sql``: `See here `__ for a demo of this in action. -.. _canned_queries_writable: +.. _queries_writable: Writable queries ~~~~~~~~~~~~~~~~ @@ -524,7 +524,7 @@ You can pre-populate form fields when the page first loads using a query string, If you specify a query in ``"on_success_message_sql"``, that query will be executed after the main query. The first column of the first row return by that query will be displayed as a success message. Named parameters from the main query will be made available to the success message query as well. -.. _canned_queries_magic_parameters: +.. _queries_magic_parameters: Magic parameters ~~~~~~~~~~~~~~~~ @@ -621,7 +621,7 @@ The form presented at ``/mydatabase/add_message`` will have just a field for ``m Additional custom magic parameters can be added by plugins using the :ref:`plugin_hook_register_magic_parameters` hook. -.. _canned_queries_json_api: +.. _queries_json_api: JSON API for writable queries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/upgrade-1.0a20.md b/docs/upgrade-1.0a20.md index fbc3f4a8..b27d8bd5 100644 --- a/docs/upgrade-1.0a20.md +++ b/docs/upgrade-1.0a20.md @@ -244,7 +244,7 @@ except (KeyError, TypeError): New code: ```python try: - query_info = await datasette.get_canned_query(database, query_name, request.actor) + query_info = await datasette.get_query(database, query_name) if query_info and "title" in query_info: title = query_info["title"] except (KeyError, TypeError): @@ -253,7 +253,7 @@ except (KeyError, TypeError): ### Update render functions to async -If your plugin's render function needs to call `datasette.get_canned_query()` or other async Datasette methods, it must be declared as async: +If your plugin's render function needs to call `datasette.get_query()` or other async Datasette methods, it must be declared as async: Old code: ```python @@ -268,7 +268,7 @@ New code: async def render_atom(datasette, request, sql, columns, rows, database, table, query_name, view_name, data): # ... if query_name: - query_info = await datasette.get_canned_query(database, query_name, request.actor) + query_info = await datasette.get_query(database, query_name) if query_info and "title" in query_info: title = query_info["title"] ``` diff --git a/tests/test_canned_queries.py b/tests/test_canned_queries.py deleted file mode 100644 index ae2c74e0..00000000 --- a/tests/test_canned_queries.py +++ /dev/null @@ -1,473 +0,0 @@ -from bs4 import BeautifulSoup as Soup -from asgiref.sync import async_to_sync -import json -import pytest -import re -from .fixtures import make_app_client - - -def update_query(client, name, **kwargs): - async_to_sync(client.ds.invoke_startup)() - async_to_sync(client.ds.update_query)("data", name, **kwargs) - - -@pytest.fixture -def canned_write_client(tmpdir): - template_dir = tmpdir / "canned_write_templates" - template_dir.mkdir() - (template_dir / "query-data-update_name.html").write_text( - """ - {% extends "query.html" %} - {% block content %}!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!{{ super() }}{% endblock %} - """, - "utf-8", - ) - with make_app_client( - extra_databases={"data.db": "create table names (name text)"}, - template_dir=str(template_dir), - config={ - "databases": { - "data": { - "queries": { - "canned_read": {"sql": "select * from names"}, - "add_name": { - "sql": "insert into names (name) values (:name)", - "write": True, - "on_success_redirect": "/data/add_name?success", - }, - "add_name_specify_id": { - "sql": "insert into names (rowid, name) values (:rowid, :name)", - "on_success_message_sql": "select 'Name added: ' || :name || ' with rowid ' || :rowid", - "write": True, - "on_error_redirect": "/data/add_name_specify_id?error", - }, - "add_name_specify_id_with_error_in_on_success_message_sql": { - "sql": "insert into names (rowid, name) values (:rowid, :name)", - "on_success_message_sql": "select this is bad SQL", - "write": True, - }, - "delete_name": { - "sql": "delete from names where rowid = :rowid", - "write": True, - "on_success_message": "Name deleted", - "allow": {"id": "root"}, - }, - "update_name": { - "sql": "update names set name = :name where rowid = :rowid", - "params": ["rowid", "name", "extra"], - "write": True, - }, - } - } - } - }, - ) as client: - yield client - - -@pytest.fixture -def canned_write_immutable_client(): - with make_app_client( - is_immutable=True, - config={ - "databases": { - "fixtures": { - "queries": { - "add": { - "sql": "insert into sortable (text) values (:text)", - "write": True, - }, - } - } - } - }, - ) as client: - yield client - - -@pytest.mark.asyncio -async def test_canned_query_with_named_parameter(ds_client): - response = await ds_client.get( - "/fixtures/neighborhood_search.json?text=town&_shape=arrays" - ) - assert response.json()["rows"] == [ - ["Corktown", "Detroit", "MI"], - ["Downtown", "Los Angeles", "CA"], - ["Downtown", "Detroit", "MI"], - ["Greektown", "Detroit", "MI"], - ["Koreatown", "Los Angeles", "CA"], - ["Mexicantown", "Detroit", "MI"], - ] - - -def test_insert(canned_write_client): - response = canned_write_client.post( - "/data/add_name", - {"name": "Hello"}, - csrftoken_from=True, - cookies={"foo": "bar"}, - ) - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert messages == [["Query executed, 1 row affected", 1]] - assert response.status == 302 - assert response.headers["Location"] == "/data/add_name?success" - - -def test_insert_blocked_cross_site(canned_write_client): - # A cross-site POST (browser-originated) must be blocked - response = canned_write_client.post( - "/data/add_name", - {"name": "Hello"}, - headers={"sec-fetch-site": "cross-site"}, - ) - assert 403 == response.status - - -def test_insert_no_cookies_no_csrf(canned_write_client): - response = canned_write_client.post("/data/add_name", {"name": "Hello"}) - assert 302 == response.status - assert "/data/add_name?success" == response.headers["Location"] - - -def test_custom_success_message(canned_write_client): - response = canned_write_client.post( - "/data/delete_name", - {"rowid": 1}, - cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, - csrftoken_from=True, - ) - assert 302 == response.status - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert [["Name deleted", 1]] == messages - - -def test_insert_error(canned_write_client): - canned_write_client.post("/data/add_name", {"name": "Hello"}, csrftoken_from=True) - response = canned_write_client.post( - "/data/add_name_specify_id", - {"rowid": 1, "name": "Should fail"}, - csrftoken_from=True, - ) - assert 302 == response.status - assert "/data/add_name_specify_id?error" == response.headers["Location"] - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert [["UNIQUE constraint failed: names.rowid", 3]] == messages - # How about with a custom error message? - update_query(canned_write_client, "add_name_specify_id", on_error_message="ERROR") - response = canned_write_client.post( - "/data/add_name_specify_id", - {"rowid": 1, "name": "Should fail"}, - csrftoken_from=True, - ) - assert [["ERROR", 3]] == canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - - -def test_on_success_message_sql(canned_write_client): - response = canned_write_client.post( - "/data/add_name_specify_id", - {"rowid": 5, "name": "Should be OK"}, - csrftoken_from=True, - ) - assert response.status == 302 - assert response.headers["Location"] == "/data/add_name_specify_id" - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert messages == [["Name added: Should be OK with rowid 5", 1]] - - -def test_error_in_on_success_message_sql(canned_write_client): - response = canned_write_client.post( - "/data/add_name_specify_id_with_error_in_on_success_message_sql", - {"rowid": 1, "name": "Should fail"}, - csrftoken_from=True, - ) - messages = canned_write_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert messages == [ - ["Error running on_success_message_sql: no such column: bad", 3] - ] - - -def test_custom_params(canned_write_client): - response = canned_write_client.get("/data/update_name?extra=foo") - assert ( - '' - in response.text - ) - - -def test_canned_query_pages_no_vary_header(canned_write_client): - # These pages no longer embed per-cookie CSRF tokens, so they must not - # set Vary: Cookie - they should be cacheable across users. - assert "vary" not in canned_write_client.get("/data").headers - assert "vary" not in canned_write_client.get("/data/update_name").headers - - -def test_json_post_body(canned_write_client): - response = canned_write_client.post( - "/data/add_name", - body=json.dumps({"name": ["Hello", "there"]}), - ) - assert 302 == response.status - assert "/data/add_name?success" == response.headers["Location"] - rows = canned_write_client.get("/data/names.json?_shape=array").json - assert rows == [{"rowid": 1, "name": "['Hello', 'there']"}] - - -@pytest.mark.parametrize( - "headers,body,querystring", - ( - (None, "name=NameGoesHere", "?_json=1"), - ({"Accept": "application/json"}, "name=NameGoesHere", None), - (None, "name=NameGoesHere&_json=1", None), - (None, '{"name": "NameGoesHere", "_json": 1}', None), - ), -) -def test_json_response(canned_write_client, headers, body, querystring): - response = canned_write_client.post( - "/data/add_name" + (querystring or ""), - body=body, - headers=headers, - ) - assert 200 == response.status - assert response.headers["content-type"] == "application/json; charset=utf-8" - assert response.json == { - "ok": True, - "message": "Query executed, 1 row affected", - "redirect": "/data/add_name?success", - } - rows = canned_write_client.get("/data/names.json?_shape=array").json - assert rows == [{"rowid": 1, "name": "NameGoesHere"}] - - -def test_canned_query_permissions_on_database_page(canned_write_client): - # Without auth shows the five public queries - anon_response = canned_write_client.get("/data.json") - query_names = {q["name"] for q in anon_response.json["queries"]} - assert query_names == { - "add_name_specify_id_with_error_in_on_success_message_sql", - "update_name", - "add_name_specify_id", - "canned_read", - "add_name", - } - assert anon_response.json["queries_more"] is False - - # With auth the database page preview shows the first five queries - response = canned_write_client.get( - "/data.json", - cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, - ) - assert response.status == 200 - query_names_and_private = sorted( - [ - {"name": q["name"], "private": q["private"]} - for q in response.json["queries"] - ], - key=lambda q: q["name"], - ) - assert query_names_and_private == [ - {"name": "add_name", "private": False}, - {"name": "add_name_specify_id", "private": False}, - { - "name": "add_name_specify_id_with_error_in_on_success_message_sql", - "private": False, - }, - {"name": "canned_read", "private": False}, - {"name": "delete_name", "private": True}, - ] - assert response.json["queries_more"] is True - - # The full query list endpoint includes the remaining query - response = canned_write_client.get( - "/data/-/queries.json?_size=10", - cookies={"ds_actor": canned_write_client.actor_cookie({"id": "root"})}, - ) - assert response.status == 200 - query_names_and_private = sorted( - [ - {"name": q["name"], "private": q["private"]} - for q in response.json["queries"] - ], - key=lambda q: q["name"], - ) - assert query_names_and_private == [ - {"name": "add_name", "private": False}, - {"name": "add_name_specify_id", "private": False}, - { - "name": "add_name_specify_id_with_error_in_on_success_message_sql", - "private": False, - }, - {"name": "canned_read", "private": False}, - {"name": "delete_name", "private": True}, - {"name": "update_name", "private": False}, - ] - - -def test_canned_query_permissions(canned_write_client): - assert 403 == canned_write_client.get("/data/delete_name").status - assert 200 == canned_write_client.get("/data/update_name").status - cookies = {"ds_actor": canned_write_client.actor_cookie({"id": "root"})} - assert 200 == canned_write_client.get("/data/delete_name", cookies=cookies).status - assert 200 == canned_write_client.get("/data/update_name", cookies=cookies).status - - -@pytest.fixture(scope="session") -def magic_parameters_client(): - with make_app_client( - extra_databases={"data.db": "create table logs (line text)"}, - config={ - "databases": { - "data": { - "queries": { - "runme_post": {"sql": "", "write": True}, - "runme_get": {"sql": ""}, - } - } - } - }, - ) as client: - yield client - - -@pytest.mark.parametrize( - "magic_parameter,expected_re", - [ - ("_actor_id", "root"), - ("_header_host", "localhost"), - ("_header_not_a_thing", ""), - ("_cookie_foo", "bar"), - ("_now_epoch", r"^\d+$"), - ("_now_date_utc", r"^\d{4}-\d{2}-\d{2}$"), - ("_now_datetime_utc", r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$"), - ("_random_chars_1", r"^\w$"), - ("_random_chars_10", r"^\w{10}$"), - ], -) -def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re): - update_query( - magic_parameters_client, - "runme_post", - sql=f"insert into logs (line) values (:{magic_parameter})", - ) - update_query( - magic_parameters_client, - "runme_get", - sql=f"select :{magic_parameter} as result", - ) - cookies = { - "ds_actor": magic_parameters_client.actor_cookie({"id": "root"}), - "foo": "bar", - } - # Test the GET version - get_response = magic_parameters_client.get( - "/data/runme_get.json?_shape=array", cookies=cookies - ) - get_actual = get_response.json[0]["result"] - assert re.match(expected_re, str(get_actual)) - # Test the form - form_response = magic_parameters_client.get("/data/runme_post") - soup = Soup(form_response.body, "html.parser") - # The magic parameter should not be represented as a form field - assert None is soup.find("input", {"name": magic_parameter}) - # Submit the form to create a log line - response = magic_parameters_client.post( - "/data/runme_post?_json=1", {}, csrftoken_from=True, cookies=cookies - ) - assert response.json == { - "ok": True, - "message": "Query executed, 1 row affected", - "redirect": None, - } - post_actual = magic_parameters_client.get( - "/data/logs.json?_sort_desc=rowid&_shape=array" - ).json[0]["line"] - assert re.match(expected_re, post_actual) - - -@pytest.mark.parametrize("use_csrf", [True, False]) -@pytest.mark.parametrize("return_json", [True, False]) -def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_json): - update_query( - magic_parameters_client, - "runme_post", - sql="insert into logs (line) values (:_header_host)", - ) - qs = "" - if return_json: - qs = "?_json=1" - response = magic_parameters_client.post( - f"/data/runme_post{qs}", - {}, - csrftoken_from=use_csrf or None, - ) - if return_json: - assert response.status == 200 - assert response.json["ok"], response.json - else: - assert response.status == 302 - messages = magic_parameters_client.ds.unsign( - response.cookies["ds_messages"], "messages" - ) - assert [["Query executed, 1 row affected", 1]] == messages - post_actual = magic_parameters_client.get( - "/data/logs.json?_sort_desc=rowid&_shape=array" - ).json[0]["line"] - assert post_actual == "localhost" - - -def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client): - response = magic_parameters_client.get( - "/data/-/query.json?sql=select+:_header_host&_shape=array" - ) - assert 400 == response.status - assert response.json["error"].startswith("You did not supply a value for binding") - - -def test_canned_write_custom_template(canned_write_client): - response = canned_write_client.get("/data/update_name") - assert response.status == 200 - assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text - assert ( - "" - in response.text - ) - # And test for link rel=alternate while we're here: - assert ( - '' - in response.text - ) - assert ( - response.headers["link"] - == '; rel="alternate"; type="application/json+datasette"' - ) - - -def test_canned_write_query_disabled_for_immutable_database( - canned_write_immutable_client, -): - response = canned_write_immutable_client.get("/fixtures/add") - assert response.status == 200 - assert ( - "This query cannot be executed because the database is immutable." - in response.text - ) - assert '' in response.text - # Submitting form should get a forbidden error - response = canned_write_immutable_client.post( - "/fixtures/add", - {"text": "text"}, - csrftoken_from=True, - ) - assert response.status == 403 - assert "Database is immutable" in response.text diff --git a/tests/test_html.py b/tests/test_html.py index 8edb9f6e..a9de5e79 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -633,7 +633,7 @@ async def test_404_content_type(ds_client): @pytest.mark.asyncio -async def test_canned_query_default_title(ds_client): +async def test_stored_query_default_title(ds_client): response = await ds_client.get("/fixtures/magic_parameters") assert response.status_code == 200 soup = Soup(response.content, "html.parser") @@ -641,7 +641,7 @@ async def test_canned_query_default_title(ds_client): @pytest.mark.asyncio -async def test_canned_query_with_custom_metadata(ds_client): +async def test_stored_query_with_custom_metadata(ds_client): response = await ds_client.get("/fixtures/neighborhood_search?text=town") assert response.status_code == 200 soup = Soup(response.content, "html.parser") @@ -700,7 +700,7 @@ async def test_show_hide_sql_query(ds_client): @pytest.mark.asyncio -async def test_canned_query_with_hide_has_no_hidden_sql(ds_client): +async def test_stored_query_with_hide_has_no_hidden_sql(ds_client): # For a stored query the show/hide should NOT have a hidden SQL field # https://github.com/simonw/datasette/issues/1411 response = await ds_client.get("/fixtures/pragma_cache_size?_hide_sql=1") @@ -720,7 +720,7 @@ async def test_canned_query_with_hide_has_no_hidden_sql(ds_client): (True, "?_show_sql=1", "_show_sql", "/_memory/one", "hide"), ), ) -def test_canned_query_show_hide_metadata_option( +def test_stored_query_show_hide_metadata_option( hide_sql, querystring, expected_hidden, @@ -981,10 +981,10 @@ def test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix): ("/fixtures/magic_parameters", None), ], ) -async def test_edit_sql_link_on_canned_queries(ds_client, path, expected): +async def test_edit_sql_link_on_stored_queries(ds_client, path, expected): response = await ds_client.get(path) assert response.status_code == 200 - expected_link = f'Edit SQL' + expected_link = f'Edit SQL' if expected: assert expected_link in response.text else: diff --git a/tests/test_jump.py b/tests/test_jump.py index f60af0fd..513a809f 100644 --- a/tests/test_jump.py +++ b/tests/test_jump.py @@ -76,7 +76,7 @@ async def ds_for_jump(): @pytest.mark.asyncio -async def test_jump_searches_tables_databases_views_and_canned_queries(ds_for_jump): +async def test_jump_searches_tables_databases_views_and_stored_queries(ds_for_jump): response = await ds_for_jump.client.get( "/-/jump.json?q=content", actor={"id": "user"} ) @@ -98,7 +98,7 @@ async def test_jump_searches_tables_databases_views_and_canned_queries(ds_for_ju @pytest.mark.asyncio -async def test_jump_uses_canned_query_names_not_titles(ds_for_jump): +async def test_jump_uses_stored_query_names_not_titles(ds_for_jump): response = await ds_for_jump.client.get( "/-/jump.json?q=datasette", actor={"id": "user"} ) From cafb6b9dbd77e438b81a7784c4434957b73fb856 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:20:29 -0700 Subject: [PATCH 140/156] Need is_trusted=True for the counters demo --- .github/workflows/deploy-latest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 166d33d0..b0640ae8 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -76,6 +76,7 @@ jobs: "update counters set value = value + 1 where name = '{}'".format(name), on_success_message_sql="select 'Counter {name} incremented to ' || value from counters where name = '{name}'".format(name=name), is_write=True, + is_trusted=True, ) await datasette.add_query( "counters", @@ -83,6 +84,7 @@ jobs: "update counters set value = value - 1 where name = '{}'".format(name), on_success_message_sql="select 'Counter {name} decremented to ' || value from counters where name = '{name}'".format(name=name), is_write=True, + is_trusted=True, ) return inner EOF From e2864fc895603c1fdf03d3c55812a0a6e56779ff Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:21:09 -0700 Subject: [PATCH 141/156] test_stored_queries.py --- tests/test_stored_queries.py | 473 +++++++++++++++++++++++++++++++++++ 1 file changed, 473 insertions(+) create mode 100644 tests/test_stored_queries.py diff --git a/tests/test_stored_queries.py b/tests/test_stored_queries.py new file mode 100644 index 00000000..2c648d5f --- /dev/null +++ b/tests/test_stored_queries.py @@ -0,0 +1,473 @@ +from bs4 import BeautifulSoup as Soup +from asgiref.sync import async_to_sync +import json +import pytest +import re +from .fixtures import make_app_client + + +def update_query(client, name, **kwargs): + async_to_sync(client.ds.invoke_startup)() + async_to_sync(client.ds.update_query)("data", name, **kwargs) + + +@pytest.fixture +def stored_write_client(tmpdir): + template_dir = tmpdir / "stored_write_templates" + template_dir.mkdir() + (template_dir / "query-data-update_name.html").write_text( + """ + {% extends "query.html" %} + {% block content %}!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!{{ super() }}{% endblock %} + """, + "utf-8", + ) + with make_app_client( + extra_databases={"data.db": "create table names (name text)"}, + template_dir=str(template_dir), + config={ + "databases": { + "data": { + "queries": { + "stored_read": {"sql": "select * from names"}, + "add_name": { + "sql": "insert into names (name) values (:name)", + "write": True, + "on_success_redirect": "/data/add_name?success", + }, + "add_name_specify_id": { + "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select 'Name added: ' || :name || ' with rowid ' || :rowid", + "write": True, + "on_error_redirect": "/data/add_name_specify_id?error", + }, + "add_name_specify_id_with_error_in_on_success_message_sql": { + "sql": "insert into names (rowid, name) values (:rowid, :name)", + "on_success_message_sql": "select this is bad SQL", + "write": True, + }, + "delete_name": { + "sql": "delete from names where rowid = :rowid", + "write": True, + "on_success_message": "Name deleted", + "allow": {"id": "root"}, + }, + "update_name": { + "sql": "update names set name = :name where rowid = :rowid", + "params": ["rowid", "name", "extra"], + "write": True, + }, + } + } + } + }, + ) as client: + yield client + + +@pytest.fixture +def stored_write_immutable_client(): + with make_app_client( + is_immutable=True, + config={ + "databases": { + "fixtures": { + "queries": { + "add": { + "sql": "insert into sortable (text) values (:text)", + "write": True, + }, + } + } + } + }, + ) as client: + yield client + + +@pytest.mark.asyncio +async def test_stored_query_with_named_parameter(ds_client): + response = await ds_client.get( + "/fixtures/neighborhood_search.json?text=town&_shape=arrays" + ) + assert response.json()["rows"] == [ + ["Corktown", "Detroit", "MI"], + ["Downtown", "Los Angeles", "CA"], + ["Downtown", "Detroit", "MI"], + ["Greektown", "Detroit", "MI"], + ["Koreatown", "Los Angeles", "CA"], + ["Mexicantown", "Detroit", "MI"], + ] + + +def test_insert(stored_write_client): + response = stored_write_client.post( + "/data/add_name", + {"name": "Hello"}, + csrftoken_from=True, + cookies={"foo": "bar"}, + ) + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [["Query executed, 1 row affected", 1]] + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name?success" + + +def test_insert_blocked_cross_site(stored_write_client): + # A cross-site POST (browser-originated) must be blocked + response = stored_write_client.post( + "/data/add_name", + {"name": "Hello"}, + headers={"sec-fetch-site": "cross-site"}, + ) + assert 403 == response.status + + +def test_insert_no_cookies_no_csrf(stored_write_client): + response = stored_write_client.post("/data/add_name", {"name": "Hello"}) + assert 302 == response.status + assert "/data/add_name?success" == response.headers["Location"] + + +def test_custom_success_message(stored_write_client): + response = stored_write_client.post( + "/data/delete_name", + {"rowid": 1}, + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, + csrftoken_from=True, + ) + assert 302 == response.status + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["Name deleted", 1]] == messages + + +def test_insert_error(stored_write_client): + stored_write_client.post("/data/add_name", {"name": "Hello"}, csrftoken_from=True) + response = stored_write_client.post( + "/data/add_name_specify_id", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + assert 302 == response.status + assert "/data/add_name_specify_id?error" == response.headers["Location"] + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["UNIQUE constraint failed: names.rowid", 3]] == messages + # How about with a custom error message? + update_query(stored_write_client, "add_name_specify_id", on_error_message="ERROR") + response = stored_write_client.post( + "/data/add_name_specify_id", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + assert [["ERROR", 3]] == stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + + +def test_on_success_message_sql(stored_write_client): + response = stored_write_client.post( + "/data/add_name_specify_id", + {"rowid": 5, "name": "Should be OK"}, + csrftoken_from=True, + ) + assert response.status == 302 + assert response.headers["Location"] == "/data/add_name_specify_id" + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [["Name added: Should be OK with rowid 5", 1]] + + +def test_error_in_on_success_message_sql(stored_write_client): + response = stored_write_client.post( + "/data/add_name_specify_id_with_error_in_on_success_message_sql", + {"rowid": 1, "name": "Should fail"}, + csrftoken_from=True, + ) + messages = stored_write_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert messages == [ + ["Error running on_success_message_sql: no such column: bad", 3] + ] + + +def test_custom_params(stored_write_client): + response = stored_write_client.get("/data/update_name?extra=foo") + assert ( + '' + in response.text + ) + + +def test_stored_query_pages_no_vary_header(stored_write_client): + # These pages no longer embed per-cookie CSRF tokens, so they must not + # set Vary: Cookie - they should be cacheable across users. + assert "vary" not in stored_write_client.get("/data").headers + assert "vary" not in stored_write_client.get("/data/update_name").headers + + +def test_json_post_body(stored_write_client): + response = stored_write_client.post( + "/data/add_name", + body=json.dumps({"name": ["Hello", "there"]}), + ) + assert 302 == response.status + assert "/data/add_name?success" == response.headers["Location"] + rows = stored_write_client.get("/data/names.json?_shape=array").json + assert rows == [{"rowid": 1, "name": "['Hello', 'there']"}] + + +@pytest.mark.parametrize( + "headers,body,querystring", + ( + (None, "name=NameGoesHere", "?_json=1"), + ({"Accept": "application/json"}, "name=NameGoesHere", None), + (None, "name=NameGoesHere&_json=1", None), + (None, '{"name": "NameGoesHere", "_json": 1}', None), + ), +) +def test_json_response(stored_write_client, headers, body, querystring): + response = stored_write_client.post( + "/data/add_name" + (querystring or ""), + body=body, + headers=headers, + ) + assert 200 == response.status + assert response.headers["content-type"] == "application/json; charset=utf-8" + assert response.json == { + "ok": True, + "message": "Query executed, 1 row affected", + "redirect": "/data/add_name?success", + } + rows = stored_write_client.get("/data/names.json?_shape=array").json + assert rows == [{"rowid": 1, "name": "NameGoesHere"}] + + +def test_stored_query_permissions_on_database_page(stored_write_client): + # Without auth shows the five public queries + anon_response = stored_write_client.get("/data.json") + query_names = {q["name"] for q in anon_response.json["queries"]} + assert query_names == { + "add_name_specify_id_with_error_in_on_success_message_sql", + "update_name", + "add_name_specify_id", + "stored_read", + "add_name", + } + assert anon_response.json["queries_more"] is False + + # With auth the database page preview shows the first five queries + response = stored_write_client.get( + "/data.json", + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, + ) + assert response.status == 200 + query_names_and_private = sorted( + [ + {"name": q["name"], "private": q["private"]} + for q in response.json["queries"] + ], + key=lambda q: q["name"], + ) + assert query_names_and_private == [ + {"name": "add_name", "private": False}, + {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, + {"name": "delete_name", "private": True}, + {"name": "stored_read", "private": False}, + ] + assert response.json["queries_more"] is True + + # The full query list endpoint includes the remaining query + response = stored_write_client.get( + "/data/-/queries.json?_size=10", + cookies={"ds_actor": stored_write_client.actor_cookie({"id": "root"})}, + ) + assert response.status == 200 + query_names_and_private = sorted( + [ + {"name": q["name"], "private": q["private"]} + for q in response.json["queries"] + ], + key=lambda q: q["name"], + ) + assert query_names_and_private == [ + {"name": "add_name", "private": False}, + {"name": "add_name_specify_id", "private": False}, + { + "name": "add_name_specify_id_with_error_in_on_success_message_sql", + "private": False, + }, + {"name": "delete_name", "private": True}, + {"name": "stored_read", "private": False}, + {"name": "update_name", "private": False}, + ] + + +def test_stored_query_permissions(stored_write_client): + assert 403 == stored_write_client.get("/data/delete_name").status + assert 200 == stored_write_client.get("/data/update_name").status + cookies = {"ds_actor": stored_write_client.actor_cookie({"id": "root"})} + assert 200 == stored_write_client.get("/data/delete_name", cookies=cookies).status + assert 200 == stored_write_client.get("/data/update_name", cookies=cookies).status + + +@pytest.fixture(scope="session") +def magic_parameters_client(): + with make_app_client( + extra_databases={"data.db": "create table logs (line text)"}, + config={ + "databases": { + "data": { + "queries": { + "runme_post": {"sql": "", "write": True}, + "runme_get": {"sql": ""}, + } + } + } + }, + ) as client: + yield client + + +@pytest.mark.parametrize( + "magic_parameter,expected_re", + [ + ("_actor_id", "root"), + ("_header_host", "localhost"), + ("_header_not_a_thing", ""), + ("_cookie_foo", "bar"), + ("_now_epoch", r"^\d+$"), + ("_now_date_utc", r"^\d{4}-\d{2}-\d{2}$"), + ("_now_datetime_utc", r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$"), + ("_random_chars_1", r"^\w$"), + ("_random_chars_10", r"^\w{10}$"), + ], +) +def test_magic_parameters(magic_parameters_client, magic_parameter, expected_re): + update_query( + magic_parameters_client, + "runme_post", + sql=f"insert into logs (line) values (:{magic_parameter})", + ) + update_query( + magic_parameters_client, + "runme_get", + sql=f"select :{magic_parameter} as result", + ) + cookies = { + "ds_actor": magic_parameters_client.actor_cookie({"id": "root"}), + "foo": "bar", + } + # Test the GET version + get_response = magic_parameters_client.get( + "/data/runme_get.json?_shape=array", cookies=cookies + ) + get_actual = get_response.json[0]["result"] + assert re.match(expected_re, str(get_actual)) + # Test the form + form_response = magic_parameters_client.get("/data/runme_post") + soup = Soup(form_response.body, "html.parser") + # The magic parameter should not be represented as a form field + assert None is soup.find("input", {"name": magic_parameter}) + # Submit the form to create a log line + response = magic_parameters_client.post( + "/data/runme_post?_json=1", {}, csrftoken_from=True, cookies=cookies + ) + assert response.json == { + "ok": True, + "message": "Query executed, 1 row affected", + "redirect": None, + } + post_actual = magic_parameters_client.get( + "/data/logs.json?_sort_desc=rowid&_shape=array" + ).json[0]["line"] + assert re.match(expected_re, post_actual) + + +@pytest.mark.parametrize("use_csrf", [True, False]) +@pytest.mark.parametrize("return_json", [True, False]) +def test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_json): + update_query( + magic_parameters_client, + "runme_post", + sql="insert into logs (line) values (:_header_host)", + ) + qs = "" + if return_json: + qs = "?_json=1" + response = magic_parameters_client.post( + f"/data/runme_post{qs}", + {}, + csrftoken_from=use_csrf or None, + ) + if return_json: + assert response.status == 200 + assert response.json["ok"], response.json + else: + assert response.status == 302 + messages = magic_parameters_client.ds.unsign( + response.cookies["ds_messages"], "messages" + ) + assert [["Query executed, 1 row affected", 1]] == messages + post_actual = magic_parameters_client.get( + "/data/logs.json?_sort_desc=rowid&_shape=array" + ).json[0]["line"] + assert post_actual == "localhost" + + +def test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client): + response = magic_parameters_client.get( + "/data/-/query.json?sql=select+:_header_host&_shape=array" + ) + assert 400 == response.status + assert response.json["error"].startswith("You did not supply a value for binding") + + +def test_stored_write_custom_template(stored_write_client): + response = stored_write_client.get("/data/update_name") + assert response.status == 200 + assert "!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!" in response.text + assert ( + "" + in response.text + ) + # And test for link rel=alternate while we're here: + assert ( + '' + in response.text + ) + assert ( + response.headers["link"] + == '; rel="alternate"; type="application/json+datasette"' + ) + + +def test_stored_write_query_disabled_for_immutable_database( + stored_write_immutable_client, +): + response = stored_write_immutable_client.get("/fixtures/add") + assert response.status == 200 + assert ( + "This query cannot be executed because the database is immutable." + in response.text + ) + assert '' in response.text + # Submitting form should get a forbidden error + response = stored_write_immutable_client.post( + "/fixtures/add", + {"text": "text"}, + csrftoken_from=True, + ) + assert response.status == 403 + assert "Database is immutable" in response.text From ca4907ab6b01f5ba5fa241534be7c9557f269050 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:30:36 -0700 Subject: [PATCH 142/156] Make _save_queries_from_config a private method --- datasette/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 52d100cc..6b66a53b 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -598,7 +598,7 @@ class Datasette: # TODO(alex) is metadata.json was loaded in, and --internal is not memory, then log # a warning to user that they should delete their metadata.json file - async def apply_queries_config(self): + async def _save_queries_from_config(self): # Apply configured query entries from datasette.yaml to the internal table. await self.get_internal_database().execute_write( "DELETE FROM queries WHERE source = 'config'" @@ -788,7 +788,7 @@ class Datasette: await await_me_maybe(hook) # Ensure internal tables and metadata are populated before startup hooks await self._refresh_schemas() - await self.apply_queries_config() + await self._save_queries_from_config() # Load column_types from config into internal DB await self._apply_column_types_config() for hook in pm.hook.startup(datasette=self): From e89ffa0e0634a306be37bdd4894057329fee54a2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:37:21 -0700 Subject: [PATCH 143/156] Fixed broken test caused by apply_queries_config() rename --- tests/test_permissions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 0e38c876..e5e75432 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -937,7 +937,7 @@ async def test_permissions_in_config( updated_config = copy.deepcopy(previous_config) updated_config.update(config) perms_ds.config = updated_config - await perms_ds.apply_queries_config() + await perms_ds._save_queries_from_config() try: # Convert old-style resource to Resource object from datasette.resources import DatabaseResource, QueryResource, TableResource @@ -964,7 +964,7 @@ async def test_permissions_in_config( assert result == expected_result finally: perms_ds.config = previous_config - await perms_ds.apply_queries_config() + await perms_ds._save_queries_from_config() @pytest.mark.asyncio From 1bcd99df90d09049616fae411c2b6cddb6a82c7a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:42:36 -0700 Subject: [PATCH 144/156] Refactor code from datasette.app into datasette.stored_queries The datasette/app.py file had grown a lot in this branch. --- datasette/app.py | 537 ++++++----------------------------- datasette/stored_queries.py | 544 ++++++++++++++++++++++++++++++++++++ 2 files changed, 634 insertions(+), 447 deletions(-) create mode 100644 datasette/stored_queries.py diff --git a/datasette/app.py b/datasette/app.py index 6b66a53b..dd54446a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -42,6 +42,7 @@ from jinja2.exceptions import TemplateNotFound from .events import Event from .column_types import SQLiteType +from . import stored_queries from .views import Context from .views.database import ( database_download, @@ -108,7 +109,6 @@ from .utils import ( module_from_path, move_plugins_and_allow, move_table_config, - named_parameters, parse_metadata, resolve_env_secrets, resolve_routes, @@ -284,17 +284,6 @@ DEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS} FAVICON_PATH = app_root / "datasette" / "static" / "favicon.png" DEFAULT_NOT_SET = object() -UNCHANGED = object() - -QUERY_OPTION_FIELDS = ( - "hide_sql", - "fragment", - "on_success_message", - "on_success_message_sql", - "on_success_redirect", - "on_error_message", - "on_error_redirect", -) ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) @@ -599,34 +588,7 @@ class Datasette: # a warning to user that they should delete their metadata.json file async def _save_queries_from_config(self): - # Apply configured query entries from datasette.yaml to the internal table. - await self.get_internal_database().execute_write( - "DELETE FROM queries WHERE source = 'config'" - ) - for dbname, db_config in ((self.config or {}).get("databases") or {}).items(): - for query_name, query_config in (db_config.get("queries") or {}).items(): - if not isinstance(query_config, dict): - query_config = {"sql": query_config} - await self.add_query( - dbname, - query_name, - query_config["sql"], - title=query_config.get("title"), - description=query_config.get("description"), - description_html=query_config.get("description_html"), - hide_sql=bool(query_config.get("hide_sql")), - fragment=query_config.get("fragment"), - parameters=query_config.get("params"), - is_write=bool(query_config.get("write")), - is_private=bool(query_config.get("is_private")), - is_trusted=bool(query_config.get("is_trusted", True)), - source="config", - on_success_message=query_config.get("on_success_message"), - on_success_message_sql=query_config.get("on_success_message_sql"), - on_success_redirect=query_config.get("on_success_redirect"), - on_error_message=query_config.get("on_error_message"), - on_error_redirect=query_config.get("on_error_redirect"), - ) + await stored_queries.save_queries_from_config(self) def get_jinja_environment(self, request: Request = None) -> Environment: environment = self._jinja_env @@ -1067,46 +1029,11 @@ class Datasette: @staticmethod def _query_row_to_dict(row): - if row is None: - return None - parameters = json.loads(row["parameters"] or "[]") - options = json.loads(row["options"] or "{}") - is_write = bool(row["is_write"]) - return { - "database": row["database_name"], - "name": row["name"], - "sql": row["sql"], - "title": row["title"], - "description": row["description"], - "description_html": row["description_html"], - "hide_sql": bool(options.get("hide_sql")), - "fragment": options.get("fragment"), - "params": parameters, - "parameters": parameters, - "is_write": is_write, - "write": is_write, - "is_private": bool(row["is_private"]), - "is_trusted": bool(row["is_trusted"]), - "source": row["source"], - "owner_id": row["owner_id"], - "on_success_message": options.get("on_success_message"), - "on_success_message_sql": options.get("on_success_message_sql"), - "on_success_redirect": options.get("on_success_redirect"), - "on_error_message": options.get("on_error_message"), - "on_error_redirect": options.get("on_error_redirect"), - } + return stored_queries.query_row_to_dict(row) @staticmethod def _query_options_json(options): - options_dict = {} - for field in QUERY_OPTION_FIELDS: - value = options.get(field) - if field == "hide_sql": - if value: - options_dict[field] = True - elif value is not None: - options_dict[field] = value - return json.dumps(options_dict, sort_keys=True) + return stored_queries.query_options_json(options) async def add_query( self, @@ -1132,57 +1059,28 @@ class Datasette: on_error_redirect=None, replace=True, ): - parameters_json = json.dumps(list(parameters or [])) - options_json = self._query_options_json( - { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - ) - sql_statement = """ - INSERT INTO queries ( - database_name, name, sql, title, description, description_html, - options, parameters, is_write, is_private, is_trusted, source, owner_id - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """ - if replace: - sql_statement += """ - ON CONFLICT(database_name, name) DO UPDATE SET - sql = excluded.sql, - title = excluded.title, - description = excluded.description, - description_html = excluded.description_html, - options = excluded.options, - parameters = excluded.parameters, - is_write = excluded.is_write, - is_private = excluded.is_private, - is_trusted = excluded.is_trusted, - source = excluded.source, - owner_id = excluded.owner_id, - updated_at = CURRENT_TIMESTAMP - """ - await self.get_internal_database().execute_write( - sql_statement, - [ - database, - name, - sql, - title, - description, - description_html, - options_json, - parameters_json, - int(bool(is_write)), - int(bool(is_private)), - int(bool(is_trusted)), - source, - owner_id, - ], + return await stored_queries.add_query( + self, + database, + name, + sql, + title=title, + description=description, + description_html=description_html, + hide_sql=hide_sql, + fragment=fragment, + parameters=parameters, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + on_success_message=on_success_message, + on_success_message_sql=on_success_message_sql, + on_success_redirect=on_success_redirect, + on_error_message=on_error_message, + on_error_redirect=on_error_redirect, + replace=replace, ) async def update_query( @@ -1190,113 +1088,52 @@ class Datasette: database, name, *, - sql=UNCHANGED, - title=UNCHANGED, - description=UNCHANGED, - description_html=UNCHANGED, - hide_sql=UNCHANGED, - fragment=UNCHANGED, - parameters=UNCHANGED, - is_write=UNCHANGED, - is_private=UNCHANGED, - is_trusted=UNCHANGED, - source=UNCHANGED, - owner_id=UNCHANGED, - on_success_message=UNCHANGED, - on_success_message_sql=UNCHANGED, - on_success_redirect=UNCHANGED, - on_error_message=UNCHANGED, - on_error_redirect=UNCHANGED, + sql=stored_queries.UNCHANGED, + title=stored_queries.UNCHANGED, + description=stored_queries.UNCHANGED, + description_html=stored_queries.UNCHANGED, + hide_sql=stored_queries.UNCHANGED, + fragment=stored_queries.UNCHANGED, + parameters=stored_queries.UNCHANGED, + is_write=stored_queries.UNCHANGED, + is_private=stored_queries.UNCHANGED, + is_trusted=stored_queries.UNCHANGED, + source=stored_queries.UNCHANGED, + owner_id=stored_queries.UNCHANGED, + on_success_message=stored_queries.UNCHANGED, + on_success_message_sql=stored_queries.UNCHANGED, + on_success_redirect=stored_queries.UNCHANGED, + on_error_message=stored_queries.UNCHANGED, + on_error_redirect=stored_queries.UNCHANGED, ): - fields = { - "sql": sql, - "title": title, - "description": description, - "description_html": description_html, - "parameters": parameters, - "is_write": is_write, - "is_private": is_private, - "is_trusted": is_trusted, - "source": source, - "owner_id": owner_id, - } - option_fields = { - "hide_sql": hide_sql, - "fragment": fragment, - "on_success_message": on_success_message, - "on_success_message_sql": on_success_message_sql, - "on_success_redirect": on_success_redirect, - "on_error_message": on_error_message, - "on_error_redirect": on_error_redirect, - } - updates = [] - params = [] - for field, value in fields.items(): - if value is UNCHANGED: - continue - if field in {"is_write", "is_private", "is_trusted"}: - value = int(bool(value)) - elif field == "parameters": - value = json.dumps(list(value or [])) - updates.append(f"{field} = ?") - params.append(value) - changed_options = { - field: value - for field, value in option_fields.items() - if value is not UNCHANGED - } - if changed_options: - rows = await self.get_internal_database().execute( - """ - SELECT options FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - row = rows.first() - options = json.loads(row["options"] or "{}") if row is not None else {} - for field, value in changed_options.items(): - if field == "hide_sql": - if value: - options[field] = True - else: - options.pop(field, None) - elif value is None: - options.pop(field, None) - else: - options[field] = value - updates.append("options = ?") - params.append(json.dumps(options, sort_keys=True)) - if not updates: - return - updates.append("updated_at = CURRENT_TIMESTAMP") - params.extend([database, name]) - await self.get_internal_database().execute_write( - """ - UPDATE queries - SET {} - WHERE database_name = ? AND name = ? - """.format(", ".join(updates)), - params, + return await stored_queries.update_query( + self, + database, + name, + sql=sql, + title=title, + description=description, + description_html=description_html, + hide_sql=hide_sql, + fragment=fragment, + parameters=parameters, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + on_success_message=on_success_message, + on_success_message_sql=on_success_message_sql, + on_success_redirect=on_success_redirect, + on_error_message=on_error_message, + on_error_redirect=on_error_redirect, ) async def remove_query(self, database, name, source=None): - sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" - params = [database, name] - if source is not None: - sql += " AND source = ?" - params.append(source) - await self.get_internal_database().execute_write(sql, params) + return await stored_queries.remove_query(self, database, name, source=source) async def get_query(self, database, name): - rows = await self.get_internal_database().execute( - """ - SELECT * FROM queries - WHERE database_name = ? AND name = ? - """, - [database, name], - ) - return self._query_row_to_dict(rows.first()) + return await stored_queries.get_query(self, database, name) async def count_queries( self, @@ -1310,62 +1147,17 @@ class Datasette: source=None, owner_id=None, ): - allowed_sql, allowed_params = await self.allowed_resources_sql( - action="view-query", + return await stored_queries.count_queries( + self, + database, actor=actor, - parent=database, + q=q, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, ) - params = dict(allowed_params) - where_clauses = [] - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - row = ( - await self.get_internal_database().execute( - """ - SELECT count(*) AS count - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - """.format( - allowed_sql=allowed_sql, - where=" AND ".join(where_clauses) or "1 = 1", - ), - params, - ) - ).first() - return row["count"] async def list_queries( self, @@ -1382,176 +1174,27 @@ class Datasette: owner_id=None, include_private=False, ): - limit = min(max(1, int(limit)), 1000) - allowed_sql, allowed_params = await self.allowed_resources_sql( - action="view-query", + return await stored_queries.list_queries( + self, + database, actor=actor, - parent=database, - include_is_private=include_private, + limit=limit, + cursor=cursor, + q=q, + is_write=is_write, + is_private=is_private, + is_trusted=is_trusted, + source=source, + owner_id=owner_id, + include_private=include_private, ) - params = dict(allowed_params) - params.update({"limit": limit + 1}) - sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" - where_clauses = [] - order_by = "q.database_name, sort_key, q.name" - if database is not None: - params["query_database"] = database - where_clauses.append("q.database_name = :query_database") - order_by = "sort_key, q.name" - - if cursor: - try: - components = urlsafe_components(cursor) - except ValueError: - components = [] - if database is None and len(components) == 3: - where_clauses.append(""" - ( - q.database_name > :cursor_database - OR ( - q.database_name = :cursor_database - AND ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_database"] = components[0] - params["cursor_sort_key"] = components[1] - params["cursor_name"] = components[2] - elif database is not None and len(components) == 2: - where_clauses.append(""" - ( - {sort_key_sql} > :cursor_sort_key - OR ( - {sort_key_sql} = :cursor_sort_key - AND q.name > :cursor_name - ) - ) - """.format(sort_key_sql=sort_key_sql)) - params["cursor_sort_key"] = components[0] - params["cursor_name"] = components[1] - - if q: - where_clauses.append(""" - ( - q.name LIKE :query_search - OR q.title LIKE :query_search - OR q.description LIKE :query_search - OR q.sql LIKE :query_search - ) - """) - params["query_search"] = "%{}%".format(q) - if is_write is not None: - where_clauses.append("q.is_write = :query_is_write") - params["query_is_write"] = int(bool(is_write)) - if is_private is not None: - where_clauses.append("q.is_private = :query_is_private") - params["query_is_private"] = int(bool(is_private)) - if is_trusted is not None: - where_clauses.append("q.is_trusted = :query_is_trusted") - params["query_is_trusted"] = int(bool(is_trusted)) - if source is not None: - where_clauses.append("q.source = :query_source") - params["query_source"] = source - if owner_id is not None: - where_clauses.append("q.owner_id = :query_owner_id") - params["query_owner_id"] = owner_id - - private_select = ", allowed.is_private AS private" if include_private else "" - rows = list( - ( - await self.get_internal_database().execute( - """ - SELECT q.*, {sort_key_sql} AS sort_key{private_select} - FROM queries q - JOIN ( - {allowed_sql} - ) allowed - ON allowed.parent = q.database_name - AND allowed.child = q.name - WHERE {where} - ORDER BY {order_by} - LIMIT :limit - """.format( - allowed_sql=allowed_sql, - private_select=private_select, - sort_key_sql=sort_key_sql, - where=" AND ".join(where_clauses) or "1 = 1", - order_by=order_by, - ), - params, - ) - ).rows - ) - has_more = len(rows) > limit - if has_more: - rows = rows[:limit] - - queries = [] - for row in rows: - query = self._query_row_to_dict(row) - if include_private: - query["private"] = bool(row["private"]) - queries.append(query) - - next_token = None - if has_more and rows: - last_row = rows[-1] - if database is None: - next_token = "{},{},{}".format( - tilde_encode(last_row["database_name"]), - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - else: - next_token = "{},{}".format( - tilde_encode(last_row["sort_key"]), - tilde_encode(last_row["name"]), - ) - return { - "queries": queries, - "next": next_token, - "has_more": has_more, - "limit": limit, - } async def ensure_query_write_permissions( self, database, sql, *, actor=None, params=None, analysis=None ): - write_actions = { - "insert": "insert-row", - "update": "update-row", - "delete": "delete-row", - } - db = self.get_database(database) - if analysis is None: - if params is None: - params = {name: "" for name in named_parameters(sql)} - try: - analysis = await db.analyze_sql(sql, params) - except sqlite3.DatabaseError as ex: - raise Forbidden(f"Could not analyze query: {ex}") from ex - - for access in analysis.table_accesses: - action = write_actions.get(access.operation) - if action is None: - continue - if access.database != database: - raise Forbidden("Writable queries may not write to attached databases") - if not await self.allowed( - action=action, - resource=TableResource(database=access.database, table=access.table), - actor=actor, - ): - raise Forbidden( - f"Permission denied: need {action} on {access.database}/{access.table}" - ) - return analysis + return await stored_queries.ensure_query_write_permissions( + self, database, sql, actor=actor, params=params, analysis=analysis + ) # Column types API diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py new file mode 100644 index 00000000..e88902e7 --- /dev/null +++ b/datasette/stored_queries.py @@ -0,0 +1,544 @@ +from __future__ import annotations + +import json + +from .resources import TableResource +from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components +from .utils.asgi import Forbidden + + +UNCHANGED = object() + +QUERY_OPTION_FIELDS = ( + "hide_sql", + "fragment", + "on_success_message", + "on_success_message_sql", + "on_success_redirect", + "on_error_message", + "on_error_redirect", +) + + +async def save_queries_from_config(datasette): + # Apply configured query entries from datasette.yaml to the internal table. + await datasette.get_internal_database().execute_write( + "DELETE FROM queries WHERE source = 'config'" + ) + for dbname, db_config in ((datasette.config or {}).get("databases") or {}).items(): + for query_name, query_config in (db_config.get("queries") or {}).items(): + if not isinstance(query_config, dict): + query_config = {"sql": query_config} + await datasette.add_query( + dbname, + query_name, + query_config["sql"], + title=query_config.get("title"), + description=query_config.get("description"), + description_html=query_config.get("description_html"), + hide_sql=bool(query_config.get("hide_sql")), + fragment=query_config.get("fragment"), + parameters=query_config.get("params"), + is_write=bool(query_config.get("write")), + is_private=bool(query_config.get("is_private")), + is_trusted=bool(query_config.get("is_trusted", True)), + source="config", + on_success_message=query_config.get("on_success_message"), + on_success_message_sql=query_config.get("on_success_message_sql"), + on_success_redirect=query_config.get("on_success_redirect"), + on_error_message=query_config.get("on_error_message"), + on_error_redirect=query_config.get("on_error_redirect"), + ) + + +def query_row_to_dict(row): + if row is None: + return None + parameters = json.loads(row["parameters"] or "[]") + options = json.loads(row["options"] or "{}") + is_write = bool(row["is_write"]) + return { + "database": row["database_name"], + "name": row["name"], + "sql": row["sql"], + "title": row["title"], + "description": row["description"], + "description_html": row["description_html"], + "hide_sql": bool(options.get("hide_sql")), + "fragment": options.get("fragment"), + "params": parameters, + "parameters": parameters, + "is_write": is_write, + "write": is_write, + "is_private": bool(row["is_private"]), + "is_trusted": bool(row["is_trusted"]), + "source": row["source"], + "owner_id": row["owner_id"], + "on_success_message": options.get("on_success_message"), + "on_success_message_sql": options.get("on_success_message_sql"), + "on_success_redirect": options.get("on_success_redirect"), + "on_error_message": options.get("on_error_message"), + "on_error_redirect": options.get("on_error_redirect"), + } + + +def query_options_json(options): + options_dict = {} + for field in QUERY_OPTION_FIELDS: + value = options.get(field) + if field == "hide_sql": + if value: + options_dict[field] = True + elif value is not None: + options_dict[field] = value + return json.dumps(options_dict, sort_keys=True) + + +async def add_query( + datasette, + database, + name, + sql, + *, + title=None, + description=None, + description_html=None, + hide_sql=False, + fragment=None, + parameters=None, + is_write=False, + is_private=False, + is_trusted=False, + source="plugin", + owner_id=None, + on_success_message=None, + on_success_message_sql=None, + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + replace=True, +): + parameters_json = json.dumps(list(parameters or [])) + options_json = query_options_json( + { + "hide_sql": hide_sql, + "fragment": fragment, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + ) + sql_statement = """ + INSERT INTO queries ( + database_name, name, sql, title, description, description_html, + options, parameters, is_write, is_private, is_trusted, source, owner_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + if replace: + sql_statement += """ + ON CONFLICT(database_name, name) DO UPDATE SET + sql = excluded.sql, + title = excluded.title, + description = excluded.description, + description_html = excluded.description_html, + options = excluded.options, + parameters = excluded.parameters, + is_write = excluded.is_write, + is_private = excluded.is_private, + is_trusted = excluded.is_trusted, + source = excluded.source, + owner_id = excluded.owner_id, + updated_at = CURRENT_TIMESTAMP + """ + await datasette.get_internal_database().execute_write( + sql_statement, + [ + database, + name, + sql, + title, + description, + description_html, + options_json, + parameters_json, + int(bool(is_write)), + int(bool(is_private)), + int(bool(is_trusted)), + source, + owner_id, + ], + ) + + +async def update_query( + datasette, + database, + name, + *, + sql=UNCHANGED, + title=UNCHANGED, + description=UNCHANGED, + description_html=UNCHANGED, + hide_sql=UNCHANGED, + fragment=UNCHANGED, + parameters=UNCHANGED, + is_write=UNCHANGED, + is_private=UNCHANGED, + is_trusted=UNCHANGED, + source=UNCHANGED, + owner_id=UNCHANGED, + on_success_message=UNCHANGED, + on_success_message_sql=UNCHANGED, + on_success_redirect=UNCHANGED, + on_error_message=UNCHANGED, + on_error_redirect=UNCHANGED, +): + fields = { + "sql": sql, + "title": title, + "description": description, + "description_html": description_html, + "parameters": parameters, + "is_write": is_write, + "is_private": is_private, + "is_trusted": is_trusted, + "source": source, + "owner_id": owner_id, + } + option_fields = { + "hide_sql": hide_sql, + "fragment": fragment, + "on_success_message": on_success_message, + "on_success_message_sql": on_success_message_sql, + "on_success_redirect": on_success_redirect, + "on_error_message": on_error_message, + "on_error_redirect": on_error_redirect, + } + updates = [] + params = [] + for field, value in fields.items(): + if value is UNCHANGED: + continue + if field in {"is_write", "is_private", "is_trusted"}: + value = int(bool(value)) + elif field == "parameters": + value = json.dumps(list(value or [])) + updates.append(f"{field} = ?") + params.append(value) + changed_options = { + field: value for field, value in option_fields.items() if value is not UNCHANGED + } + if changed_options: + rows = await datasette.get_internal_database().execute( + """ + SELECT options FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + row = rows.first() + options = json.loads(row["options"] or "{}") if row is not None else {} + for field, value in changed_options.items(): + if field == "hide_sql": + if value: + options[field] = True + else: + options.pop(field, None) + elif value is None: + options.pop(field, None) + else: + options[field] = value + updates.append("options = ?") + params.append(json.dumps(options, sort_keys=True)) + if not updates: + return + updates.append("updated_at = CURRENT_TIMESTAMP") + params.extend([database, name]) + await datasette.get_internal_database().execute_write( + """ + UPDATE queries + SET {} + WHERE database_name = ? AND name = ? + """.format(", ".join(updates)), + params, + ) + + +async def remove_query(datasette, database, name, source=None): + sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" + params = [database, name] + if source is not None: + sql += " AND source = ?" + params.append(source) + await datasette.get_internal_database().execute_write(sql, params) + + +async def get_query(datasette, database, name): + rows = await datasette.get_internal_database().execute( + """ + SELECT * FROM queries + WHERE database_name = ? AND name = ? + """, + [database, name], + ) + return query_row_to_dict(rows.first()) + + +async def count_queries( + datasette, + database=None, + *, + actor=None, + q=None, + is_write=None, + is_private=None, + is_trusted=None, + source=None, + owner_id=None, +): + allowed_sql, allowed_params = await datasette.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + ) + params = dict(allowed_params) + where_clauses = [] + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + row = ( + await datasette.get_internal_database().execute( + """ + SELECT count(*) AS count + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + """.format( + allowed_sql=allowed_sql, + where=" AND ".join(where_clauses) or "1 = 1", + ), + params, + ) + ).first() + return row["count"] + + +async def list_queries( + datasette, + database=None, + *, + actor=None, + limit=50, + cursor=None, + q=None, + is_write=None, + is_private=None, + is_trusted=None, + source=None, + owner_id=None, + include_private=False, +): + limit = min(max(1, int(limit)), 1000) + allowed_sql, allowed_params = await datasette.allowed_resources_sql( + action="view-query", + actor=actor, + parent=database, + include_is_private=include_private, + ) + params = dict(allowed_params) + params.update({"limit": limit + 1}) + sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))" + where_clauses = [] + order_by = "q.database_name, sort_key, q.name" + if database is not None: + params["query_database"] = database + where_clauses.append("q.database_name = :query_database") + order_by = "sort_key, q.name" + + if cursor: + try: + components = urlsafe_components(cursor) + except ValueError: + components = [] + if database is None and len(components) == 3: + where_clauses.append(""" + ( + q.database_name > :cursor_database + OR ( + q.database_name = :cursor_database + AND ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_database"] = components[0] + params["cursor_sort_key"] = components[1] + params["cursor_name"] = components[2] + elif database is not None and len(components) == 2: + where_clauses.append(""" + ( + {sort_key_sql} > :cursor_sort_key + OR ( + {sort_key_sql} = :cursor_sort_key + AND q.name > :cursor_name + ) + ) + """.format(sort_key_sql=sort_key_sql)) + params["cursor_sort_key"] = components[0] + params["cursor_name"] = components[1] + + if q: + where_clauses.append(""" + ( + q.name LIKE :query_search + OR q.title LIKE :query_search + OR q.description LIKE :query_search + OR q.sql LIKE :query_search + ) + """) + params["query_search"] = "%{}%".format(q) + if is_write is not None: + where_clauses.append("q.is_write = :query_is_write") + params["query_is_write"] = int(bool(is_write)) + if is_private is not None: + where_clauses.append("q.is_private = :query_is_private") + params["query_is_private"] = int(bool(is_private)) + if is_trusted is not None: + where_clauses.append("q.is_trusted = :query_is_trusted") + params["query_is_trusted"] = int(bool(is_trusted)) + if source is not None: + where_clauses.append("q.source = :query_source") + params["query_source"] = source + if owner_id is not None: + where_clauses.append("q.owner_id = :query_owner_id") + params["query_owner_id"] = owner_id + + private_select = ", allowed.is_private AS private" if include_private else "" + rows = list( + ( + await datasette.get_internal_database().execute( + """ + SELECT q.*, {sort_key_sql} AS sort_key{private_select} + FROM queries q + JOIN ( + {allowed_sql} + ) allowed + ON allowed.parent = q.database_name + AND allowed.child = q.name + WHERE {where} + ORDER BY {order_by} + LIMIT :limit + """.format( + allowed_sql=allowed_sql, + private_select=private_select, + sort_key_sql=sort_key_sql, + where=" AND ".join(where_clauses) or "1 = 1", + order_by=order_by, + ), + params, + ) + ).rows + ) + has_more = len(rows) > limit + if has_more: + rows = rows[:limit] + + queries = [] + for row in rows: + query = query_row_to_dict(row) + if include_private: + query["private"] = bool(row["private"]) + queries.append(query) + + next_token = None + if has_more and rows: + last_row = rows[-1] + if database is None: + next_token = "{},{},{}".format( + tilde_encode(last_row["database_name"]), + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + else: + next_token = "{},{}".format( + tilde_encode(last_row["sort_key"]), + tilde_encode(last_row["name"]), + ) + return { + "queries": queries, + "next": next_token, + "has_more": has_more, + "limit": limit, + } + + +async def ensure_query_write_permissions( + datasette, database, sql, *, actor=None, params=None, analysis=None +): + write_actions = { + "insert": "insert-row", + "update": "update-row", + "delete": "delete-row", + } + db = datasette.get_database(database) + if analysis is None: + if params is None: + params = {name: "" for name in named_parameters(sql)} + try: + analysis = await db.analyze_sql(sql, params) + except sqlite3.DatabaseError as ex: + raise Forbidden(f"Could not analyze query: {ex}") from ex + + for access in analysis.table_accesses: + action = write_actions.get(access.operation) + if action is None: + continue + if access.database != database: + raise Forbidden("Writable queries may not write to attached databases") + if not await datasette.allowed( + action=action, + resource=TableResource(database=access.database, table=access.table), + actor=actor, + ): + raise Forbidden( + f"Permission denied: need {action} on {access.database}/{access.table}" + ) + return analysis From 58e2e3a8ab9adfeda2686be5213eaf3f4118da33 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:43:34 -0700 Subject: [PATCH 145/156] Ran cog --- docs/authentication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/authentication.rst b/docs/authentication.rst index cec47f97..f8b1dc49 100644 --- a/docs/authentication.rst +++ b/docs/authentication.rst @@ -496,7 +496,7 @@ Here's how to restrict access to your entire Datasette instance to just the ``"i title: My private Datasette instance allow: id: root - + .. tab:: datasette.json From c3ceabae03c2c9cf78f63f7c12c8d90301097cb4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:51:40 -0700 Subject: [PATCH 146/156] Ran Black --- datasette/stored_queries.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index e88902e7..5b005c07 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -6,7 +6,6 @@ from .resources import TableResource from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components from .utils.asgi import Forbidden - UNCHANGED = object() QUERY_OPTION_FIELDS = ( From d6de8e752083a014bdd7ffb701dc08519da82554 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 15:52:16 -0700 Subject: [PATCH 147/156] Link to save query from /-/execute-write --- datasette/templates/execute_write.html | 29 +++++++++++++++++++++++++- datasette/views/database.py | 22 +++++++++++++++++++ tests/test_queries.py | 17 +++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/datasette/templates/execute_write.html b/datasette/templates/execute_write.html index 7a627a7a..6b626f8d 100644 --- a/datasette/templates/execute_write.html +++ b/datasette/templates/execute_write.html @@ -119,7 +119,10 @@ {% endif %}
      -

      +

      + + {% if save_query_base_url %}Save this query{% endif %} +

      ", + "on_success_message_sql": "select 'secret'", + } + }, + ) + form_response = await ds.client.post( + "/data/-/queries/store", + actor={"id": "root"}, + data={ + "name": "unsafe_form", + "sql": "select 1", + "description_html": "", + }, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == [ + "Invalid keys: description_html, on_success_message_sql" + ] + assert form_response.status_code == 400 + assert "Invalid keys: description_html" in form_response.text + assert await ds.get_query("data", "unsafe") is None + assert await ds.get_query("data", "unsafe_form") is None + + @pytest.mark.asyncio async def test_query_store_api_creates_writable_query(): ds = Datasette(memory=True, default_deny=True) @@ -959,6 +1000,42 @@ async def test_query_update_and_delete_api(): assert await ds.get_query("data", "editable") is None +@pytest.mark.asyncio +async def test_query_update_api_rejects_config_only_fields(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("query_update_config_only_fields", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "editable", + "insert into dogs (name) values (:name)", + is_write=True, + source="user", + owner_id="root", + ) + + response = await ds.client.post( + "/data/editable/-/update", + actor={"id": "root"}, + json={ + "update": { + "description_html": "", + "on_success_message_sql": "select 'secret'", + } + }, + ) + + assert response.status_code == 400 + assert response.json()["errors"] == [ + "Invalid keys: description_html, on_success_message_sql" + ] + query = await ds.get_query("data", "editable") + assert query["description_html"] is None + assert query["on_success_message_sql"] is None + + @pytest.mark.asyncio async def test_query_update_api_rejects_trusted_queries_but_internal_update_allowed(): ds = Datasette( From b1289a73f9869e83a433a088c2a6c48285e67f2d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 26 May 2026 16:51:00 -0700 Subject: [PATCH 156/156] stored_queries.StoredQuery dataclass --- datasette/app.py | 102 ++++++------ datasette/stored_queries.py | 258 ++++++++++++++++++++---------- datasette/views/database.py | 56 +++---- datasette/views/query_helpers.py | 19 +-- datasette/views/stored_queries.py | 37 +++-- docs/internals.rst | 14 +- tests/test_queries.py | 128 +++++++-------- 7 files changed, 357 insertions(+), 257 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 96683895..56b89789 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -1029,8 +1029,8 @@ class Datasette: ) @staticmethod - def _query_row_to_dict(row): - return stored_queries.query_row_to_dict(row) + def _query_row_to_stored_query(row) -> stored_queries.StoredQuery | None: + return stored_queries.query_row_to_stored_query(row) @staticmethod def _query_options_json(options): @@ -1038,28 +1038,28 @@ class Datasette: async def add_query( self, - database, - name, - sql, + database: str, + name: str, + sql: str, *, - title=None, - description=None, - description_html=None, - hide_sql=False, - fragment=None, - parameters=None, - is_write=False, - is_private=False, - is_trusted=False, - source="plugin", - owner_id=None, - on_success_message=None, - on_success_message_sql=None, - on_success_redirect=None, - on_error_message=None, - on_error_redirect=None, - replace=True, - ): + title: str | None = None, + description: str | None = None, + description_html: str | None = None, + hide_sql: bool = False, + fragment: str | None = None, + parameters: Iterable[str] | None = None, + is_write: bool = False, + is_private: bool = False, + is_trusted: bool = False, + source: str = "plugin", + owner_id: str | None = None, + on_success_message: str | None = None, + on_success_message_sql: str | None = None, + on_success_redirect: str | None = None, + on_error_message: str | None = None, + on_error_redirect: str | None = None, + replace: bool = True, + ) -> None: return await stored_queries.add_query( self, database, @@ -1086,8 +1086,8 @@ class Datasette: async def update_query( self, - database, - name, + database: str, + name: str, *, sql=stored_queries.UNCHANGED, title=stored_queries.UNCHANGED, @@ -1106,7 +1106,7 @@ class Datasette: on_success_redirect=stored_queries.UNCHANGED, on_error_message=stored_queries.UNCHANGED, on_error_redirect=stored_queries.UNCHANGED, - ): + ) -> None: return await stored_queries.update_query( self, database, @@ -1130,24 +1130,28 @@ class Datasette: on_error_redirect=on_error_redirect, ) - async def remove_query(self, database, name, source=None): + async def remove_query( + self, database: str, name: str, source: str | None = None + ) -> None: return await stored_queries.remove_query(self, database, name, source=source) - async def get_query(self, database, name): + async def get_query( + self, database: str, name: str + ) -> stored_queries.StoredQuery | None: return await stored_queries.get_query(self, database, name) async def count_queries( self, - database=None, + database: str | None = None, *, - actor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, - ): + actor: dict[str, Any] | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + ) -> int: return await stored_queries.count_queries( self, database, @@ -1162,19 +1166,19 @@ class Datasette: async def list_queries( self, - database=None, + database: str | None = None, *, - actor=None, - limit=50, - cursor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, - include_private=False, - ): + actor: dict[str, Any] | None = None, + limit: int = 50, + cursor: str | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + include_private: bool = False, + ) -> stored_queries.StoredQueryPage: return await stored_queries.list_queries( self, database, diff --git a/datasette/stored_queries.py b/datasette/stored_queries.py index a28b71bf..bcfdfdb4 100644 --- a/datasette/stored_queries.py +++ b/datasette/stored_queries.py @@ -1,6 +1,8 @@ from __future__ import annotations +from dataclasses import dataclass import json +from typing import Any, Iterable from .resources import TableResource from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components @@ -19,7 +21,76 @@ QUERY_OPTION_FIELDS = ( ) -async def save_queries_from_config(datasette): +@dataclass +class StoredQuery: + database: str + name: str + sql: str + title: str | None + description: str | None + description_html: str | None + hide_sql: bool + fragment: str | None + parameters: list[str] + is_write: bool + is_private: bool + is_trusted: bool + source: str + owner_id: str | None + on_success_message: str | None + on_success_message_sql: str | None + on_success_redirect: str | None + on_error_message: str | None + on_error_redirect: str | None + private: bool | None = None + + +@dataclass +class StoredQueryPage: + queries: list[StoredQuery] + next: str | None + has_more: bool + limit: int + + +def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]: + data = { + "database": query.database, + "name": query.name, + "sql": query.sql, + "title": query.title, + "description": query.description, + "description_html": query.description_html, + "hide_sql": query.hide_sql, + "fragment": query.fragment, + "params": list(query.parameters), + "parameters": list(query.parameters), + "is_write": query.is_write, + "is_private": query.is_private, + "is_trusted": query.is_trusted, + "source": query.source, + "owner_id": query.owner_id, + "on_success_message": query.on_success_message, + "on_success_message_sql": query.on_success_message_sql, + "on_success_redirect": query.on_success_redirect, + "on_error_message": query.on_error_message, + "on_error_redirect": query.on_error_redirect, + } + if query.private is not None: + data["private"] = query.private + return data + + +def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]: + return { + "queries": [stored_query_to_dict(query) for query in page.queries], + "next": page.next, + "has_more": page.has_more, + "limit": page.limit, + } + + +async def save_queries_from_config(datasette: Any) -> None: # Apply configured query entries from datasette.yaml to the internal table. await datasette.get_internal_database().execute_write( "DELETE FROM queries WHERE source = 'config'" @@ -50,36 +121,38 @@ async def save_queries_from_config(datasette): ) -def query_row_to_dict(row): +def query_row_to_stored_query( + row: Any, private: bool | None = None +) -> StoredQuery | None: if row is None: return None parameters = json.loads(row["parameters"] or "[]") options = json.loads(row["options"] or "{}") - return { - "database": row["database_name"], - "name": row["name"], - "sql": row["sql"], - "title": row["title"], - "description": row["description"], - "description_html": row["description_html"], - "hide_sql": bool(options.get("hide_sql")), - "fragment": options.get("fragment"), - "params": parameters, - "parameters": parameters, - "is_write": bool(row["is_write"]), - "is_private": bool(row["is_private"]), - "is_trusted": bool(row["is_trusted"]), - "source": row["source"], - "owner_id": row["owner_id"], - "on_success_message": options.get("on_success_message"), - "on_success_message_sql": options.get("on_success_message_sql"), - "on_success_redirect": options.get("on_success_redirect"), - "on_error_message": options.get("on_error_message"), - "on_error_redirect": options.get("on_error_redirect"), - } + return StoredQuery( + database=row["database_name"], + name=row["name"], + sql=row["sql"], + title=row["title"], + description=row["description"], + description_html=row["description_html"], + hide_sql=bool(options.get("hide_sql")), + fragment=options.get("fragment"), + parameters=parameters, + is_write=bool(row["is_write"]), + is_private=bool(row["is_private"]), + is_trusted=bool(row["is_trusted"]), + source=row["source"], + owner_id=row["owner_id"], + on_success_message=options.get("on_success_message"), + on_success_message_sql=options.get("on_success_message_sql"), + on_success_redirect=options.get("on_success_redirect"), + on_error_message=options.get("on_error_message"), + on_error_redirect=options.get("on_error_redirect"), + private=private, + ) -def query_options_json(options): +def query_options_json(options: dict[str, Any]) -> str: options_dict = {} for field in QUERY_OPTION_FIELDS: value = options.get(field) @@ -92,29 +165,29 @@ def query_options_json(options): async def add_query( - datasette, - database, - name, - sql, + datasette: Any, + database: str, + name: str, + sql: str, *, - title=None, - description=None, - description_html=None, - hide_sql=False, - fragment=None, - parameters=None, - is_write=False, - is_private=False, - is_trusted=False, - source="plugin", - owner_id=None, - on_success_message=None, - on_success_message_sql=None, - on_success_redirect=None, - on_error_message=None, - on_error_redirect=None, - replace=True, -): + title: str | None = None, + description: str | None = None, + description_html: str | None = None, + hide_sql: bool = False, + fragment: str | None = None, + parameters: Iterable[str] | None = None, + is_write: bool = False, + is_private: bool = False, + is_trusted: bool = False, + source: str = "plugin", + owner_id: str | None = None, + on_success_message: str | None = None, + on_success_message_sql: str | None = None, + on_success_redirect: str | None = None, + on_error_message: str | None = None, + on_error_redirect: str | None = None, + replace: bool = True, +) -> None: parameters_json = json.dumps(list(parameters or [])) options_json = query_options_json( { @@ -170,9 +243,9 @@ async def add_query( async def update_query( - datasette, - database, - name, + datasette: Any, + database: str, + name: str, *, sql=UNCHANGED, title=UNCHANGED, @@ -191,7 +264,7 @@ async def update_query( on_success_redirect=UNCHANGED, on_error_message=UNCHANGED, on_error_redirect=UNCHANGED, -): +) -> None: fields = { "sql": sql, "title": title, @@ -263,7 +336,9 @@ async def update_query( ) -async def remove_query(datasette, database, name, source=None): +async def remove_query( + datasette: Any, database: str, name: str, source: str | None = None +) -> None: sql = "DELETE FROM queries WHERE database_name = ? AND name = ?" params = [database, name] if source is not None: @@ -272,7 +347,7 @@ async def remove_query(datasette, database, name, source=None): await datasette.get_internal_database().execute_write(sql, params) -async def get_query(datasette, database, name): +async def get_query(datasette: Any, database: str, name: str) -> StoredQuery | None: rows = await datasette.get_internal_database().execute( """ SELECT * FROM queries @@ -280,21 +355,21 @@ async def get_query(datasette, database, name): """, [database, name], ) - return query_row_to_dict(rows.first()) + return query_row_to_stored_query(rows.first()) async def count_queries( - datasette, - database=None, + datasette: Any, + database: str | None = None, *, - actor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, -): + actor: dict[str, Any] | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, +) -> int: allowed_sql, allowed_params = await datasette.allowed_resources_sql( action="view-query", actor=actor, @@ -354,20 +429,20 @@ async def count_queries( async def list_queries( - datasette, - database=None, + datasette: Any, + database: str | None = None, *, - actor=None, - limit=50, - cursor=None, - q=None, - is_write=None, - is_private=None, - is_trusted=None, - source=None, - owner_id=None, - include_private=False, -): + actor: dict[str, Any] | None = None, + limit: int = 50, + cursor: str | None = None, + q: str | None = None, + is_write: bool | None = None, + is_private: bool | None = None, + is_trusted: bool | None = None, + source: str | None = None, + owner_id: str | None = None, + include_private: bool = False, +) -> StoredQueryPage: limit = min(max(1, int(limit)), 1000) allowed_sql, allowed_params = await datasette.allowed_resources_sql( action="view-query", @@ -480,9 +555,10 @@ async def list_queries( queries = [] for row in rows: - query = query_row_to_dict(row) - if include_private: - query["private"] = bool(row["private"]) + query = query_row_to_stored_query( + row, private=bool(row["private"]) if include_private else None + ) + assert query is not None queries.append(query) next_token = None @@ -499,17 +575,23 @@ async def list_queries( tilde_encode(last_row["sort_key"]), tilde_encode(last_row["name"]), ) - return { - "queries": queries, - "next": next_token, - "has_more": has_more, - "limit": limit, - } + return StoredQueryPage( + queries=queries, + next=next_token, + has_more=has_more, + limit=limit, + ) async def ensure_query_write_permissions( - datasette, database, sql, *, actor=None, params=None, analysis=None -): + datasette: Any, + database: str, + sql: str, + *, + actor: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + analysis: Any = None, +) -> Any: write_actions = { "insert": "insert-row", "update": "update-row", diff --git a/datasette/views/database.py b/datasette/views/database.py index 98ca989c..b558b002 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -13,6 +13,7 @@ import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import stored_query_to_dict from datasette.utils import ( add_cors_headers, await_me_maybe, @@ -99,8 +100,8 @@ class DatabaseView(View): limit=5, include_private=True, ) - stored_queries = queries_page["queries"] - queries_more = queries_page["has_more"] + stored_queries = queries_page.queries + queries_more = queries_page.has_more queries_count = ( await datasette.count_queries(database, actor=request.actor) if queries_more @@ -136,7 +137,7 @@ class DatabaseView(View): "tables": tables, "hidden_count": len([t for t in tables if t["hidden"]]), "views": sql_views, - "queries": stored_queries, + "queries": [stored_query_to_dict(query) for query in stored_queries], "queries_more": queries_more, "queries_count": queries_count, "allow_execute_sql": allow_execute_sql, @@ -447,7 +448,7 @@ class QueryView(View): if not await datasette.allowed( action="view-query", - resource=QueryResource(database=db.name, query=stored_query["name"]), + resource=QueryResource(database=db.name, query=stored_query.name), actor=request.actor, ): raise Forbidden("You do not have permission to view this query") @@ -480,20 +481,18 @@ class QueryView(View): or request.args.get("_json") or params.get("_json") ) - params_for_query = MagicParameters( - stored_query["sql"], params, request, datasette - ) + params_for_query = MagicParameters(stored_query.sql, params, request, datasette) await params_for_query.execute_params() ok = None redirect_url = None try: cursor = await db.execute_write( - stored_query["sql"], params_for_query, request=request + stored_query.sql, params_for_query, request=request ) # success message can come from on_success_message or on_success_message_sql message = None message_type = datasette.INFO - on_success_message_sql = stored_query.get("on_success_message_sql") + on_success_message_sql = stored_query.on_success_message_sql if on_success_message_sql: try: message_result = ( @@ -505,18 +504,19 @@ class QueryView(View): message = "Error running on_success_message_sql: {}".format(ex) message_type = datasette.ERROR if not message: - message = stored_query.get( - "on_success_message" - ) or "Query executed, {} row{} affected".format( - cursor.rowcount, "" if cursor.rowcount == 1 else "s" + message = ( + stored_query.on_success_message + or "Query executed, {} row{} affected".format( + cursor.rowcount, "" if cursor.rowcount == 1 else "s" + ) ) - redirect_url = stored_query.get("on_success_redirect") + redirect_url = stored_query.on_success_redirect ok = True except Exception as ex: - message = stored_query.get("on_error_message") or str(ex) + message = stored_query.on_error_message or str(ex) message_type = datasette.ERROR - redirect_url = stored_query.get("on_error_redirect") + redirect_url = stored_query.on_error_redirect ok = False if should_return_json: return Response.json( @@ -562,7 +562,7 @@ class QueryView(View): ) if stored_query is None: raise - stored_query_write = bool(stored_query.get("is_write")) + stored_query_write = stored_query.is_write private = False if stored_query: @@ -570,7 +570,7 @@ class QueryView(View): visible, private = await datasette.check_visibility( request.actor, action="view-query", - resource=QueryResource(database=database, query=stored_query["name"]), + resource=QueryResource(database=database, query=stored_query.name), ) if not visible: raise Forbidden("You do not have permission to view this query") @@ -591,14 +591,14 @@ class QueryView(View): sql = None if stored_query: - sql = stored_query["sql"] + sql = stored_query.sql elif "sql" in params: sql = params.pop("sql") # Extract any :named parameters named_parameters = [] - if stored_query and stored_query.get("params"): - named_parameters = stored_query["params"] + if stored_query and stored_query.parameters: + named_parameters = stored_query.parameters if not named_parameters and sql: named_parameters = derive_named_parameters(sql) named_parameter_values = { @@ -686,7 +686,7 @@ class QueryView(View): columns=columns, rows=rows, sql=sql, - query_name=stored_query["name"] if stored_query else None, + query_name=stored_query.name if stored_query else None, database=database, table=None, request=request, @@ -721,7 +721,7 @@ class QueryView(View): if stored_query: templates.insert( 0, - f"query-{to_css_class(database)}-{to_css_class(stored_query['name'])}.html", + f"query-{to_css_class(database)}-{to_css_class(stored_query.name)}.html", ) environment = datasette.get_jinja_environment(request) @@ -740,7 +740,7 @@ class QueryView(View): ) metadata = await datasette.get_database_metadata(database) if stored_query: - metadata = dict(stored_query) + metadata = stored_query_to_dict(stored_query) metadata.pop("source", None) renderers = {} @@ -775,7 +775,7 @@ class QueryView(View): ) show_hide_hidden = "" - if stored_query and stored_query.get("hide_sql"): + if stored_query and stored_query.hide_sql: if bool(params.get("_show_sql")): show_hide_link = path_with_removed_args(request, {"_show_sql"}) show_hide_text = "hide" @@ -843,7 +843,7 @@ class QueryView(View): datasette=datasette, actor=request.actor, database=database, - query_name=stored_query["name"] if stored_query else None, + query_name=stored_query.name if stored_query else None, request=request, sql=sql, params=params, @@ -863,7 +863,7 @@ class QueryView(View): "sql": sql, "params": params, }, - stored_query=stored_query["name"] if stored_query else None, + stored_query=stored_query.name if stored_query else None, private=private, stored_query_write=stored_query_write, db_is_immutable=not db.is_mutable, @@ -907,7 +907,7 @@ class QueryView(View): datasette, request, database=database, - query_name=stored_query["name"] if stored_query else None, + query_name=stored_query.name if stored_query else None, ), query_actions=query_actions, ), diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index de732431..46d71b8e 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -2,6 +2,7 @@ import json import re from datasette.resources import DatabaseResource, TableResource +from datasette.stored_queries import StoredQuery from datasette.utils import ( named_parameters as derive_named_parameters, escape_sqlite, @@ -281,18 +282,18 @@ async def _prepare_execute_write(datasette, db, sql, params, actor): return parameter_names, params, analysis -async def _ensure_stored_query_execution_permissions(datasette, db, query, actor): - if query.get("is_trusted"): +async def _ensure_stored_query_execution_permissions( + datasette, db, query: StoredQuery, actor +): + if query.is_trusted: return - if query.get("is_write"): + if query.is_write: await datasette.ensure_permission( action="execute-write-sql", resource=DatabaseResource(db.name), actor=actor, ) - await datasette.ensure_query_write_permissions( - db.name, query["sql"], actor=actor - ) + await datasette.ensure_query_write_permissions(db.name, query.sql, actor=actor) else: await datasette.ensure_permission( action="execute-sql", @@ -482,7 +483,7 @@ async def _prepare_query_create(datasette, request, db, data): } -async def _prepare_query_update(datasette, request, db, existing, update): +async def _prepare_query_update(datasette, request, db, existing: StoredQuery, update): invalid_keys = set(update) - _query_update_fields if invalid_keys: raise QueryValidationError( @@ -490,8 +491,8 @@ async def _prepare_query_update(datasette, request, db, existing, update): ) update = _apply_query_data_types(update) - sql = update.get("sql", existing["sql"]) - query_is_write = existing["is_write"] + sql = update.get("sql", existing.sql) + query_is_write = existing.is_write derived = _derived_query_parameters(sql) parameters = None diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 1a2c5d00..8c4e849e 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -1,6 +1,7 @@ from urllib.parse import parse_qsl, urlencode from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import stored_query_to_dict from datasette.utils import sqlite3, tilde_decode from datasette.utils.asgi import Response @@ -100,7 +101,7 @@ class QueryListView(BaseView): ) query_list_path = self.query_list_path(database) next_url = None - if page["next"]: + if page.next: pairs = [ (key, value) for key, value in parse_qsl( @@ -108,7 +109,7 @@ class QueryListView(BaseView): ) if key != "_next" ] - pairs.append(("_next", page["next"])) + pairs.append(("_next", page.next)) next_url = "{}?{}".format( query_list_path, urlencode(pairs), @@ -194,13 +195,13 @@ class QueryListView(BaseView): "database_color": ( self.ds.get_database(database).color if database is not None else None ), - "queries": page["queries"], - "next": page["next"], + "queries": page.queries, + "next": page.next, "next_url": next_url, - "has_more": page["has_more"], - "limit": page["limit"], - "show_private_note": any(query["is_private"] for query in page["queries"]), - "show_trusted_note": any(query["is_trusted"] for query in page["queries"]), + "has_more": page.has_more, + "limit": page.limit, + "show_private_note": any(query.is_private for query in page.queries), + "show_trusted_note": any(query.is_trusted for query in page.queries), "query_list_path": query_list_path, "show_database": database is None, "facets": facets, @@ -213,7 +214,12 @@ class QueryListView(BaseView): }, } if format_ == "json": - return Response.json(data) + return Response.json( + { + **data, + "queries": [stored_query_to_dict(query) for query in page.queries], + } + ) return await self.render( ["query_list.html"], request, @@ -374,8 +380,11 @@ class QueryStoreView(QueryCreateView): return _error([str(ex)], 400) query = await self.ds.get_query(db.name, name) + assert query is not None if is_json: - return Response.json({"ok": True, "query": query}, status=201) + return Response.json( + {"ok": True, "query": stored_query_to_dict(query)}, status=201 + ) self.ds.add_message(request, "Query saved", self.ds.INFO) return Response.redirect(self.ds.urls.path(self.ds.urls.table(db.name, name))) @@ -395,7 +404,7 @@ class QueryDefinitionView(BaseView): actor=request.actor, ): return _error(["Permission denied"], 403) - return Response.json({"ok": True, "query": query}) + return Response.json({"ok": True, "query": stored_query_to_dict(query)}) class QueryUpdateView(BaseView): @@ -413,7 +422,7 @@ class QueryUpdateView(BaseView): actor=request.actor, ): return _error(["Permission denied: need update-query"], 403) - if existing.get("is_trusted"): + if existing.is_trusted: return _error(["Trusted queries cannot be updated using the API"], 403) try: @@ -444,10 +453,12 @@ class QueryUpdateView(BaseView): await self.ds.update_query(db.name, query_name, **update_kwargs) if data.get("return"): + query = await self.ds.get_query(db.name, query_name) + assert query is not None return Response.json( { "ok": True, - "query": await self.ds.get_query(db.name, query_name), + "query": stored_query_to_dict(query), } ) return Response.json({"ok": True}) diff --git a/docs/internals.rst b/docs/internals.rst index 66724aa9..4980ee8b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1039,11 +1039,11 @@ Example: await .get_query(database, name) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Returns a stored query dictionary, or ``None`` if the query does not exist. +Returns a ``StoredQuery`` dataclass instance, or ``None`` if the query does not exist. -The dictionary contains ``database``, ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``params``, ``is_write``, ``is_private``, ``is_trusted``, ``source``, ``owner_id``, ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``. +``StoredQuery`` has the following attributes: ``database``, ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write``, ``is_private``, ``is_trusted``, ``source``, ``owner_id``, ``on_success_message``, ``on_success_message_sql``, ``on_success_redirect``, ``on_error_message`` and ``on_error_redirect``. -``parameters`` and ``params`` contain the same list of explicit parameter names. +``parameters`` is a list of explicit parameter names. .. _datasette_list_queries: @@ -1087,12 +1087,12 @@ Lists stored queries visible to the specified actor. ``owner_id`` - string, optional Filter by owner actor ID. ``include_private`` - boolean, optional - Set to ``True`` to include a ``private`` boolean in each returned query dictionary indicating if anonymous users would be unable to view that query. + Set to ``True`` to populate a ``private`` boolean on each returned ``StoredQuery`` indicating if anonymous users would be unable to view that query. -The return value is a dictionary with these keys: +The return value is a ``StoredQueryPage`` dataclass instance with these attributes: -``queries`` - list of dictionaries - Stored query dictionaries, in the same format returned by :ref:`datasette_get_query`. +``queries`` - list of StoredQuery instances + Stored queries in the same format returned by :ref:`datasette_get_query`. ``next`` - string or None Pagination cursor for the next page, if one exists. ``has_more`` - boolean diff --git a/tests/test_queries.py b/tests/test_queries.py index 70fb7a03..59fab8c0 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -4,6 +4,7 @@ import pytest from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource +from datasette.stored_queries import StoredQuery, StoredQueryPage from datasette.utils.asgi import Forbidden @@ -87,38 +88,41 @@ async def test_add_get_and_remove_query(): } query = await ds.get_query("data", "top_customers") - assert query == { - "database": "data", - "name": "top_customers", - "sql": "select * from customers where region = :region", - "title": "Top customers", - "description": "Customers by region", - "description_html": None, - "hide_sql": True, - "fragment": "chart", - "params": ["region"], - "parameters": ["region"], - "is_write": False, - "is_private": False, - "is_trusted": True, - "source": "user", - "owner_id": "alice", - "on_success_message": None, - "on_success_message_sql": None, - "on_success_redirect": None, - "on_error_message": None, - "on_error_redirect": None, - } + assert query == StoredQuery( + database="data", + name="top_customers", + sql="select * from customers where region = :region", + title="Top customers", + description="Customers by region", + description_html=None, + hide_sql=True, + fragment="chart", + parameters=["region"], + is_write=False, + is_private=False, + is_trusted=True, + source="user", + owner_id="alice", + on_success_message=None, + on_success_message_sql=None, + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + ) queries_page = await ds.list_queries("data", actor=None) - assert queries_page["queries"] == [query] - assert queries_page["next"] is None + assert queries_page == StoredQueryPage( + queries=[query], + next=None, + has_more=False, + limit=50, + ) await ds.remove_query("data", "top_customers") assert await ds.get_query("data", "top_customers") is None queries_page = await ds.list_queries("data", actor=None) - assert queries_page["queries"] == [] - assert queries_page["next"] is None + assert queries_page.queries == [] + assert queries_page.next is None @pytest.mark.asyncio @@ -156,13 +160,12 @@ async def test_update_query_only_updates_provided_fields(): ) query = await ds.get_query("data", "redirect") - assert query["title"] == "Updated" - assert query["parameters"] == [] - assert query["params"] == [] - assert query["on_success_redirect"] is None - assert query["sql"] == "select 1" - assert query["is_private"] is False - assert query["is_trusted"] is False + assert query.title == "Updated" + assert query.parameters == [] + assert query.on_success_redirect is None + assert query.sql == "select 1" + assert query.is_private is False + assert query.is_trusted is False options_row = ( await ds.get_internal_database().execute( """ @@ -198,28 +201,27 @@ async def test_config_queries_imported_to_internal_table(): ds.add_memory_database("query_config", name="data") await ds.invoke_startup() - assert await ds.get_query("data", "configured") == { - "database": "data", - "name": "configured", - "sql": "select :name as name", - "title": "Configured query", - "description": None, - "description_html": "

      Configured HTML

      ", - "hide_sql": False, - "fragment": None, - "params": ["name"], - "parameters": ["name"], - "is_write": False, - "is_private": False, - "is_trusted": True, - "source": "config", - "owner_id": None, - "on_success_message": None, - "on_success_message_sql": "select 'Hello ' || :name", - "on_success_redirect": None, - "on_error_message": None, - "on_error_redirect": None, - } + assert await ds.get_query("data", "configured") == StoredQuery( + database="data", + name="configured", + sql="select :name as name", + title="Configured query", + description=None, + description_html="

      Configured HTML

      ", + hide_sql=False, + fragment=None, + parameters=["name"], + is_write=False, + is_private=False, + is_trusted=True, + source="config", + owner_id=None, + on_success_message=None, + on_success_message_sql="select 'Hello ' || :name", + on_success_redirect=None, + on_error_message=None, + on_error_redirect=None, + ) @pytest.mark.asyncio @@ -1032,8 +1034,8 @@ async def test_query_update_api_rejects_config_only_fields(): "Invalid keys: description_html, on_success_message_sql" ] query = await ds.get_query("data", "editable") - assert query["description_html"] is None - assert query["on_success_message_sql"] is None + assert query.description_html is None + assert query.on_success_message_sql is None @pytest.mark.asyncio @@ -1072,9 +1074,9 @@ async def test_query_update_api_rejects_trusted_queries_but_internal_update_allo "Trusted queries cannot be updated using the API" ] query = await ds.get_query("data", "trusted_report") - assert query["is_trusted"] is True - assert query["sql"] == "select 1 as one" - assert query["title"] == "Original" + assert query.is_trusted is True + assert query.sql == "select 1 as one" + assert query.title == "Original" await ds.update_query( "data", @@ -1083,9 +1085,9 @@ async def test_query_update_api_rejects_trusted_queries_but_internal_update_allo title="Internal", ) query = await ds.get_query("data", "trusted_report") - assert query["is_trusted"] is True - assert query["sql"] == "select 3 as three" - assert query["title"] == "Internal" + assert query.is_trusted is True + assert query.sql == "select 3 as three" + assert query.title == "Internal" @pytest.mark.asyncio