From 5f9f567acbc58c9fcd88af440e68034510fb5d2b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 16:06:01 -0700 Subject: [PATCH 0001/1075] Show SQL query when reporting time limit error, closes #1819 --- datasette/database.py | 5 ++++- datasette/views/base.py | 21 +++++++++++++-------- tests/test_api.py | 12 +++++++++++- tests/test_html.py | 10 +++++++--- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 44467370..46094bd7 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -476,7 +476,10 @@ class WriteTask: class QueryInterrupted(Exception): - pass + def __init__(self, e, sql, params): + self.e = e + self.sql = sql + self.params = params class MultipleValues(Exception): diff --git a/datasette/views/base.py b/datasette/views/base.py index 221e1882..67aa3a42 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -1,10 +1,12 @@ import asyncio import csv import hashlib -import re import sys +import textwrap import time import urllib +from markupsafe import escape + import pint @@ -24,11 +26,9 @@ from datasette.utils import ( path_with_removed_args, path_with_format, sqlite3, - HASH_LENGTH, ) from datasette.utils.asgi import ( AsgiStream, - Forbidden, NotFound, Response, BadRequest, @@ -371,13 +371,18 @@ class DataView(BaseView): ) = response_or_template_contexts else: data, extra_template_data, templates = response_or_template_contexts - except QueryInterrupted: + except QueryInterrupted as ex: raise DatasetteError( - """ - SQL query took too long. The time limit is controlled by the + textwrap.dedent( + """ +

SQL query took too long. The time limit is controlled by the sql_time_limit_ms - configuration option. - """, + configuration option.

+ 
{}
+ """.format( + escape(ex.sql) + ) + ).strip(), title="SQL Interrupted", status=400, message_is_html=True, diff --git a/tests/test_api.py b/tests/test_api.py index 7a2bf91f..ad74d16e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -656,7 +656,17 @@ def test_custom_sql(app_client): def test_sql_time_limit(app_client_shorter_time_limit): response = app_client_shorter_time_limit.get("/fixtures.json?sql=select+sleep(0.5)") assert 400 == response.status - assert "SQL Interrupted" == response.json["title"] + assert response.json == { + "ok": False, + "error": ( + "

SQL query took too long. The time limit is controlled by the\n" + 'sql_time_limit_ms\n' + "configuration option.

\n" + "
select sleep(0.5)
" + ), + "status": 400, + "title": "SQL Interrupted", + } def test_custom_sql_time_limit(app_client): diff --git a/tests/test_html.py b/tests/test_html.py index bf915247..a99b0b6c 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -168,10 +168,14 @@ def test_disallowed_custom_sql_pragma(app_client): def test_sql_time_limit(app_client_shorter_time_limit): response = app_client_shorter_time_limit.get("/fixtures?sql=select+sleep(0.5)") assert 400 == response.status - expected_html_fragment = """ + expected_html_fragments = [ + """ sql_time_limit_ms - """.strip() - assert expected_html_fragment in response.text + """.strip(), + "
select sleep(0.5)
", + ] + for expected_html_fragment in expected_html_fragments: + assert expected_html_fragment in response.text def test_row_page_does_not_truncate(): From 7fb4ea4e39a15e1f7d3202949794d98af1cfa272 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Sep 2022 21:06:40 -0700 Subject: [PATCH 0002/1075] Update note about render_cell signature, refs #1826 --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f208e727..c9cab8ab 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -9,7 +9,7 @@ Each plugin can implement one or more hooks using the ``@hookimpl`` decorator ag When you implement a plugin hook you can accept any or all of the parameters that are documented as being passed to that hook. -For example, you can implement the ``render_cell`` plugin hook like this even though the full documented hook signature is ``render_cell(value, column, table, database, datasette)``: +For example, you can implement the ``render_cell`` plugin hook like this even though the full documented hook signature is ``render_cell(row, value, column, table, database, datasette)``: .. code-block:: python From 984b1df12cf19a6731889fc0665bb5f622e07b7c Mon Sep 17 00:00:00 2001 From: Adam Simpson Date: Wed, 28 Sep 2022 00:21:36 -0400 Subject: [PATCH 0003/1075] Add documentation for serving via OpenRC (#1825) * Add documentation for serving via OpenRC --- docs/deploying.rst | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index d4ad8836..c8552758 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -74,18 +74,30 @@ Once the service has started you can confirm that Datasette is running on port 8 curl 127.0.0.1:8000/-/versions.json # Should output JSON showing the installed version -Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx``. +Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx`` - see :ref:`deploying_proxy`. -Ubuntu offer `a tutorial on installing nginx `__. Once it is installed you can add configuration to proxy traffic through to Datasette that looks like this:: +.. _deploying_openrc: - server { - server_name mysubdomain.myhost.net; +Running Datasette using OpenRC +=============================== +OpenRC is the service manager on non-systemd Linux distributions like `Alpine Linux `__ and `Gentoo `__. - location / { - proxy_pass http://127.0.0.1:8000/; - proxy_set_header Host $host; - } - } +Create an init script at ``/etc/init.d/datasette`` with the following contents: + +.. code-block:: sh + + #!/sbin/openrc-run + + name="datasette" + command="datasette" + command_args="serve -h 0.0.0.0 /path/to/db.db" + command_background=true + pidfile="/run/${RC_SVCNAME}.pid" + +You then need to configure the service to run at boot and start it:: + + rc-update add datasette + rc-service datasette start .. _deploying_buildpacks: From 34defdc10aa293294ca01cfab70780755447e1d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 28 Sep 2022 17:39:36 -0700 Subject: [PATCH 0004/1075] Browse the plugins directory --- docs/writing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 01ee8c90..a3fc88ec 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -234,7 +234,7 @@ To avoid accidentally conflicting with a database file that may be loaded into D - ``/-/upload-excel`` -Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `datasette-plugin topic `__ on GitHub. +Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `plugins directory `. If your plugin includes functionality that relates to a specific database you could also register a URL route like this: From c92c4318e9892101f75fa158410c0a12c1d80b6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Sep 2022 10:55:40 -0700 Subject: [PATCH 0005/1075] Bump furo from 2022.9.15 to 2022.9.29 (#1827) Bumps [furo](https://github.com/pradyunsg/furo) from 2022.9.15 to 2022.9.29. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2022.09.15...2022.09.29) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index afcba1f0..fe258adb 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "furo==2022.9.15", + "furo==2022.9.29", "sphinx-autobuild", "codespell", "blacken-docs", From 883e326dd6ef95f854f7750ef2d4b0e17082fa96 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 2 Oct 2022 14:26:16 -0700 Subject: [PATCH 0006/1075] Drop word-wrap: anywhere, refs #1828, #1805 --- datasette/static/app.css | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 08b724f6..712b9925 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -446,7 +446,6 @@ th { } table a:link { text-decoration: none; - word-wrap: anywhere; } .rows-and-columns td:before { display: block; From 4218c9cd742b79b1e3cb80878e42b7e39d16ded2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Oct 2022 11:45:36 -0700 Subject: [PATCH 0007/1075] reST markup fix --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index c9cab8ab..832a76b0 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -268,7 +268,7 @@ you have one: def extra_js_urls(): return ["/-/static-plugins/your-plugin/app.js"] -Note that `your-plugin` here should be the hyphenated plugin name - the name that is displayed in the list on the `/-/plugins` debug page. +Note that ``your-plugin`` here should be the hyphenated plugin name - the name that is displayed in the list on the ``/-/plugins`` debug page. If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details. From b6ba117b7978b58b40e3c3c2b723b92c3010ed53 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Oct 2022 18:25:52 -0700 Subject: [PATCH 0008/1075] Clarify request or None for two hooks --- docs/plugin_hooks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 832a76b0..b61f953a 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1281,7 +1281,7 @@ menu_links(datasette, actor, request) ``actor`` - dictionary or None The currently authenticated :ref:`actor `. -``request`` - :ref:`internals_request` +``request`` - :ref:`internals_request` or None The current HTTP request. This can be ``None`` if the request object is not available. This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon. @@ -1330,7 +1330,7 @@ table_actions(datasette, actor, database, table, request) ``table`` - string The name of the table. -``request`` - :ref:`internals_request` +``request`` - :ref:`internals_request` or None The current HTTP request. This can be ``None`` if the request object is not available. This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items. From bbf33a763537a1d913180b22bd3b5fe4a5e5b252 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Oct 2022 21:32:11 -0700 Subject: [PATCH 0009/1075] Test for bool(results), closes #1832 --- tests/test_internals_database.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 9e81c1d6..4e33beed 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -30,6 +30,14 @@ async def test_results_first(db): assert isinstance(row, sqlite3.Row) +@pytest.mark.asyncio +@pytest.mark.parametrize("expected", (True, False)) +async def test_results_bool(db, expected): + where = "" if expected else "where pk = 0" + results = await db.execute("select * from facetable {}".format(where)) + assert bool(results) is expected + + @pytest.mark.parametrize( "query,expected", [ From eff112498ecc499323c26612d707908831446d25 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Thu, 6 Oct 2022 16:06:06 -0400 Subject: [PATCH 0010/1075] Useuse inspect data for hash and file size on startup Thanks, @fgregg Closes #1834 --- datasette/database.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 46094bd7..d75bd70c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -48,9 +48,13 @@ class Database: self._read_connection = None self._write_connection = None if not self.is_mutable and not self.is_memory: - p = Path(path) - self.hash = inspect_hash(p) - self.cached_size = p.stat().st_size + if self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.hash = self.ds.inspect_data[self.name]["hash"] + self.cached_size = self.ds.inspect_data[self.name]["size"] + else: + p = Path(path) + self.hash = inspect_hash(p) + self.cached_size = p.stat().st_size @property def cached_table_counts(self): From b7fec7f9020b79c1fe60cc5a2def86b50eeb5af9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 7 Oct 2022 16:03:09 -0700 Subject: [PATCH 0011/1075] .sqlite/.sqlite3 extensions for config directory mode Closes #1646 --- datasette/app.py | 5 ++++- docs/settings.rst | 2 +- tests/test_config_dir.py | 11 +++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 03d1dacc..32a911c2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -217,7 +217,10 @@ class Datasette: self._secret = secret or secrets.token_hex(32) self.files = tuple(files or []) + tuple(immutables or []) if config_dir: - self.files += tuple([str(p) for p in config_dir.glob("*.db")]) + db_files = [] + for ext in ("db", "sqlite", "sqlite3"): + db_files.extend(config_dir.glob("*.{}".format(ext))) + self.files += tuple(str(f) for f in db_files) if ( config_dir and (config_dir / "inspect-data.json").exists() diff --git a/docs/settings.rst b/docs/settings.rst index 8437fb04..a6d50543 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -46,7 +46,7 @@ Datasette will detect the files in that directory and automatically configure it The files that can be included in this directory are as follows. All are optional. -* ``*.db`` - SQLite database files that will be served by Datasette +* ``*.db`` (or ``*.sqlite3`` or ``*.sqlite``) - SQLite database files that will be served by Datasette * ``metadata.json`` - :ref:`metadata` for those databases - ``metadata.yaml`` or ``metadata.yml`` can be used as well * ``inspect-data.json`` - the result of running ``datasette inspect *.db --inspect-file=inspect-data.json`` from the configuration directory - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running * ``settings.json`` - settings that would normally be passed using ``--setting`` - here they should be stored as a JSON object of key/value pairs diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index f5ecf0d6..c2af3836 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -49,7 +49,7 @@ def config_dir(tmp_path_factory): (config_dir / "metadata.json").write_text(json.dumps(METADATA), "utf-8") (config_dir / "settings.json").write_text(json.dumps(SETTINGS), "utf-8") - for dbname in ("demo.db", "immutable.db"): + for dbname in ("demo.db", "immutable.db", "j.sqlite3", "k.sqlite"): db = sqlite3.connect(str(config_dir / dbname)) db.executescript( """ @@ -151,12 +151,11 @@ def test_databases(config_dir_client): response = config_dir_client.get("/-/databases.json") assert 200 == response.status databases = response.json - assert 2 == len(databases) + assert 4 == len(databases) databases.sort(key=lambda d: d["name"]) - assert "demo" == databases[0]["name"] - assert databases[0]["is_mutable"] - assert "immutable" == databases[1]["name"] - assert not databases[1]["is_mutable"] + for db, expected_name in zip(databases, ("demo", "immutable", "j", "k")): + assert expected_name == db["name"] + assert db["is_mutable"] == (expected_name != "immutable") @pytest.mark.parametrize("filename", ("metadata.yml", "metadata.yaml")) From 1a5e5f2aa951e5bd731067a49819efba68fbe8ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Oct 2022 14:42:52 -0700 Subject: [PATCH 0012/1075] Refactor breadcrumbs to respect permissions, refs #1831 --- datasette/app.py | 40 ++++++++++++++++++++++ datasette/templates/_crumbs.html | 15 ++++++++ datasette/templates/base.html | 4 +-- datasette/templates/database.html | 9 ----- datasette/templates/error.html | 7 ---- datasette/templates/logout.html | 7 ---- datasette/templates/permissions_debug.html | 7 ---- datasette/templates/query.html | 8 ++--- datasette/templates/row.html | 9 ++--- datasette/templates/show_json.html | 7 ---- datasette/templates/table.html | 8 ++--- tests/test_permissions.py | 1 + tests/test_plugins.py | 2 +- 13 files changed, 65 insertions(+), 59 deletions(-) create mode 100644 datasette/templates/_crumbs.html diff --git a/datasette/app.py b/datasette/app.py index 32a911c2..5fa4955c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -631,6 +631,44 @@ class Datasette: else: return [] + async def _crumb_items(self, request, table=None, database=None): + crumbs = [] + # Top-level link + if await self.permission_allowed( + actor=request.actor, action="view-instance", default=True + ): + crumbs.append({"href": self.urls.instance(), "label": "home"}) + # Database link + if database: + if await self.permission_allowed( + actor=request.actor, + action="view-database", + resource=database, + default=True, + ): + crumbs.append( + { + "href": self.urls.database(database), + "label": database, + } + ) + # Table link + if table: + assert database, "table= requires database=" + if await self.permission_allowed( + actor=request.actor, + action="view-table", + resource=(database, table), + default=True, + ): + crumbs.append( + { + "href": self.urls.table(database, table), + "label": table, + } + ) + return crumbs + async def permission_allowed(self, actor, action, resource=None, default=False): """Check permissions using the permissions_allowed plugin hook""" result = None @@ -1009,6 +1047,8 @@ class Datasette: template_context = { **context, **{ + "request": request, + "crumb_items": self._crumb_items, "urls": self.urls, "actor": request.actor if request else None, "menu_links": menu_links, diff --git a/datasette/templates/_crumbs.html b/datasette/templates/_crumbs.html new file mode 100644 index 00000000..bd1ff0da --- /dev/null +++ b/datasette/templates/_crumbs.html @@ -0,0 +1,15 @@ +{% macro nav(request, database=None, table=None) -%} +{% if crumb_items is defined %} + {% set items=crumb_items(request=request, database=database, table=table) %} + {% if items %} +

+ {% for item in items %} + {{ item.label }} + {% if not loop.last %} + / + {% endif %} + {% endfor %} +

+ {% endif %} +{% endif %} +{%- endmacro %} diff --git a/datasette/templates/base.html b/datasette/templates/base.html index c3a71acb..87c939ac 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -1,4 +1,4 @@ - +{% import "_crumbs.html" as crumbs with context %} {% block title %}{% endblock %} @@ -17,7 +17,7 @@
-
+ + + + +{% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} +{% include "_execute_write_analysis_scripts.html" %} + + + +{% endblock %} diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index fa4859b1..a8c9a391 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -205,32 +205,32 @@

Queries

-
-

- - - {% if filters.is_write %}{% endif %} - {% if filters.is_private %}{% endif %} - {% if filters.source %}{% endif %} - {% if filters.owner_id %}{% endif %} - -

-
- - - {% if queries %} +
+

+ + + {% if filters.is_write %}{% endif %} + {% if filters.is_private %}{% endif %} + {% if filters.source %}{% endif %} + {% if filters.owner_id %}{% endif %} + +

+
+ + +
diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index f30a30bc..9efe3f81 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -436,6 +436,35 @@ async def _query_create_form_context( } +async def _query_edit_form_context( + datasette, + request, + db, + existing: StoredQuery, + *, + sql=None, + title=None, + description=None, + is_private=None, +): + sql = existing.sql if sql is None else sql + title = existing.title if title is None else title + description = existing.description if description is None else description + is_private = existing.is_private if is_private is None else is_private + analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor) + return { + "database": db.name, + "database_color": db.color, + "name": existing.name, + "sql": sql, + "title": title or "", + "description": description or "", + "is_private": is_private, + "query_url": datasette.urls.table(db.name, existing.name), + **analysis_data, + } + + async def _inserted_row_url(datasette, db, analysis, cursor): if cursor.rowcount != 1: return None diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 8c4e849e..2753f876 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -18,6 +18,7 @@ from .query_helpers import ( _query_create_analysis_data, _query_create_form_context, _query_create_form_error_message, + _query_edit_form_context, _query_list_limit, ) @@ -464,13 +465,164 @@ class QueryUpdateView(BaseView): return Response.json({"ok": True}) -class QueryDeleteView(BaseView): - name = "query-delete" +class QueryEditView(BaseView): + name = "query-edit" + has_json_alternate = False - async def post(self, request): + async def _load(self, request): db = await self.ds.resolve_database(request) query_name = tilde_decode(request.url_vars["query"]) existing = await self.ds.get_query(db.name, query_name) + return db, query_name, existing + + async def _render_form( + self, + request, + db, + existing, + *, + sql=None, + title=None, + description=None, + is_private=None, + status=200, + ): + response = await self.render( + ["query_edit.html"], + request, + await _query_edit_form_context( + self.ds, + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + ), + ) + response.status = status + return response + + async def get(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + await self.ds.ensure_permission( + action="update-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ) + if existing.is_trusted: + return _error(["Trusted queries cannot be edited"], 403) + return await self._render_form(request, db, existing) + + async def post(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + if not await self.ds.allowed( + action="update-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ): + return _error(["Permission denied: need update-query"], 403) + if existing.is_trusted: + return _error(["Trusted queries cannot be edited"], 403) + + data, _ = await _json_or_form_payload(request) + if not isinstance(data, dict): + return _error(["Invalid form submission"], 400) + sql = data.get("sql") + sql = existing.sql if sql is None else sql.strip() + title = data.get("title") or "" + description = data.get("description") or "" + is_private = _as_bool(data.get("is_private")) + + update = { + "title": title, + "description": description, + "is_private": is_private, + } + if sql != existing.sql: + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + self.ds.add_message( + request, + "Permission denied: need execute-sql to change the SQL", + self.ds.ERROR, + ) + return await self._render_form( + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + status=403, + ) + update["sql"] = sql + + try: + update_kwargs = await _prepare_query_update( + self.ds, request, db, existing, update + ) + except QueryValidationError as ex: + self.ds.add_message(request, ex.message, self.ds.ERROR) + return await self._render_form( + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + status=ex.status, + ) + + await self.ds.update_query(db.name, query_name, **update_kwargs) + self.ds.add_message(request, "Query updated", self.ds.INFO) + return Response.redirect( + self.ds.urls.path(self.ds.urls.table(db.name, query_name)) + ) + + +class QueryDeleteView(BaseView): + name = "query-delete" + has_json_alternate = False + + async def _load(self, request): + db = await self.ds.resolve_database(request) + query_name = tilde_decode(request.url_vars["query"]) + existing = await self.ds.get_query(db.name, query_name) + return db, query_name, existing + + async def get(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + await self.ds.ensure_permission( + action="delete-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ) + return await self.render( + ["query_delete.html"], + request, + { + "database": db.name, + "database_color": db.color, + "query": stored_query_to_dict(existing), + "query_url": self.ds.urls.table(db.name, query_name), + }, + ) + + async def post(self, request): + db, query_name, existing = await self._load(request) if existing is None: return _error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( @@ -479,5 +631,14 @@ class QueryDeleteView(BaseView): actor=request.actor, ): return _error(["Permission denied: need delete-query"], 403) + + data, is_json = await _json_or_form_payload(request) await self.ds.remove_query(db.name, query_name) - return Response.json({"ok": True}) + if is_json: + return Response.json({"ok": True}) + self.ds.add_message( + request, + "Query “{}” deleted".format(existing.title or query_name), + self.ds.INFO, + ) + return Response.redirect(self.ds.urls.path(self.ds.urls.database(db.name))) diff --git a/docs/changelog.rst b/docs/changelog.rst index d5f8fa14..75e4f3e8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v1_0_a33: + +1.0a33 (unreleased) +------------------- + +- Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) + .. _v1_0_a32: 1.0a32 (2026-05-31) diff --git a/docs/plugins.rst b/docs/plugins.rst index d578e9e2..c2eb282a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -271,6 +271,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "register_token_handler" ] }, + { + "name": "datasette.default_query_actions", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "query_actions" + ] + }, { "name": "datasette.events", "static": false, diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index c0ba67f0..371348fb 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -142,6 +142,15 @@ Datasette stores both configured queries and user-created queries in the ``queri Stored queries created by users default to private. Private stored queries can only be viewed, updated or deleted by the actor that created them. Broad ``view-query``, ``update-query`` or ``delete-query`` permission grants still do not allow other actors to access another actor's private stored queries. +Editing and deleting stored queries ++++++++++++++++++++++++++++++++++++ + +The page for a stored query includes a "Query actions" menu with **Edit this query** and **Delete this query** links for actors who have permission to use them. + +The owner of a stored query can always edit and delete it. For queries that are not private, any actor granted the ``update-query`` or ``delete-query`` permission can edit or delete the query, even if they did not create it. Private queries can only be edited or deleted by their owner, regardless of any broad permission grants. + +Editing a query lets you change its title, description, SQL and whether it is private. Changing the SQL also requires the ``execute-sql`` permission (and the relevant write permissions for writable queries). The same operations are available through the JSON API by sending a ``POST`` to ``///-/update`` or ``///-/delete``. Trusted stored queries cannot be edited or deleted through the web interface or the JSON API. + Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. SQL functions are allowed and are not separately restricted by Datasette permissions. .. _trusted_stored_queries: diff --git a/tests/test_docs.py b/tests/test_docs.py index 9cf39f41..51caf595 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -77,6 +77,7 @@ def documented_views(): "QueryCreateAnalyzeView", "QueryDeleteView", "QueryDefinitionView", + "QueryEditView", "QueryListView", "QueryParametersView", "QueryStoreView", diff --git a/tests/test_queries.py b/tests/test_queries.py index 25e423d4..6e9bcbdb 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -3,6 +3,7 @@ import re from html import unescape import pytest +from bs4 import BeautifulSoup as Soup from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource @@ -712,6 +713,10 @@ async def test_query_list_search_filter_and_html(): "/data/-/queries?is_private=1", actor={"id": "root"}, ) + no_results_response = await ds.client.get( + "/data/-/queries?q=nope", + actor={"id": "root"}, + ) assert html_response.status_code == 200 assert "Demo query 02" in html_response.text @@ -799,6 +804,13 @@ async def test_query_list_search_filter_and_html(): 'Not private0' not in filtered_private_response.text ) + assert no_results_response.status_code == 200 + assert "No queries found." in no_results_response.text + assert 'class="query-list-filters core"' not in no_results_response.text + assert 'id="query-search"' not in no_results_response.text + assert 'class="query-list-facets"' not in no_results_response.text + assert "

Mode

" not in no_results_response.text + assert "

Visibility

" not in no_results_response.text @pytest.mark.asyncio @@ -1114,6 +1126,227 @@ async def test_query_update_api_rejects_trusted_queries_but_internal_update_allo assert query.title == "Internal" +async def _make_ds_with_user_query(name, *, is_private=False, owner_id="owner"): + ds = Datasette(memory=True, settings={"default_allow_sql": True}) + db = ds.add_memory_database(name, name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "saved", + "select * from dogs", + title="Saved query", + description="A saved query", + source="user", + owner_id=owner_id, + is_private=is_private, + ) + return ds + + +@pytest.mark.asyncio +async def test_query_edit_form_renders_and_updates_for_owner(): + ds = await _make_ds_with_user_query("query_edit_owner") + actor = {"id": "owner"} + + # GET renders the form pre-filled with existing values + get_response = await ds.client.get("/data/saved/-/edit", actor=actor) + assert get_response.status_code == 200 + assert 'value="Saved query"' in get_response.text + assert ">A saved query" in get_response.text + assert "select * from dogs" in get_response.text + # URL slug is shown but not editable + assert 'name="name"' not in get_response.text + + # POST updates the query and redirects back to the query page + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=actor, + data={ + "title": "Updated title", + "description": "Updated description", + "sql": "select id from dogs", + "is_private": "1", + }, + ) + assert post_response.status_code == 302 + assert post_response.headers["location"] == "/data/saved" + + query = await ds.get_query("data", "saved") + assert query.title == "Updated title" + assert query.description == "Updated description" + assert query.sql == "select id from dogs" + assert query.is_private is True + + +@pytest.mark.asyncio +async def test_query_edit_metadata_only_does_not_require_execute_sql(): + # An owner who can no longer execute SQL can still edit title/description + ds = await _make_ds_with_user_query("query_edit_metadata_only") + actor = {"id": "owner"} + + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=actor, + data={ + "title": "Renamed", + "description": "A saved query", + "sql": "select * from dogs", + }, + ) + assert post_response.status_code == 302 + query = await ds.get_query("data", "saved") + assert query.title == "Renamed" + + +@pytest.mark.asyncio +async def test_private_query_edit_delete_restricted_to_owner(): + ds = await _make_ds_with_user_query( + "query_edit_private", is_private=True, owner_id="owner" + ) + + # A different actor cannot view, edit or delete the private query + other = {"id": "intruder"} + assert (await ds.client.get("/data/saved/-/edit", actor=other)).status_code == 403 + assert (await ds.client.get("/data/saved/-/delete", actor=other)).status_code == 403 + delete_attempt = await ds.client.post( + "/data/saved/-/delete", + actor=other, + data={}, + ) + assert delete_attempt.status_code == 403 + assert await ds.get_query("data", "saved") is not None + + # The owner can edit and delete + owner = {"id": "owner"} + assert (await ds.client.get("/data/saved/-/edit", actor=owner)).status_code == 200 + + +@pytest.mark.asyncio +async def test_non_private_query_editable_by_permitted_non_owner(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "execute-sql": {"id": "editor"}, + "update-query": {"id": "editor"}, + "delete-query": {"id": "editor"}, + } + } + } + }, + ) + db = ds.add_memory_database("query_non_private_editor", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "saved", + "select * from dogs", + title="Shared", + source="user", + owner_id="owner", + is_private=False, + ) + + editor = {"id": "editor"} + # Editor (not the owner) can edit because the query is not private + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=editor, + data={ + "title": "Edited by editor", + "description": "", + "sql": "select * from dogs", + }, + ) + assert post_response.status_code == 302 + query = await ds.get_query("data", "saved") + assert query.title == "Edited by editor" + + # Editor can also delete it + delete_response = await ds.client.post( + "/data/saved/-/delete", + actor=editor, + data={}, + ) + assert delete_response.status_code == 302 + assert await ds.get_query("data", "saved") is None + + +@pytest.mark.asyncio +async def test_query_delete_confirmation_and_form_delete(): + ds = await _make_ds_with_user_query("query_delete_form") + actor = {"id": "owner"} + + get_response = await ds.client.get("/data/saved/-/delete", actor=actor) + assert get_response.status_code == 200 + assert "Are you sure" in get_response.text + assert "select * from dogs" in get_response.text + soup = Soup(get_response.text, "html.parser") + form = soup.select_one("form.query-delete-form") + assert form is not None + assert "core" in form["class"] + assert form.select_one('input[type="submit"][value="Delete query"]') is not None + + post_response = await ds.client.post( + "/data/saved/-/delete", + actor=actor, + data={}, + ) + assert post_response.status_code == 302 + assert post_response.headers["location"] == "/data" + assert await ds.get_query("data", "saved") is None + + +@pytest.mark.asyncio +async def test_query_action_menu_shows_edit_and_delete_for_owner(): + ds = await _make_ds_with_user_query("query_action_menu") + + owner_response = await ds.client.get("/data/saved", actor={"id": "owner"}) + assert owner_response.status_code == 200 + assert "/data/saved/-/edit" in owner_response.text + assert "/data/saved/-/delete" in owner_response.text + + # A different actor (the query is public) cannot edit/delete by default + other_response = await ds.client.get("/data/saved", actor={"id": "stranger"}) + assert other_response.status_code == 200 + assert "/data/saved/-/edit" not in other_response.text + assert "/data/saved/-/delete" not in other_response.text + + +@pytest.mark.asyncio +async def test_query_edit_rejected_for_trusted_query(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "execute-sql": {"id": "editor"}, + "update-query": {"id": "editor"}, + }, + "queries": {"trusted_report": {"sql": "select 1 as one"}}, + } + } + }, + ) + ds.add_memory_database("query_edit_trusted", name="data") + await ds.invoke_startup() + + response = await ds.client.get( + "/data/trusted_report/-/edit", actor={"id": "editor"} + ) + assert response.status_code == 403 + # Edit/delete links should not appear on a trusted/config query page + page = await ds.client.get("/data/trusted_report", actor={"id": "editor"}) + assert "/data/trusted_report/-/edit" not in page.text + + @pytest.mark.asyncio async def test_query_store_api_rejects_magic_parameters(): ds = Datasette(memory=True, default_deny=True) From 03f1ffdf8fbf0ed7da46be48f3fcc3f4698e1e21 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:45:01 -0700 Subject: [PATCH 1043/1075] Centralize JSON extra parsing --- datasette/extras.py | 6 ++++++ datasette/renderer.py | 3 ++- datasette/views/table.py | 7 ++----- tests/test_table_api.py | 11 +++++++++++ 4 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 datasette/extras.py diff --git a/datasette/extras.py b/datasette/extras.py new file mode 100644 index 00000000..01a9fb4b --- /dev/null +++ b/datasette/extras.py @@ -0,0 +1,6 @@ +def extra_names_from_request(request): + extra_bits = request.args.getlist("_extra") + extras = set() + for bit in extra_bits: + extras.update(part for part in bit.split(",") if part) + return extras diff --git a/datasette/renderer.py b/datasette/renderer.py index acf23e59..f40e3dbb 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -1,4 +1,5 @@ import json +from datasette.extras import extra_names_from_request from datasette.utils import ( value_as_boolean, remove_infinites, @@ -108,7 +109,7 @@ def json_renderer(request, args, data, error, truncated=None): # Don't include "columns" in output # https://github.com/simonw/datasette/issues/2136 - if isinstance(data, dict) and "columns" not in request.args.getlist("_extra"): + if isinstance(data, dict) and "columns" not in extra_names_from_request(request): data.pop("columns", None) # Handle _nl option for _shape=array diff --git a/datasette/views/table.py b/datasette/views/table.py index 4df1e1b4..9ba249f4 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -6,6 +6,7 @@ import urllib from asyncinject import Registry import markupsafe +from datasette.extras import extra_names_from_request from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette.events import ( @@ -850,11 +851,7 @@ class TableDropView(BaseView): def _get_extras(request): - extra_bits = request.args.getlist("_extra") - extras = set() - for bit in extra_bits: - extras.update(bit.split(",")) - return extras + return extra_names_from_request(request) async def _columns_to_select(table_columns, pks, request): diff --git a/tests/test_table_api.py b/tests/test_table_api.py index ceeb646d..eeb3dc8b 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1376,6 +1376,17 @@ async def test_table_extras(ds_client, extra, expected_json): assert response.json() == expected_json +@pytest.mark.asyncio +async def test_table_extra_columns_can_be_comma_separated(ds_client): + response = await ds_client.get( + "/fixtures/primary_key_multiple_columns.json?_extra=columns,count" + ) + assert response.status_code == 200 + data = response.json() + assert data["columns"] == ["id", "content", "content2"] + assert data["count"] == 1 + + @pytest.mark.asyncio async def test_extra_render_cell(): """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook""" From 17bbe6855c34630c14b077e08247d453d371cdea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:52:10 -0700 Subject: [PATCH 1044/1075] Refactor table JSON extras into classes --- datasette/extras.py | 94 ++++ datasette/views/table.py | 584 ++----------------------- datasette/views/table_extras.py | 746 ++++++++++++++++++++++++++++++++ 3 files changed, 884 insertions(+), 540 deletions(-) create mode 100644 datasette/views/table_extras.py diff --git a/datasette/extras.py b/datasette/extras.py index 01a9fb4b..786ec4f4 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,6 +1,100 @@ +import re +from enum import Enum +from typing import ClassVar + +from asyncinject import Registry + + def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") extras = set() for bit in extra_bits: extras.update(part for part in bit.split(",") if part) return extras + + +class ExtraScope(Enum): + TABLE = "table" + + +class Provider: + name: ClassVar[str | None] = None + scopes: ClassVar[frozenset[ExtraScope]] = frozenset() + public: ClassVar[bool] = False + + @classmethod + def key(cls): + return cls.name or _camel_to_snake(cls.__name__) + + @classmethod + def available_for(cls, scope): + return scope in cls.scopes + + async def resolve(self, context): + raise NotImplementedError + + +class Extra(Provider): + description: ClassVar[str | None] = None + public: ClassVar[bool] = True + stable: ClassVar[bool] = True + expensive: ClassVar[bool] = False + docs_note: ClassVar[str | None] = None + + @classmethod + def documentation(cls): + return { + "name": cls.key(), + "description": cls.description, + "scopes": [ + scope.value for scope in sorted(cls.scopes, key=lambda s: s.value) + ], + "stable": cls.stable, + "expensive": cls.expensive, + "docs_note": cls.docs_note, + } + + +class ExtraRegistry: + def __init__(self, classes): + self.classes = list(classes) + self.classes_by_name = {cls.key(): cls for cls in self.classes} + + def classes_for_scope(self, scope, include_internal=True): + classes = [ + cls + for cls in self.classes + if cls.available_for(scope) and (include_internal or cls.public) + ] + return classes + + def public_classes_for_scope(self, scope): + return self.classes_for_scope(scope, include_internal=False) + + async def resolve(self, requested, context, scope): + registry = Registry() + + async def context_provider(): + return context + + registry.register(context_provider, name="context") + + for cls in self.classes_for_scope(scope): + registry.register(cls().resolve, name=cls.key()) + + public_names = {cls.key() for cls in self.public_classes_for_scope(scope)} + requested_public_names = [ + name + for name in requested + if name in public_names and name in registry._registry + ] + resolved = await registry.resolve_multi(requested_public_names) + return { + name: resolved[name] for name in requested_public_names if name in resolved + } + + +def _camel_to_snake(name): + name = re.sub(r"(Extra|Provider)$", "", name) + name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() diff --git a/datasette/views/table.py b/datasette/views/table.py index 9ba249f4..c2d520f8 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -3,12 +3,10 @@ import itertools import json import urllib -from asyncinject import Registry import markupsafe from datasette.extras import extra_names_from_request from datasette.plugins import pm -from datasette.database import QueryInterrupted from datasette.events import ( AlterTableEvent, DropTableEvent, @@ -47,6 +45,12 @@ from datasette.filters import Filters import sqlite_utils from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView +from .table_extras import ( + TABLE_EXTRA_BUNDLES, + TableExtraContext, + resolve_table_extras, + table_extra_registry, +) LINK_WITH_LABEL = ( '{label} {id}' @@ -1465,560 +1469,60 @@ async def table_view_data( if extra_extras: extras.update(extra_extras) - async def extra_count_sql(): - return count_sql - - async def extra_count(): - "Total count of rows matching these filters" - # Calculate the total count for this query - count = None - if ( - not db.is_mutable - and datasette.inspect_data - and count_sql == f"select count(*) from {table_name} " - ): - # We can use a previously cached table row count - try: - count = datasette.inspect_data[database_name]["tables"][table_name][ - "count" - ] - except KeyError: - pass - - # Otherwise run a select count(*) ... - if count_sql and count is None and not nocount: - count_sql_limited = ( - f"select count(*) from (select * {from_sql} limit 10001)" - ) - try: - count_rows = list(await db.execute(count_sql_limited, from_sql_params)) - count = count_rows[0][0] - except QueryInterrupted: - pass - return count - - async def facet_instances(extra_count): - facet_instances = [] - facet_classes = list( - itertools.chain.from_iterable(pm.hook.register_facet_classes()) - ) - for facet_class in facet_classes: - facet_instances.append( - facet_class( - datasette, - request, - database_name, - sql=sql_no_order_no_limit, - params=params, - table=table_name, - table_config=table_metadata, - row_count=extra_count, - ) - ) - return facet_instances - - async def extra_facet_results(facet_instances): - "Results of facets calculated against this data" - facet_results = {} - facets_timed_out = [] - - if not nofacet: - # Run them in parallel - facet_awaitables = [facet.facet_results() for facet in facet_instances] - facet_awaitable_results = await run_sequential(*facet_awaitables) - for ( - instance_facet_results, - instance_facets_timed_out, - ) in facet_awaitable_results: - for facet_info in instance_facet_results: - base_key = facet_info["name"] - key = base_key - i = 1 - while key in facet_results: - i += 1 - key = f"{base_key}_{i}" - facet_results[key] = facet_info - facets_timed_out.extend(instance_facets_timed_out) - - return { - "results": facet_results, - "timed_out": facets_timed_out, - } - - async def extra_suggested_facets(facet_instances): - "Suggestions for facets that might return interesting results" - suggested_facets = [] - # Calculate suggested facets - if ( - datasette.setting("suggest_facets") - and datasette.setting("allow_facet") - and not _next - and not nofacet - and not nosuggest - ): - # Run them in parallel - facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] - for suggest_result in await run_sequential(*facet_suggest_awaitables): - suggested_facets.extend(suggest_result) - return suggested_facets - # Faceting if not datasette.setting("allow_facet") and any( arg.startswith("_facet") for arg in request.args ): raise BadRequest("_facet= is not allowed") - # human_description_en combines filters AND search, if provided - async def extra_human_description_en(): - "Human-readable description of the filters" - human_description_en = filters.human_description_en( - extra=extra_human_descriptions - ) - if sort or sort_desc: - human_description_en = " ".join( - [b for b in [human_description_en, sorted_by] if b] - ) - return human_description_en - - if sort or sort_desc: - sorted_by = "sorted by {}{}".format( - (sort or sort_desc), " descending" if sort_desc else "" - ) - - async def extra_next_url(): - "Full URL for the next page of results" - return next_url - - async def extra_columns(): - "Column names returned by this query" - return columns - - async def extra_all_columns(): - "All columns in the table, regardless of _col/_nocol filtering" - return list(table_columns) - - async def extra_primary_keys(): - "Primary keys for this table" - return pks - - async def extra_actions(): - async def actions(): - links = [] - kwargs = { - "datasette": datasette, - "database": database_name, - "actor": request.actor, - "request": request, - } - if is_view: - kwargs["view"] = table_name - method = pm.hook.view_actions - else: - kwargs["table"] = table_name - method = pm.hook.table_actions - for hook in method(**kwargs): - extra_links = await await_me_maybe(hook) - if extra_links: - links.extend(extra_links) - return links - - return actions - - async def extra_is_view(): - return is_view - - async def extra_debug(): - "Extra debug information" - return { - "resolved": repr(resolved), - "url_vars": request.url_vars, - "nofacet": nofacet, - "nosuggest": nosuggest, - } - - async def extra_request(): - "Full information about the request" - return { - "url": request.url, - "path": request.path, - "full_path": request.full_path, - "host": request.host, - "args": request.args._data, - } - - async def run_display_columns_and_rows(): - display_columns, display_rows = await display_columns_and_rows( - datasette, - database_name, - table_name, - results.description, - rows, - link_column=not is_view, - truncate_cells=datasette.setting("truncate_cells_html"), - sortable_columns=sortable_columns, - request=request, - ) - return { - "columns": display_columns, - "rows": display_rows, - } - - async def extra_display_columns(run_display_columns_and_rows): - return run_display_columns_and_rows["columns"] - - async def extra_display_rows(run_display_columns_and_rows): - return run_display_columns_and_rows["rows"] - - async def extra_render_cell(): - "Rendered HTML for each cell using the render_cell plugin hook" - pks_for_display = pks if pks else (["rowid"] if not is_view else []) - col_names = [col[0] for col in results.description] - ct_map = await datasette.get_column_types(database_name, table_name) - rendered_rows = [] - for row in rows: - rendered_row = {} - for value, column in zip(row, col_names): - ct = ct_map.get(column) - plugin_display_value = None - # Try column type render_cell first - if ct: - candidate = await ct.render_cell( - value=value, - column=column, - table=table_name, - database=database_name, - datasette=datasette, - request=request, - ) - if candidate is not None: - plugin_display_value = candidate - if plugin_display_value is None: - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=table_name, - pks=pks_for_display, - database=database_name, - datasette=datasette, - request=request, - column_type=ct, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - rendered_row[column] = str(plugin_display_value) - rendered_rows.append(rendered_row) - return rendered_rows - - async def extra_query(): - "Details of the underlying SQL query" - return { - "sql": sql, - "params": params, - } - - async def extra_column_types(): - "Column type assignments for this table" - ct_map = await datasette.get_column_types(database_name, table_name) - return { - col_name: { - "type": ct.name, - "config": ct.config, - } - for col_name, ct in ct_map.items() - } - - async def extra_set_column_type_ui(): - "Column type UI metadata for this table" - if is_view: - return None - - if not await datasette.allowed( - action="set-column-type", - resource=TableResource(database=database_name, table=table_name), - actor=request.actor, - ): - return None - - column_details = await datasette._get_resource_column_details( - database_name, table_name - ) - ct_map = await datasette.get_column_types(database_name, table_name) - columns = {} - for column_name, column_detail in column_details.items(): - current = ct_map.get(column_name) - columns[column_name] = { - "current": ( - {"type": current.name, "config": current.config} - if current is not None - else None - ), - "options": [ - { - "name": name, - "description": ct_cls.description, - } - for name, ct_cls in sorted(datasette._column_types.items()) - if datasette._column_type_is_applicable(ct_cls, column_detail) - ], - } - return { - "path": "{}/-/set-column-type".format( - datasette.urls.table(database_name, table_name) - ), - "columns": columns, - } - - async def extra_metadata(): - "Metadata about the table and database" - tablemetadata = await datasette.get_resource_metadata(database_name, table_name) - - rows = await datasette.get_internal_database().execute( - """ - SELECT - column_name, - value - FROM metadata_columns - WHERE database_name = ? - AND resource_name = ? - AND key = 'description' - """, - [database_name, table_name], - ) - tablemetadata["columns"] = dict(rows) - return tablemetadata - - async def extra_database(): - return database_name - - async def extra_table(): - return table_name - - async def extra_database_color(): - return db.color - - async def extra_form_hidden_args(): - form_hidden_args = [] - for key in request.args: - if ( - key.startswith("_") - and key not in ("_sort", "_sort_desc", "_search", "_next") - and "__" not in key - ): - for value in request.args.getlist(key): - form_hidden_args.append((key, value)) - return form_hidden_args - - async def extra_filters(): - return filters - - async def extra_custom_table_templates(): - return [ - f"_table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", - f"_table-table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", - "_table.html", - ] - - async def extra_sorted_facet_results(extra_facet_results): - facet_configs = table_metadata.get("facets", []) - if facet_configs: - # Build ordered list of facet names from metadata config - metadata_facet_names = [] - for fc in facet_configs: - if isinstance(fc, str): - metadata_facet_names.append(fc) - elif isinstance(fc, dict): - metadata_facet_names.append(list(fc.values())[0]) - metadata_order = {name: i for i, name in enumerate(metadata_facet_names)} - metadata_facets = [] - request_facets = [] - for f in extra_facet_results["results"].values(): - if f["name"] in metadata_order: - metadata_facets.append(f) - else: - request_facets.append(f) - metadata_facets.sort(key=lambda f: metadata_order[f["name"]]) - request_facets.sort( - key=lambda f: (len(f["results"]), f["name"]), - reverse=True, - ) - return metadata_facets + request_facets - else: - return sorted( - extra_facet_results["results"].values(), - key=lambda f: (len(f["results"]), f["name"]), - reverse=True, - ) - - async def extra_table_definition(): - return await db.get_table_definition(table_name) - - async def extra_view_definition(): - return await db.get_view_definition(table_name) - - async def extra_renderers(extra_expandable_columns, extra_query): - renderers = {} - url_labels_extra = {} - if extra_expandable_columns: - url_labels_extra = {"_labels": "on"} - for key, (_, can_render) in datasette.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=datasette, - columns=columns or [], - rows=rows or [], - sql=extra_query.get("sql", None), - query_name=None, - database=database_name, - table=table_name, - request=request, - view_name="table", - ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = datasette.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format=key, - extra_qs={**url_labels_extra}, - ) - ) - return renderers - - async def extra_private(): - return private - - async def extra_expandable_columns(): - expandables = [] - db = datasette.databases[database_name] - for fk in await db.foreign_keys_for_table(table_name): - label_column = await db.label_column_for_table(fk["other_table"]) - expandables.append((fk, label_column)) - return expandables - - async def extra_extras(): - "Available ?_extra= blocks" - all_extras = [ - (key[len("extra_") :], fn.__doc__) - for key, fn in registry._registry.items() - if key.startswith("extra_") - ] - return [ - { - "name": name, - "description": doc, - "toggle_url": datasette.absolute_url( - request, - datasette.urls.path( - path_with_added_args(request, {"_extra": name}) - if name not in extras - else path_with_removed_args(request, {"_extra": name}) - ), - ), - "selected": name in extras, - } - for name, doc in all_extras - ] - - async def extra_facets_timed_out(extra_facet_results): - return extra_facet_results["timed_out"] - - bundles = { - "html": [ - "suggested_facets", - "facet_results", - "facets_timed_out", - "count", - "count_sql", - "human_description_en", - "next_url", - "metadata", - "query", - "columns", - "display_columns", - "display_rows", - "database", - "table", - "database_color", - "actions", - "filters", - "renderers", - "custom_table_templates", - "sorted_facet_results", - "table_definition", - "view_definition", - "is_view", - "private", - "primary_keys", - "all_columns", - "expandable_columns", - "form_hidden_args", - "set_column_type_ui", - ] - } - - for key, values in bundles.items(): + for key, values in TABLE_EXTRA_BUNDLES.items(): if f"_{key}" in extras: extras.update(values) extras.discard(f"_{key}") - registry = Registry( - extra_count, - extra_count_sql, - extra_facet_results, - extra_facets_timed_out, - extra_suggested_facets, - facet_instances, - extra_human_description_en, - extra_next_url, - extra_columns, - extra_all_columns, - extra_primary_keys, - run_display_columns_and_rows, - extra_display_columns, - extra_display_rows, - extra_render_cell, - extra_debug, - extra_request, - extra_query, - extra_column_types, - extra_set_column_type_ui, - extra_metadata, - extra_extras, - extra_database, - extra_table, - extra_database_color, - extra_actions, - extra_filters, - extra_renderers, - extra_custom_table_templates, - extra_sorted_facet_results, - extra_table_definition, - extra_view_definition, - extra_is_view, - extra_private, - extra_expandable_columns, - extra_form_hidden_args, + table_extra_context = TableExtraContext( + datasette=datasette, + request=request, + resolved=resolved, + db=db, + database_name=database_name, + table_name=table_name, + is_view=is_view, + private=private, + rows=rows, + columns=columns, + results_description=results.description, + table_columns=table_columns, + pks=pks, + count_sql=count_sql, + from_sql=from_sql, + from_sql_params=from_sql_params, + nocount=nocount, + nofacet=nofacet, + nosuggest=nosuggest, + next_arg=request.args.get("_next"), + next_value=next_value, + next_url=next_url, + sql=sql, + sql_no_order_no_limit=sql_no_order_no_limit, + params=params, + table_metadata=table_metadata, + filters=filters, + extra_human_descriptions=extra_human_descriptions, + sort=sort, + sort_desc=sort_desc, + sortable_columns=sortable_columns, + extras=extras, + extra_registry=table_extra_registry, + display_columns_and_rows=display_columns_and_rows, + run_sequential=run_sequential, ) - results = await registry.resolve_multi( - ["extra_{}".format(extra) for extra in extras] - ) data = { "ok": True, "next": next_value and str(next_value) or None, } - data.update( - { - key.replace("extra_", ""): value - for key, value in results.items() - if key.startswith("extra_") and key.replace("extra_", "") in extras - } - ) + data.update(await resolve_table_extras(extras, table_extra_context)) raw_sqlite_rows = rows[:page_size] # Apply transform_value for columns with assigned types ct_map = await datasette.get_column_types(database_name, table_name) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py new file mode 100644 index 00000000..2ec2adf0 --- /dev/null +++ b/datasette/views/table_extras.py @@ -0,0 +1,746 @@ +import itertools +from dataclasses import dataclass + +from datasette.database import QueryInterrupted +from datasette.extras import Extra, ExtraRegistry, ExtraScope, Provider +from datasette.plugins import pm +from datasette.resources import TableResource +from datasette.utils import ( + await_me_maybe, + call_with_supported_arguments, + path_with_added_args, + path_with_format, + path_with_removed_args, + to_css_class, +) + + +@dataclass(frozen=True) +class TableExtraContext: + datasette: object + request: object + resolved: object + db: object + database_name: str + table_name: str + is_view: bool + private: bool + rows: list + columns: list + results_description: list + table_columns: list + pks: list + count_sql: str + from_sql: str + from_sql_params: dict + nocount: object + nofacet: object + nosuggest: object + next_arg: object + next_value: object + next_url: str | None + sql: str + sql_no_order_no_limit: str + params: dict + table_metadata: dict + filters: object + extra_human_descriptions: list + sort: str | None + sort_desc: str | None + sortable_columns: set + extras: set + extra_registry: ExtraRegistry + display_columns_and_rows: object + run_sequential: object + + +class CountSqlExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.count_sql + + +class CountExtra(Extra): + description = "Total count of rows matching these filters" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context): + count = None + if ( + not context.db.is_mutable + and context.datasette.inspect_data + and context.count_sql == f"select count(*) from {context.table_name} " + ): + try: + count = context.datasette.inspect_data[context.database_name]["tables"][ + context.table_name + ]["count"] + except KeyError: + pass + + if context.count_sql and count is None and not context.nocount: + count_sql_limited = ( + f"select count(*) from (select * {context.from_sql} limit 10001)" + ) + try: + count_rows = list( + await context.db.execute(count_sql_limited, context.from_sql_params) + ) + count = count_rows[0][0] + except QueryInterrupted: + pass + return count + + +class FacetInstancesProvider(Provider): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, count): + facet_instances = [] + facet_classes = list( + itertools.chain.from_iterable(pm.hook.register_facet_classes()) + ) + for facet_class in facet_classes: + facet_instances.append( + facet_class( + context.datasette, + context.request, + context.database_name, + sql=context.sql_no_order_no_limit, + params=context.params, + table=context.table_name, + table_config=context.table_metadata, + row_count=count, + ) + ) + return facet_instances + + +class FacetResultsExtra(Extra): + description = "Results of facets calculated against this data" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context, facet_instances): + facet_results = {} + facets_timed_out = [] + + if not context.nofacet: + facet_awaitables = [facet.facet_results() for facet in facet_instances] + facet_awaitable_results = await context.run_sequential(*facet_awaitables) + for ( + instance_facet_results, + instance_facets_timed_out, + ) in facet_awaitable_results: + for facet_info in instance_facet_results: + base_key = facet_info["name"] + key = base_key + i = 1 + while key in facet_results: + i += 1 + key = f"{base_key}_{i}" + facet_results[key] = facet_info + facets_timed_out.extend(instance_facets_timed_out) + + return { + "results": facet_results, + "timed_out": facets_timed_out, + } + + +class FacetsTimedOutExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, facet_results): + return facet_results["timed_out"] + + +class SuggestedFacetsExtra(Extra): + description = "Suggestions for facets that might return interesting results" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context, facet_instances): + suggested_facets = [] + if ( + context.datasette.setting("suggest_facets") + and context.datasette.setting("allow_facet") + and not context.next_arg + and not context.nofacet + and not context.nosuggest + ): + facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] + for suggest_result in await context.run_sequential( + *facet_suggest_awaitables + ): + suggested_facets.extend(suggest_result) + return suggested_facets + + +class HumanDescriptionEnExtra(Extra): + description = "Human-readable description of the filters" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + human_description_en = context.filters.human_description_en( + extra=context.extra_human_descriptions + ) + if context.sort or context.sort_desc: + sorted_by = "sorted by {}{}".format( + (context.sort or context.sort_desc), + " descending" if context.sort_desc else "", + ) + human_description_en = " ".join( + [b for b in [human_description_en, sorted_by] if b] + ) + return human_description_en + + +class NextUrlExtra(Extra): + description = "Full URL for the next page of results" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.next_url + + +class ColumnsExtra(Extra): + description = "Column names returned by this query" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.columns + + +class AllColumnsExtra(Extra): + description = "All columns in the table, regardless of _col/_nocol filtering" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return list(context.table_columns) + + +class PrimaryKeysExtra(Extra): + description = "Primary keys for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.pks + + +class ActionsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + async def actions(): + links = [] + kwargs = { + "datasette": context.datasette, + "database": context.database_name, + "actor": context.request.actor, + "request": context.request, + } + if context.is_view: + kwargs["view"] = context.table_name + method = pm.hook.view_actions + else: + kwargs["table"] = context.table_name + method = pm.hook.table_actions + for hook in method(**kwargs): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + + return actions + + +class IsViewExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.is_view + + +class DebugExtra(Extra): + description = "Extra debug information" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "resolved": repr(context.resolved), + "url_vars": context.request.url_vars, + "nofacet": context.nofacet, + "nosuggest": context.nosuggest, + } + + +class RequestExtra(Extra): + description = "Full information about the request" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "url": context.request.url, + "path": context.request.path, + "full_path": context.request.full_path, + "host": context.request.host, + "args": context.request.args._data, + } + + +class DisplayColumnsAndRowsProvider(Provider): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + display_columns, display_rows = await context.display_columns_and_rows( + context.datasette, + context.database_name, + context.table_name, + context.results_description, + context.rows, + link_column=not context.is_view, + truncate_cells=context.datasette.setting("truncate_cells_html"), + sortable_columns=context.sortable_columns, + request=context.request, + ) + return { + "columns": display_columns, + "rows": display_rows, + } + + +class DisplayColumnsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, display_columns_and_rows): + return display_columns_and_rows["columns"] + + +class DisplayRowsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, display_columns_and_rows): + return display_columns_and_rows["rows"] + + +class RenderCellExtra(Extra): + description = "Rendered HTML for each cell using the render_cell plugin hook" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + pks_for_display = ( + context.pks if context.pks else (["rowid"] if not context.is_view else []) + ) + col_names = [col[0] for col in context.results_description] + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + rendered_rows = [] + for row in context.rows: + rendered_row = {} + for value, column in zip(row, col_names): + ct = ct_map.get(column) + plugin_display_value = None + if ct: + candidate = await ct.render_cell( + value=value, + column=column, + table=context.table_name, + database=context.database_name, + datasette=context.datasette, + request=context.request, + ) + if candidate is not None: + plugin_display_value = candidate + if plugin_display_value is None: + for candidate in pm.hook.render_cell( + row=row, + value=value, + column=column, + table=context.table_name, + pks=pks_for_display, + database=context.database_name, + datasette=context.datasette, + request=context.request, + column_type=ct, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value: + rendered_row[column] = str(plugin_display_value) + rendered_rows.append(rendered_row) + return rendered_rows + + +class QueryExtra(Extra): + description = "Details of the underlying SQL query" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "sql": context.sql, + "params": context.params, + } + + +class ColumnTypesExtra(Extra): + description = "Column type assignments for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + return { + col_name: { + "type": ct.name, + "config": ct.config, + } + for col_name, ct in ct_map.items() + } + + +class SetColumnTypeUiExtra(Extra): + description = "Column type UI metadata for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + if context.is_view: + return None + + if not await context.datasette.allowed( + action="set-column-type", + resource=TableResource( + database=context.database_name, table=context.table_name + ), + actor=context.request.actor, + ): + return None + + column_details = await context.datasette._get_resource_column_details( + context.database_name, context.table_name + ) + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + columns = {} + for column_name, column_detail in column_details.items(): + current = ct_map.get(column_name) + columns[column_name] = { + "current": ( + {"type": current.name, "config": current.config} + if current is not None + else None + ), + "options": [ + { + "name": name, + "description": ct_cls.description, + } + for name, ct_cls in sorted(context.datasette._column_types.items()) + if context.datasette._column_type_is_applicable( + ct_cls, column_detail + ) + ], + } + return { + "path": "{}/-/set-column-type".format( + context.datasette.urls.table(context.database_name, context.table_name) + ), + "columns": columns, + } + + +class MetadataExtra(Extra): + description = "Metadata about the table and database" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + tablemetadata = await context.datasette.get_resource_metadata( + context.database_name, context.table_name + ) + + rows = await context.datasette.get_internal_database().execute( + """ + SELECT + column_name, + value + FROM metadata_columns + WHERE database_name = ? + AND resource_name = ? + AND key = 'description' + """, + [context.database_name, context.table_name], + ) + tablemetadata["columns"] = dict(rows) + return tablemetadata + + +class DatabaseExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.database_name + + +class TableExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.table_name + + +class DatabaseColorExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.db.color + + +class FormHiddenArgsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + form_hidden_args = [] + for key in context.request.args: + if ( + key.startswith("_") + and key not in ("_sort", "_sort_desc", "_search", "_next") + and "__" not in key + ): + for value in context.request.args.getlist(key): + form_hidden_args.append((key, value)) + return form_hidden_args + + +class FiltersExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.filters + + +class CustomTableTemplatesExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return [ + f"_table-{to_css_class(context.database_name)}-{to_css_class(context.table_name)}.html", + f"_table-table-{to_css_class(context.database_name)}-{to_css_class(context.table_name)}.html", + "_table.html", + ] + + +class SortedFacetResultsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, facet_results): + facet_configs = context.table_metadata.get("facets", []) + if facet_configs: + metadata_facet_names = [] + for fc in facet_configs: + if isinstance(fc, str): + metadata_facet_names.append(fc) + elif isinstance(fc, dict): + metadata_facet_names.append(list(fc.values())[0]) + metadata_order = {name: i for i, name in enumerate(metadata_facet_names)} + metadata_facets = [] + request_facets = [] + for f in facet_results["results"].values(): + if f["name"] in metadata_order: + metadata_facets.append(f) + else: + request_facets.append(f) + metadata_facets.sort(key=lambda f: metadata_order[f["name"]]) + request_facets.sort( + key=lambda f: (len(f["results"]), f["name"]), + reverse=True, + ) + return metadata_facets + request_facets + else: + return sorted( + facet_results["results"].values(), + key=lambda f: (len(f["results"]), f["name"]), + reverse=True, + ) + + +class TableDefinitionExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return await context.db.get_table_definition(context.table_name) + + +class ViewDefinitionExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return await context.db.get_view_definition(context.table_name) + + +class RenderersExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, expandable_columns, query): + renderers = {} + url_labels_extra = {} + if expandable_columns: + url_labels_extra = {"_labels": "on"} + for key, (_, can_render) in context.datasette.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=context.datasette, + columns=context.columns or [], + rows=context.rows or [], + sql=query.get("sql", None), + query_name=None, + database=context.database_name, + table=context.table_name, + request=context.request, + view_name="table", + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = context.datasette.urls.path( + path_with_format( + request=context.request, + path=context.request.scope.get("route_path"), + format=key, + extra_qs={**url_labels_extra}, + ) + ) + return renderers + + +class PrivateExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.private + + +class ExpandableColumnsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + expandables = [] + db = context.datasette.databases[context.database_name] + for fk in await db.foreign_keys_for_table(context.table_name): + label_column = await db.label_column_for_table(fk["other_table"]) + expandables.append((fk, label_column)) + return expandables + + +class ExtrasExtra(Extra): + description = "Available ?_extra= blocks" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + all_extras = [ + (cls.key(), cls.description) + for cls in context.extra_registry.public_classes_for_scope(ExtraScope.TABLE) + ] + return [ + { + "name": name, + "description": description, + "toggle_url": context.datasette.absolute_url( + context.request, + context.datasette.urls.path( + path_with_added_args(context.request, {"_extra": name}) + if name not in context.extras + else path_with_removed_args(context.request, {"_extra": name}) + ), + ), + "selected": name in context.extras, + } + for name, description in all_extras + ] + + +TABLE_EXTRA_BUNDLES = { + "html": [ + "suggested_facets", + "facet_results", + "facets_timed_out", + "count", + "count_sql", + "human_description_en", + "next_url", + "metadata", + "query", + "columns", + "display_columns", + "display_rows", + "database", + "table", + "database_color", + "actions", + "filters", + "renderers", + "custom_table_templates", + "sorted_facet_results", + "table_definition", + "view_definition", + "is_view", + "private", + "primary_keys", + "all_columns", + "expandable_columns", + "form_hidden_args", + "set_column_type_ui", + ] +} + + +TABLE_EXTRA_CLASSES = [ + CountExtra, + CountSqlExtra, + FacetResultsExtra, + FacetsTimedOutExtra, + SuggestedFacetsExtra, + FacetInstancesProvider, + HumanDescriptionEnExtra, + NextUrlExtra, + ColumnsExtra, + AllColumnsExtra, + PrimaryKeysExtra, + DisplayColumnsAndRowsProvider, + DisplayColumnsExtra, + DisplayRowsExtra, + RenderCellExtra, + DebugExtra, + RequestExtra, + QueryExtra, + ColumnTypesExtra, + SetColumnTypeUiExtra, + MetadataExtra, + ExtrasExtra, + DatabaseExtra, + TableExtra, + DatabaseColorExtra, + ActionsExtra, + FiltersExtra, + RenderersExtra, + CustomTableTemplatesExtra, + SortedFacetResultsExtra, + TableDefinitionExtra, + ViewDefinitionExtra, + IsViewExtra, + PrivateExtra, + ExpandableColumnsExtra, + FormHiddenArgsExtra, +] + + +table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) + + +async def resolve_table_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) From 111eeaf3702cd5ee417532beb80c746d49d92a11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:56:00 -0700 Subject: [PATCH 1045/1075] Document table JSON extras from metadata --- datasette/views/table_extras.py | 18 +++++++ docs/json_api.rst | 95 +++++++++++++++++++++++++++++++++ docs/json_api_doc.py | 20 +++++++ 3 files changed, 133 insertions(+) create mode 100644 docs/json_api_doc.py diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 2ec2adf0..e71c15d6 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -55,6 +55,7 @@ class TableExtraContext: class CountSqlExtra(Extra): + description = "SQL query used to calculate the total count" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -151,6 +152,7 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): + description = "Facet calculations that timed out" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -231,6 +233,7 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): + description = "Table or view actions made available by plugin hooks" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -258,6 +261,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): + description = "Whether this resource is a view instead of a table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -313,6 +317,7 @@ class DisplayColumnsAndRowsProvider(Provider): class DisplayColumnsExtra(Extra): + description = "Column metadata used by the HTML table display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -320,6 +325,7 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): + description = "Row data formatted for the HTML table display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -482,6 +488,7 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): + description = "Database name" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -489,6 +496,7 @@ class DatabaseExtra(Extra): class TableExtra(Extra): + description = "Table name" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -496,6 +504,7 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): + description = "Color assigned to the database" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -503,6 +512,7 @@ class DatabaseColorExtra(Extra): class FormHiddenArgsExtra(Extra): + description = "Hidden form arguments used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -519,6 +529,7 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): + description = "Filters object used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -526,6 +537,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): + description = "Custom template names considered for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -537,6 +549,7 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): + description = "Facet results sorted for display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -571,6 +584,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): + description = "SQL definition for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -578,6 +592,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): + description = "SQL definition for this view" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -585,6 +600,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): + description = "Alternative output renderers available for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, expandable_columns, query): @@ -619,6 +635,7 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): + description = "Whether this table is private to the current actor" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -626,6 +643,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): + description = "Foreign key columns that can be expanded with labels" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 65031bf4..af60a527 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -235,6 +235,101 @@ query string arguments: Only available if the :ref:`setting_trace_debug` setting is enabled. +.. _json_api_extra: + +Expanding table JSON responses +------------------------------ + +Table JSON responses can be expanded with one or more ``?_extra=`` parameters. +These can be repeated or comma-separated: + +:: + + ?_extra=columns&_extra=count,next_url + +The available table extras are listed below. + +.. [[[cog + from json_api_doc import table_extras + table_extras(cog) +.. ]]] + +.. list-table:: + :header-rows: 1 + + * - Extra + - Description + * - ``count`` + - Total count of rows matching these filters (May execute additional queries.) + * - ``count_sql`` + - SQL query used to calculate the total count + * - ``facet_results`` + - Results of facets calculated against this data (May execute additional queries.) + * - ``facets_timed_out`` + - Facet calculations that timed out + * - ``suggested_facets`` + - Suggestions for facets that might return interesting results (May execute additional queries.) + * - ``human_description_en`` + - Human-readable description of the filters + * - ``next_url`` + - Full URL for the next page of results + * - ``columns`` + - Column names returned by this query + * - ``all_columns`` + - All columns in the table, regardless of _col/_nocol filtering + * - ``primary_keys`` + - Primary keys for this table + * - ``display_columns`` + - Column metadata used by the HTML table display + * - ``display_rows`` + - Row data formatted for the HTML table display + * - ``render_cell`` + - Rendered HTML for each cell using the render_cell plugin hook + * - ``debug`` + - Extra debug information + * - ``request`` + - Full information about the request + * - ``query`` + - Details of the underlying SQL query + * - ``column_types`` + - Column type assignments for this table + * - ``set_column_type_ui`` + - Column type UI metadata for this table + * - ``metadata`` + - Metadata about the table and database + * - ``extras`` + - Available ?_extra= blocks + * - ``database`` + - Database name + * - ``table`` + - Table name + * - ``database_color`` + - Color assigned to the database + * - ``actions`` + - Table or view actions made available by plugin hooks + * - ``filters`` + - Filters object used by the HTML table interface + * - ``renderers`` + - Alternative output renderers available for this table + * - ``custom_table_templates`` + - Custom template names considered for this table + * - ``sorted_facet_results`` + - Facet results sorted for display + * - ``table_definition`` + - SQL definition for this table + * - ``view_definition`` + - SQL definition for this view + * - ``is_view`` + - Whether this resource is a view instead of a table + * - ``private`` + - Whether this table is private to the current actor + * - ``expandable_columns`` + - Foreign key columns that can be expanded with labels + * - ``form_hidden_args`` + - Hidden form arguments used by the HTML table interface + +.. [[[end]]] + .. _table_arguments: Table arguments diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py new file mode 100644 index 00000000..f07c3ba7 --- /dev/null +++ b/docs/json_api_doc.py @@ -0,0 +1,20 @@ +def table_extras(cog): + from datasette.extras import ExtraScope + from datasette.views.table_extras import table_extra_registry + + cog.out("\n.. list-table::\n") + cog.out(" :header-rows: 1\n\n") + cog.out(" * - Extra\n") + cog.out(" - Description\n") + for cls in table_extra_registry.public_classes_for_scope(ExtraScope.TABLE): + description = cls.description or "" + notes = [] + if cls.expensive: + notes.append("May execute additional queries.") + if cls.docs_note: + notes.append(cls.docs_note) + if notes: + description = "{} ({})".format(description, " ".join(notes)).strip() + cog.out(" * - ``{}``\n".format(cls.key())) + cog.out(" - {}\n".format(description)) + cog.out("\n") From 79c8aff31df16e514616a7778fad1386ac9b4b2c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:10:58 -0700 Subject: [PATCH 1046/1075] Add generated examples for table JSON extras --- datasette/extras.py | 11 + datasette/views/table_extras.py | 87 +++++- docs/json_api.rst | 470 +++++++++++++++++++++++++++----- docs/json_api_doc.py | 64 ++++- tests/test_docs.py | 10 + 5 files changed, 561 insertions(+), 81 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 786ec4f4..d867f26c 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,4 +1,5 @@ import re +from dataclasses import dataclass from enum import Enum from typing import ClassVar @@ -17,6 +18,14 @@ class ExtraScope(Enum): TABLE = "table" +@dataclass(frozen=True) +class ExtraExample: + path: str | None = None + key: str | None = None + value: object | None = None + note: str | None = None + + class Provider: name: ClassVar[str | None] = None scopes: ClassVar[frozenset[ExtraScope]] = frozenset() @@ -36,6 +45,7 @@ class Provider: class Extra(Provider): description: ClassVar[str | None] = None + example: ClassVar[ExtraExample | None] = None public: ClassVar[bool] = True stable: ClassVar[bool] = True expensive: ClassVar[bool] = False @@ -52,6 +62,7 @@ class Extra(Provider): "stable": cls.stable, "expensive": cls.expensive, "docs_note": cls.docs_note, + "example": cls.example, } diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index e71c15d6..0eefeaa9 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -2,7 +2,7 @@ import itertools from dataclasses import dataclass from datasette.database import QueryInterrupted -from datasette.extras import Extra, ExtraRegistry, ExtraScope, Provider +from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider from datasette.plugins import pm from datasette.resources import TableResource from datasette.utils import ( @@ -56,6 +56,7 @@ class TableExtraContext: class CountSqlExtra(Extra): description = "SQL query used to calculate the total count" + example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -64,6 +65,7 @@ class CountSqlExtra(Extra): class CountExtra(Extra): description = "Total count of rows matching these filters" + example = ExtraExample("/fixtures/facetable.json?_extra=count") scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -121,6 +123,22 @@ class FacetInstancesProvider(Provider): class FacetResultsExtra(Extra): description = "Results of facets calculated against this data" + example = ExtraExample( + value={ + "results": { + "state": { + "name": "state", + "type": "column", + "results": [ + {"value": "CA", "label": "CA", "count": 10}, + {"value": "MI", "label": "MI", "count": 4}, + ], + } + }, + "timed_out": [], + }, + note="Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.", + ) scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -153,6 +171,9 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): description = "Facet calculations that timed out" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -161,6 +182,15 @@ class FacetsTimedOutExtra(Extra): class SuggestedFacetsExtra(Extra): description = "Suggestions for facets that might return interesting results" + example = ExtraExample( + value=[ + { + "name": "state", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state", + } + ], + note="Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.", + ) scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -183,6 +213,9 @@ class SuggestedFacetsExtra(Extra): class HumanDescriptionEnExtra(Extra): description = "Human-readable description of the filters" + example = ExtraExample( + "/fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -202,6 +235,7 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" + example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -210,6 +244,7 @@ class NextUrlExtra(Extra): class ColumnsExtra(Extra): description = "Column names returned by this query" + example = ExtraExample("/fixtures/facetable.json?_extra=columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -218,6 +253,7 @@ class ColumnsExtra(Extra): class AllColumnsExtra(Extra): description = "All columns in the table, regardless of _col/_nocol filtering" + example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -226,6 +262,7 @@ class AllColumnsExtra(Extra): class PrimaryKeysExtra(Extra): description = "Primary keys for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -262,6 +299,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): description = "Whether this resource is a view instead of a table" + example = ExtraExample("/fixtures/simple_view.json?_extra=is_view") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -318,6 +356,28 @@ class DisplayColumnsAndRowsProvider(Provider): class DisplayColumnsExtra(Extra): description = "Column metadata used by the HTML table display" + example = ExtraExample( + value=[ + { + "name": "pk", + "sortable": True, + "is_pk": True, + "type": "INTEGER", + "notnull": 0, + }, + { + "name": "created", + "sortable": True, + "is_pk": False, + "type": "TEXT", + "notnull": 0, + "description": None, + "column_type": None, + "column_type_config": None, + }, + ], + note="Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.", + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -334,6 +394,13 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" + example = ExtraExample( + value=[ + {}, + {"content": "Custom rendered HTML"}, + ], + note="Only columns whose rendered value differs from the default are included.", + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -385,6 +452,7 @@ class RenderCellExtra(Extra): class QueryExtra(Extra): description = "Details of the underlying SQL query" + example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -396,6 +464,7 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" + example = ExtraExample(value={}) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -464,6 +533,7 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): description = "Metadata about the table and database" + example = ExtraExample("/fixtures/facetable.json?_extra=metadata") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -489,6 +559,7 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): description = "Database name" + example = ExtraExample("/fixtures/facetable.json?_extra=database") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -497,6 +568,7 @@ class DatabaseExtra(Extra): class TableExtra(Extra): description = "Table name" + example = ExtraExample("/fixtures/facetable.json?_extra=table") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -505,6 +577,7 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" + example = ExtraExample("/fixtures/facetable.json?_extra=database_color") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -513,6 +586,9 @@ class DatabaseColorExtra(Extra): class FormHiddenArgsExtra(Extra): description = "Hidden form arguments used by the HTML table interface" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -538,6 +614,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -550,6 +627,9 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): description = "Facet results sorted for display" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -585,6 +665,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): description = "SQL definition for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=table_definition") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -593,6 +674,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): description = "SQL definition for this view" + example = ExtraExample("/fixtures/simple_view.json?_extra=view_definition") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -601,6 +683,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=renderers") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, expandable_columns, query): @@ -636,6 +719,7 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): description = "Whether this table is private to the current actor" + example = ExtraExample("/fixtures/facetable.json?_extra=private") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -644,6 +728,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" + example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index af60a527..d12a388e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -254,79 +254,405 @@ The available table extras are listed below. table_extras(cog) .. ]]] -.. list-table:: - :header-rows: 1 +``count`` + Total count of rows matching these filters (May execute additional queries.) - * - Extra - - Description - * - ``count`` - - Total count of rows matching these filters (May execute additional queries.) - * - ``count_sql`` - - SQL query used to calculate the total count - * - ``facet_results`` - - Results of facets calculated against this data (May execute additional queries.) - * - ``facets_timed_out`` - - Facet calculations that timed out - * - ``suggested_facets`` - - Suggestions for facets that might return interesting results (May execute additional queries.) - * - ``human_description_en`` - - Human-readable description of the filters - * - ``next_url`` - - Full URL for the next page of results - * - ``columns`` - - Column names returned by this query - * - ``all_columns`` - - All columns in the table, regardless of _col/_nocol filtering - * - ``primary_keys`` - - Primary keys for this table - * - ``display_columns`` - - Column metadata used by the HTML table display - * - ``display_rows`` - - Row data formatted for the HTML table display - * - ``render_cell`` - - Rendered HTML for each cell using the render_cell plugin hook - * - ``debug`` - - Extra debug information - * - ``request`` - - Full information about the request - * - ``query`` - - Details of the underlying SQL query - * - ``column_types`` - - Column type assignments for this table - * - ``set_column_type_ui`` - - Column type UI metadata for this table - * - ``metadata`` - - Metadata about the table and database - * - ``extras`` - - Available ?_extra= blocks - * - ``database`` - - Database name - * - ``table`` - - Table name - * - ``database_color`` - - Color assigned to the database - * - ``actions`` - - Table or view actions made available by plugin hooks - * - ``filters`` - - Filters object used by the HTML table interface - * - ``renderers`` - - Alternative output renderers available for this table - * - ``custom_table_templates`` - - Custom template names considered for this table - * - ``sorted_facet_results`` - - Facet results sorted for display - * - ``table_definition`` - - SQL definition for this table - * - ``view_definition`` - - SQL definition for this view - * - ``is_view`` - - Whether this resource is a view instead of a table - * - ``private`` - - Whether this table is private to the current actor - * - ``expandable_columns`` - - Foreign key columns that can be expanded with labels - * - ``form_hidden_args`` - - Hidden form arguments used by the HTML table interface + ``GET /fixtures/facetable.json?_extra=count`` + + .. code-block:: json + + 15 + +``count_sql`` + SQL query used to calculate the total count + + ``GET /fixtures/facetable.json?_size=0&_extra=count_sql`` + + .. code-block:: json + + "select count(*) from facetable " + +``facet_results`` + Results of facets calculated against this data (May execute additional queries.) + + Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results. + + .. code-block:: json + + { + "results": { + "state": { + "name": "state", + "type": "column", + "results": [ + { + "value": "CA", + "label": "CA", + "count": 10 + }, + { + "value": "MI", + "label": "MI", + "count": 4 + } + ] + } + }, + "timed_out": [] + } + +``facets_timed_out`` + Facet calculations that timed out + + ``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out`` + + .. code-block:: json + + [] + +``suggested_facets`` + Suggestions for facets that might return interesting results (May execute additional queries.) + + Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets. + + .. code-block:: json + + [ + { + "name": "state", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state" + } + ] + +``human_description_en`` + Human-readable description of the filters + + ``GET /fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en`` + + .. code-block:: json + + "where state = \"CA\" sorted by pk" + +``next_url`` + Full URL for the next page of results + + ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` + + .. code-block:: json + + "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" + +``columns`` + Column names returned by this query + + ``GET /fixtures/facetable.json?_extra=columns`` + + .. code-block:: json + + [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + "n" + ] + +``all_columns`` + All columns in the table, regardless of _col/_nocol filtering + + ``GET /fixtures/facetable.json?_col=pk&_extra=all_columns`` + + .. code-block:: json + + [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + "n" + ] + +``primary_keys`` + Primary keys for this table + + ``GET /fixtures/facetable.json?_extra=primary_keys`` + + .. code-block:: json + + [ + "pk" + ] + +``display_columns`` + Column metadata used by the HTML table display + + Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns. + + .. code-block:: json + + [ + { + "name": "pk", + "sortable": true, + "is_pk": true, + "type": "INTEGER", + "notnull": 0 + }, + { + "name": "created", + "sortable": true, + "is_pk": false, + "type": "TEXT", + "notnull": 0, + "description": null, + "column_type": null, + "column_type_config": null + } + ] + +``display_rows`` + Row data formatted for the HTML table display + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + [ + {}, + { + "content": "Custom rendered HTML" + } + ] + +``debug`` + Extra debug information + +``request`` + Full information about the request + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/facetable.json?_size=1&_extra=query`` + + .. code-block:: json + + { + "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 2", + "params": {} + } + +``column_types`` + Column type assignments for this table + + .. code-block:: json + + {} + +``set_column_type_ui`` + Column type UI metadata for this table + +``metadata`` + Metadata about the table and database + + ``GET /fixtures/facetable.json?_extra=metadata`` + + .. code-block:: json + + { + "columns": {} + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/facetable.json?_extra=database`` + + .. code-block:: json + + "fixtures" + +``table`` + Table name + + ``GET /fixtures/facetable.json?_extra=table`` + + .. code-block:: json + + "facetable" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/facetable.json?_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``actions`` + Table or view actions made available by plugin hooks + +``filters`` + Filters object used by the HTML table interface + +``renderers`` + Alternative output renderers available for this table + + ``GET /fixtures/facetable.json?_extra=renderers`` + + .. code-block:: json + + { + "json": "/fixtures/facetable.json?_extra=renderers&_format=json&_labels=on" + } + +``custom_table_templates`` + Custom template names considered for this table + + ``GET /fixtures/facetable.json?_extra=custom_table_templates`` + + .. code-block:: json + + [ + "_table-fixtures-facetable.html", + "_table-table-fixtures-facetable.html", + "_table.html" + ] + +``sorted_facet_results`` + Facet results sorted for display + + ``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results`` + + .. code-block:: json + + [ + { + "name": "state", + "type": "column", + "hideable": true, + "toggle_url": "/fixtures/facetable.json?_extra=sorted_facet_results", + "results": [ + { + "value": "CA", + "label": "CA", + "count": 10, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=CA", + "selected": false + }, + { + "value": "MI", + "label": "MI", + "count": 4, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MI", + "selected": false + }, + { + "value": "MC", + "label": "MC", + "count": 1, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MC", + "selected": false + } + ], + "truncated": false + } + ] + +``table_definition`` + SQL definition for this table + + ``GET /fixtures/facetable.json?_extra=table_definition`` + + .. code-block:: json + + "CREATE TABLE facetable (\n pk integer primary key,\n created text,\n planet_int integer,\n on_earth integer,\n state text,\n _city_id integer,\n _neighborhood text,\n tags text,\n complex_array text,\n distinct_some_null,\n n text,\n FOREIGN KEY (\"_city_id\") REFERENCES [facet_cities](id)\n);" + +``view_definition`` + SQL definition for this view + + ``GET /fixtures/simple_view.json?_extra=view_definition`` + + .. code-block:: json + + "CREATE VIEW simple_view AS\n SELECT content, upper(content) AS upper_content FROM simple_primary_key;" + +``is_view`` + Whether this resource is a view instead of a table + + ``GET /fixtures/simple_view.json?_extra=is_view`` + + .. code-block:: json + + true + +``private`` + Whether this table is private to the current actor + + ``GET /fixtures/facetable.json?_extra=private`` + + .. code-block:: json + + false + +``expandable_columns`` + Foreign key columns that can be expanded with labels + + ``GET /fixtures/facetable.json?_extra=expandable_columns`` + + .. code-block:: json + + [ + [ + { + "column": "_city_id", + "other_table": "facet_cities", + "other_column": "id" + }, + "name" + ] + ] + +``form_hidden_args`` + Hidden form arguments used by the HTML table interface + + ``GET /fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args`` + + .. code-block:: json + + [ + [ + "_facet", + "state" + ], + [ + "_size", + "1" + ], + [ + "_extra", + "form_hidden_args" + ] + ] .. [[[end]]] diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index f07c3ba7..69ec6e5e 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -1,12 +1,20 @@ +import asyncio +import json +import pathlib +import tempfile +import textwrap + + def table_extras(cog): from datasette.extras import ExtraScope from datasette.views.table_extras import table_extra_registry - cog.out("\n.. list-table::\n") - cog.out(" :header-rows: 1\n\n") - cog.out(" * - Extra\n") - cog.out(" - Description\n") - for cls in table_extra_registry.public_classes_for_scope(ExtraScope.TABLE): + classes = table_extra_registry.public_classes_for_scope(ExtraScope.TABLE) + + live_examples = asyncio.run(_fetch_live_examples(classes)) + cog.out("\n") + for cls in classes: + example = cls.example description = cls.description or "" notes = [] if cls.expensive: @@ -15,6 +23,46 @@ def table_extras(cog): notes.append(cls.docs_note) if notes: description = "{} ({})".format(description, " ".join(notes)).strip() - cog.out(" * - ``{}``\n".format(cls.key())) - cog.out(" - {}\n".format(description)) - cog.out("\n") + + cog.out("``{}``\n".format(cls.key())) + cog.out(" {}\n\n".format(description)) + if example is None: + continue + + if example.path: + value = live_examples[(example.path, example.key or cls.key())] + cog.out(" ``GET {}``\n\n".format(example.path)) + else: + value = example.value + if example.note: + cog.out(" {}\n\n".format(example.note)) + cog.out(" .. code-block:: json\n\n") + cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) + cog.out("\n\n") + + +async def _fetch_live_examples(classes): + from datasette.app import Datasette + from datasette.fixtures import write_fixture_database + + examples = {} + with tempfile.TemporaryDirectory() as tmpdir: + db_path = pathlib.Path(tmpdir) / "fixtures.db" + write_fixture_database(db_path) + datasette = Datasette([str(db_path)], settings={"num_sql_threads": 1}) + try: + for cls in classes: + example = cls.example + if example is None or not example.path: + continue + key = example.key or cls.key() + response = await datasette.client.get(example.path) + assert response.status_code == 200, example.path + data = response.json() + assert key in data, "{} missing from {}".format(key, example.path) + examples[(example.path, key)] = data[key] + finally: + for db in datasette.databases.values(): + if not db.is_memory: + db.close() + return examples diff --git a/tests/test_docs.py b/tests/test_docs.py index 51caf595..784755e9 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -112,6 +112,16 @@ def test_table_filters_are_documented(documented_table_filters, subtests): assert f.key in documented_table_filters +def test_table_extra_examples_are_documented(): + from datasette.views.table_extras import CountExtra + + assert CountExtra.example.path == "/fixtures/facetable.json?_extra=count" + content = (docs_path / "json_api.rst").read_text() + section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + assert "GET /fixtures/facetable.json?_extra=count" in section + assert ".. code-block:: json" in section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 22f80b819625b9f6b5aa0661f58d97c89882a932 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:13:53 -0700 Subject: [PATCH 1047/1075] Clarify render_cell JSON extra example --- datasette/views/table_extras.py | 20 +++++++++++++++----- docs/json_api.rst | 26 +++++++++++++++++++------- tests/test_docs.py | 8 ++++++++ 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 0eefeaa9..b6e653c4 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -395,11 +395,21 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" example = ExtraExample( - value=[ - {}, - {"content": "Custom rendered HTML"}, - ], - note="Only columns whose rendered value differs from the default are included.", + value={ + "rows": [ + {"id": 1, "content": "hello"}, + {"id": 4, "content": "RENDER_CELL_DEMO"}, + ], + "render_cell": [ + {}, + {"content": "Custom rendered HTML"}, + ], + }, + note=( + "The ``render_cell`` array has one item per row, in the same order as " + "the ``rows`` array. Each object is keyed by column name. Only columns " + "whose rendered value differs from the default are included." + ), ) scopes = frozenset({ExtraScope.TABLE}) diff --git a/docs/json_api.rst b/docs/json_api.rst index d12a388e..24d59577 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -428,16 +428,28 @@ The available table extras are listed below. ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook - Only columns whose rendered value differs from the default are included. + The ``render_cell`` array has one item per row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. .. code-block:: json - [ - {}, - { - "content": "Custom rendered HTML" - } - ] + { + "rows": [ + { + "id": 1, + "content": "hello" + }, + { + "id": 4, + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + {}, + { + "content": "Custom rendered HTML" + } + ] + } ``debug`` Extra debug information diff --git a/tests/test_docs.py b/tests/test_docs.py index 784755e9..c4e0a849 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -122,6 +122,14 @@ def test_table_extra_examples_are_documented(): assert ".. code-block:: json" in section +def test_render_cell_extra_example_explains_row_and_column_mapping(): + content = (docs_path / "json_api.rst").read_text() + section = content.split("``render_cell``")[-1].split("``query``")[0] + assert "same order as the ``rows`` array" in section + assert '"rows": [' in section + assert '"render_cell": [' in section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 0fa872d43842d87af9e7b8c193f90addfcf164ba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:20:06 -0700 Subject: [PATCH 1048/1075] Add debug and request JSON extra examples --- datasette/views/table_extras.py | 2 ++ docs/json_api.rst | 31 +++++++++++++++++++++++++++++++ tests/test_docs.py | 13 +++++++++++++ 3 files changed, 46 insertions(+) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index b6e653c4..e888ee9f 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -308,6 +308,7 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" + example = ExtraExample("/fixtures/facetable.json?_extra=debug") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -321,6 +322,7 @@ class DebugExtra(Extra): class RequestExtra(Extra): description = "Full information about the request" + example = ExtraExample("/fixtures/facetable.json?_extra=request") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 24d59577..d418d16c 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -454,9 +454,40 @@ The available table extras are listed below. ``debug`` Extra debug information + ``GET /fixtures/facetable.json?_extra=debug`` + + .. code-block:: json + + { + "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", + "url_vars": { + "database": "fixtures", + "table": "facetable", + "format": "json" + }, + "nofacet": null, + "nosuggest": null + } + ``request`` Full information about the request + ``GET /fixtures/facetable.json?_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/facetable.json?_extra=request", + "path": "/fixtures/facetable.json", + "full_path": "/fixtures/facetable.json?_extra=request", + "host": "localhost", + "args": { + "_extra": [ + "request" + ] + } + } + ``query`` Details of the underlying SQL query diff --git a/tests/test_docs.py b/tests/test_docs.py index c4e0a849..3aa67730 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -130,6 +130,19 @@ def test_render_cell_extra_example_explains_row_and_column_mapping(): assert '"render_cell": [' in section +def test_debug_and_request_extra_examples_are_documented(): + content = (docs_path / "json_api.rst").read_text() + section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + + debug_section = section.split("``debug``")[-1].split("``request``")[0] + assert "GET /fixtures/facetable.json?_extra=debug" in debug_section + assert '"url_vars": {' in debug_section + + request_section = section.split("``request``")[-1].split("``query``")[0] + assert "GET /fixtures/facetable.json?_extra=request" in request_section + assert '"full_path":' in request_section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 4d6daa175a67c4c6e895fe8b32ae051b1e9136a7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 02:56:27 -0700 Subject: [PATCH 1049/1075] Add row and query JSON extras --- datasette/extras.py | 7 + datasette/views/database.py | 46 +++- datasette/views/row.py | 67 ++---- datasette/views/table_extras.py | 258 ++++++++++++++++++--- docs/json_api.rst | 386 +++++++++++++++++++++++++++++++- docs/json_api_doc.py | 141 ++++++++---- tests/test_api.py | 22 ++ tests/test_docs.py | 16 +- tests/test_table_api.py | 49 ++++ 9 files changed, 862 insertions(+), 130 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index d867f26c..f655e517 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -16,6 +16,8 @@ def extra_names_from_request(request): class ExtraScope(Enum): TABLE = "table" + ROW = "row" + QUERY = "query" @dataclass(frozen=True) @@ -46,11 +48,16 @@ class Provider: class Extra(Provider): description: ClassVar[str | None] = None example: ClassVar[ExtraExample | None] = None + examples: ClassVar[dict[ExtraScope, ExtraExample | list[ExtraExample]]] = {} public: ClassVar[bool] = True stable: ClassVar[bool] = True expensive: ClassVar[bool] = False docs_note: ClassVar[str | None] = None + @classmethod + def example_for_scope(cls, scope): + return cls.examples.get(scope, cls.example) + @classmethod def documentation(cls): return { diff --git a/datasette/views/database.py b/datasette/views/database.py index a1647ca9..96a58758 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -11,6 +11,7 @@ import sqlite_utils import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent +from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import stored_query_to_dict @@ -38,6 +39,11 @@ from datasette.plugins import pm from .base import BaseView, DatasetteError, View, _error, stream_csv from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns +from .table_extras import ( + QueryExtraContext, + resolve_query_extras, + table_extra_registry, +) from . import Context @@ -692,6 +698,34 @@ class QueryView(View): except DatasetteError: raise + extras = extra_names_from_request(request) + metadata = None + data = {"ok": True, "rows": rows, "columns": columns} + if extras: + metadata = await datasette.get_database_metadata(database) + if stored_query: + metadata = stored_query_to_dict(stored_query) + metadata.pop("source", None) + query_extra_context = QueryExtraContext( + datasette=datasette, + request=request, + db=db, + database_name=database, + private=private, + rows=rows, + columns=columns, + sql=sql, + params=params_for_query, + query_name=stored_query.name if stored_query else None, + stored_query=stored_query, + stored_query_write=stored_query_write, + error=query_error, + metadata=metadata, + extras=extras, + extra_registry=table_extra_registry, + ) + data.update(await resolve_query_extras(extras, query_extra_context)) + # Handle formats from plugins if format_ == "csv": if not sql: @@ -721,7 +755,7 @@ class QueryView(View): error=query_error, # These will be deprecated in Datasette 1.0: args=request.args, - data={"ok": True, "rows": rows, "columns": columns}, + data=data, ) if asyncio.iscoroutine(result): result = await result @@ -770,11 +804,11 @@ class QueryView(View): ) } ) - metadata = await datasette.get_database_metadata(database) - if stored_query: - metadata = stored_query_to_dict(stored_query) - metadata.pop("source", None) - + if metadata is None: + metadata = await datasette.get_database_metadata(database) + if stored_query: + metadata = stored_query_to_dict(stored_query) + metadata.pop("source", None) renderers = {} for key, (_, can_render) in datasette.renderers.items(): it_can_render = call_with_supported_arguments( diff --git a/datasette/views/row.py b/datasette/views/row.py index 4eacfe49..3fe213d7 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -15,6 +15,7 @@ import json import markupsafe import sqlite_utils from .table import display_columns_and_rows, _get_extras +from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry class RowView(DataView): @@ -172,52 +173,26 @@ class RowView(DataView): extras.add("foreign_key_tables") # Process extras - if "foreign_key_tables" in extras: - data["foreign_key_tables"] = await self.foreign_key_tables( - database, table, pk_values - ) - - if "render_cell" in extras: - # Call render_cell plugin hook for each cell - ct_map = await self.ds.get_column_types(database, table) - rendered_rows = [] - for row in rows: - rendered_row = {} - for value, column in zip(row, columns): - ct = ct_map.get(column) - plugin_display_value = None - # Try column type render_cell first - if ct: - candidate = await ct.render_cell( - value=value, - column=column, - table=table, - database=database, - datasette=self.ds, - request=request, - ) - if candidate is not None: - plugin_display_value = candidate - if plugin_display_value is None: - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=table, - pks=resolved.pks, - database=database, - datasette=self.ds, - request=request, - column_type=ct, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - rendered_row[column] = str(plugin_display_value) - rendered_rows.append(rendered_row) - data["render_cell"] = rendered_rows + row_extra_context = RowExtraContext( + datasette=self.ds, + request=request, + resolved=resolved, + db=db, + database_name=database, + table_name=table, + private=private, + rows=rows, + columns=columns, + results_description=results.description, + pks=pks, + pk_values=pk_values, + sql=resolved.sql, + params=resolved.params, + extras=extras, + extra_registry=table_extra_registry, + foreign_key_tables=self.foreign_key_tables, + ) + data.update(await resolve_row_extras(extras, row_extra_context)) return ( data, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index e888ee9f..ec104be3 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -8,6 +8,7 @@ from datasette.resources import TableResource from datasette.utils import ( await_me_maybe, call_with_supported_arguments, + named_parameters as derive_named_parameters, path_with_added_args, path_with_format, path_with_removed_args, @@ -52,6 +53,50 @@ class TableExtraContext: extra_registry: ExtraRegistry display_columns_and_rows: object run_sequential: object + scope: ExtraScope = ExtraScope.TABLE + + +@dataclass(frozen=True) +class RowExtraContext: + datasette: object + request: object + resolved: object + db: object + database_name: str + table_name: str + private: bool + rows: list + columns: list + results_description: list + pks: list + pk_values: list + sql: str + params: dict + extras: set + extra_registry: ExtraRegistry + foreign_key_tables: object + scope: ExtraScope = ExtraScope.ROW + + +@dataclass(frozen=True) +class QueryExtraContext: + datasette: object + request: object + db: object + database_name: str + private: bool + rows: list + columns: list + sql: str | None + params: dict + query_name: str | None + stored_query: object + stored_query_write: bool + error: str | None + metadata: dict + extras: set + extra_registry: ExtraRegistry + scope: ExtraScope = ExtraScope.QUERY class CountSqlExtra(Extra): @@ -245,7 +290,15 @@ class NextUrlExtra(Extra): class ColumnsExtra(Extra): description = "Column names returned by this query" example = ExtraExample("/fixtures/facetable.json?_extra=columns") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=columns" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=columns" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.columns @@ -263,7 +316,12 @@ class AllColumnsExtra(Extra): class PrimaryKeysExtra(Extra): description = "Primary keys for this table" example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=primary_keys" + ) + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): return context.pks @@ -309,21 +367,49 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" example = ExtraExample("/fixtures/facetable.json?_extra=debug") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=debug" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=debug" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - return { - "resolved": repr(context.resolved), + debug = { "url_vars": context.request.url_vars, - "nofacet": context.nofacet, - "nosuggest": context.nosuggest, } + if context.scope == ExtraScope.TABLE: + debug["resolved"] = repr(context.resolved) + elif context.scope == ExtraScope.ROW: + debug["resolved"] = { + "table": context.table_name, + "sql": context.sql, + "params": context.params, + "pks": context.pks, + "pk_values": context.pk_values, + } + if hasattr(context, "nofacet"): + debug["nofacet"] = context.nofacet + if hasattr(context, "nosuggest"): + debug["nosuggest"] = context.nosuggest + return debug class RequestExtra(Extra): description = "Full information about the request" example = ExtraExample("/fixtures/facetable.json?_extra=request") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=request" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=request" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return { @@ -413,15 +499,48 @@ class RenderCellExtra(Extra): "whose rendered value differs from the default are included." ), ) - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + value={ + "rows": [{"id": 4, "content": "RENDER_CELL_DEMO"}], + "render_cell": [{"content": "Custom rendered HTML"}], + }, + note=( + "The ``render_cell`` array has one item for the requested row. " + "The object is keyed by column name. Only columns whose rendered " + "value differs from the default are included." + ), + ), + ExtraScope.QUERY: ExtraExample( + value={ + "rows": [{"content": "RENDER_CELL_DEMO"}], + "render_cell": [{"content": "Custom rendered HTML"}], + }, + note=( + "The ``render_cell`` array has one item per query result row, in " + "the same order as the ``rows`` array. Each object is keyed by " + "column name. Only columns whose rendered value differs from the " + "default are included." + ), + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + table_name = getattr(context, "table_name", None) + is_view = getattr(context, "is_view", False) + pks = getattr(context, "pks", []) pks_for_display = ( - context.pks if context.pks else (["rowid"] if not context.is_view else []) + pks if pks else (["rowid"] if table_name and not is_view else []) ) - col_names = [col[0] for col in context.results_description] - ct_map = await context.datasette.get_column_types( - context.database_name, context.table_name + if hasattr(context, "results_description"): + col_names = [col[0] for col in context.results_description] + else: + col_names = context.columns + ct_map = ( + await context.datasette.get_column_types(context.database_name, table_name) + if table_name + else {} ) rendered_rows = [] for row in context.rows: @@ -433,7 +552,7 @@ class RenderCellExtra(Extra): candidate = await ct.render_cell( value=value, column=column, - table=context.table_name, + table=table_name, database=context.database_name, datasette=context.datasette, request=context.request, @@ -445,7 +564,7 @@ class RenderCellExtra(Extra): row=row, value=value, column=column, - table=context.table_name, + table=table_name, pks=pks_for_display, database=context.database_name, datasette=context.datasette, @@ -465,19 +584,36 @@ class RenderCellExtra(Extra): class QueryExtra(Extra): description = "Details of the underlying SQL query" example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=query" + ), + ExtraScope.QUERY: [ + ExtraExample("/fixtures/-/query.json?sql=select+1+as+one&_extra=query"), + ExtraExample("/fixtures/neighborhood_search.json?text=town&_extra=query"), + ], + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + params = context.params + if context.scope == ExtraScope.QUERY and context.sql: + parameter_names = set(derive_named_parameters(context.sql)) + params = { + key: value + for key, value in dict(context.params).items() + if key in parameter_names + } return { "sql": context.sql, - "params": context.params, + "params": params, } class ColumnTypesExtra(Extra): description = "Column type assignments for this table" example = ExtraExample(value={}) - scopes = frozenset({ExtraScope.TABLE}) + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): ct_map = await context.datasette.get_column_types( @@ -544,11 +680,22 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): - description = "Metadata about the table and database" + description = "Metadata about the table, database or stored query" example = ExtraExample("/fixtures/facetable.json?_extra=metadata") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=metadata" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/neighborhood_search.json?text=town&_extra=metadata" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + if context.scope == ExtraScope.QUERY: + return context.metadata + tablemetadata = await context.datasette.get_resource_metadata( context.database_name, context.table_name ) @@ -572,7 +719,15 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): description = "Database name" example = ExtraExample("/fixtures/facetable.json?_extra=database") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=database" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=database" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.database_name @@ -581,7 +736,10 @@ class DatabaseExtra(Extra): class TableExtra(Extra): description = "Table name" example = ExtraExample("/fixtures/facetable.json?_extra=table") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample("/fixtures/simple_primary_key/1.json?_extra=table") + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): return context.table_name @@ -590,7 +748,15 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" example = ExtraExample("/fixtures/facetable.json?_extra=database_color") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=database_color" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=database_color" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.db.color @@ -703,6 +869,8 @@ class RenderersExtra(Extra): url_labels_extra = {} if expandable_columns: url_labels_extra = {"_labels": "on"} + table_name = getattr(context, "table_name", None) + view_name = "table" if context.scope == ExtraScope.TABLE else "database" for key, (_, can_render) in context.datasette.renderers.items(): it_can_render = call_with_supported_arguments( can_render, @@ -710,11 +878,11 @@ class RenderersExtra(Extra): columns=context.columns or [], rows=context.rows or [], sql=query.get("sql", None), - query_name=None, + query_name=getattr(context, "query_name", None), database=context.database_name, - table=context.table_name, + table=table_name, request=context.request, - view_name="table", + view_name=view_name, ) it_can_render = await await_me_maybe(it_can_render) if it_can_render: @@ -730,9 +898,17 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): - description = "Whether this table is private to the current actor" + description = "Whether this resource is private to the current actor" example = ExtraExample("/fixtures/facetable.json?_extra=private") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=private" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.private @@ -752,14 +928,27 @@ class ExpandableColumnsExtra(Extra): return expandables +class ForeignKeyTablesExtra(Extra): + description = "Tables that link to this row using foreign keys" + example = ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" + ) + scopes = frozenset({ExtraScope.ROW}) + + async def resolve(self, context): + return await context.foreign_key_tables( + context.database_name, context.table_name, context.pk_values + ) + + class ExtrasExtra(Extra): description = "Available ?_extra= blocks" - scopes = frozenset({ExtraScope.TABLE}) + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): all_extras = [ (cls.key(), cls.description) - for cls in context.extra_registry.public_classes_for_scope(ExtraScope.TABLE) + for cls in context.extra_registry.public_classes_for_scope(context.scope) ] return [ { @@ -850,6 +1039,7 @@ TABLE_EXTRA_CLASSES = [ IsViewExtra, PrivateExtra, ExpandableColumnsExtra, + ForeignKeyTablesExtra, FormHiddenArgsExtra, ] @@ -859,3 +1049,11 @@ table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) async def resolve_table_extras(extras, context): return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) + + +async def resolve_row_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.ROW) + + +async def resolve_query_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.QUERY) diff --git a/docs/json_api.rst b/docs/json_api.rst index d418d16c..379d26a0 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -237,23 +237,26 @@ query string arguments: .. _json_api_extra: -Expanding table JSON responses ------------------------------- +Expanding JSON responses +------------------------ -Table JSON responses can be expanded with one or more ``?_extra=`` parameters. +Table, row and query JSON responses can be expanded with one or more ``?_extra=`` parameters. These can be repeated or comma-separated: :: ?_extra=columns&_extra=count,next_url -The available table extras are listed below. - .. [[[cog from json_api_doc import table_extras table_extras(cog) .. ]]] +Table JSON responses +~~~~~~~~~~~~~~~~~~~~ + +The available table extras are listed below. + ``count`` Total count of rows matching these filters (May execute additional queries.) @@ -459,12 +462,12 @@ The available table extras are listed below. .. code-block:: json { - "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", "url_vars": { "database": "fixtures", "table": "facetable", "format": "json" }, + "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", "nofacet": null, "nosuggest": null } @@ -511,7 +514,7 @@ The available table extras are listed below. Column type UI metadata for this table ``metadata`` - Metadata about the table and database + Metadata about the table, database or stored query ``GET /fixtures/facetable.json?_extra=metadata`` @@ -649,7 +652,7 @@ The available table extras are listed below. true ``private`` - Whether this table is private to the current actor + Whether this resource is private to the current actor ``GET /fixtures/facetable.json?_extra=private`` @@ -697,6 +700,373 @@ The available table extras are listed below. ] ] +Row JSON responses +~~~~~~~~~~~~~~~~~~ + +The following extras are available for row JSON responses. + +``columns`` + Column names returned by this query + + ``GET /fixtures/simple_primary_key/1.json?_extra=columns`` + + .. code-block:: json + + [ + "id", + "content" + ] + +``primary_keys`` + Primary keys for this table + + ``GET /fixtures/simple_primary_key/1.json?_extra=primary_keys`` + + .. code-block:: json + + [ + "id" + ] + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + The ``render_cell`` array has one item for the requested row. The object is keyed by column name. Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + { + "rows": [ + { + "id": 4, + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + { + "content": "Custom rendered HTML" + } + ] + } + +``debug`` + Extra debug information + + ``GET /fixtures/simple_primary_key/1.json?_extra=debug`` + + .. code-block:: json + + { + "url_vars": { + "database": "fixtures", + "table": "simple_primary_key", + "pks": "1", + "format": "json" + }, + "resolved": { + "table": "simple_primary_key", + "sql": "select * from simple_primary_key where \"id\"=:p0", + "params": { + "p0": "1" + }, + "pks": [ + "id" + ], + "pk_values": [ + "1" + ] + } + } + +``request`` + Full information about the request + + ``GET /fixtures/simple_primary_key/1.json?_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/simple_primary_key/1.json?_extra=request", + "path": "/fixtures/simple_primary_key/1.json", + "full_path": "/fixtures/simple_primary_key/1.json?_extra=request", + "host": "localhost", + "args": { + "_extra": [ + "request" + ] + } + } + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/simple_primary_key/1.json?_extra=query`` + + .. code-block:: json + + { + "sql": "select * from simple_primary_key where \"id\"=:p0", + "params": { + "p0": "1" + } + } + +``column_types`` + Column type assignments for this table + + .. code-block:: json + + {} + +``metadata`` + Metadata about the table, database or stored query + + ``GET /fixtures/simple_primary_key/1.json?_extra=metadata`` + + .. code-block:: json + + { + "columns": {} + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/simple_primary_key/1.json?_extra=database`` + + .. code-block:: json + + "fixtures" + +``table`` + Table name + + ``GET /fixtures/simple_primary_key/1.json?_extra=table`` + + .. code-block:: json + + "simple_primary_key" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/simple_primary_key/1.json?_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``private`` + Whether this resource is private to the current actor + + ``GET /fixtures/simple_primary_key/1.json?_extra=private`` + + .. code-block:: json + + false + +``foreign_key_tables`` + Tables that link to this row using foreign keys + + ``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables`` + + .. code-block:: json + + [ + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f1", + "count": 1, + "link": "/fixtures/complex_foreign_keys?f1=1" + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f2", + "count": 0, + "link": "/fixtures/complex_foreign_keys?f2=1" + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f3", + "count": 1, + "link": "/fixtures/complex_foreign_keys?f3=1" + }, + { + "other_table": "foreign_key_references", + "column": "id", + "other_column": "foreign_key_with_blank_label", + "count": 0, + "link": "/fixtures/foreign_key_references?foreign_key_with_blank_label=1" + }, + { + "other_table": "foreign_key_references", + "column": "id", + "other_column": "foreign_key_with_label", + "count": 1, + "link": "/fixtures/foreign_key_references?foreign_key_with_label=1" + } + ] + +Query JSON responses +~~~~~~~~~~~~~~~~~~~~ + +The following extras are available for arbitrary SQL query responses and stored, named query responses. + +``columns`` + Column names returned by this query + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=columns`` + + .. code-block:: json + + [ + "one" + ] + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + The ``render_cell`` array has one item per query result row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + { + "rows": [ + { + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + { + "content": "Custom rendered HTML" + } + ] + } + +``debug`` + Extra debug information + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=debug`` + + .. code-block:: json + + { + "url_vars": { + "database": "fixtures", + "format": "json" + } + } + +``request`` + Full information about the request + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/-/query.json?sql=select+1+as+one&_extra=request", + "path": "/fixtures/-/query.json", + "full_path": "/fixtures/-/query.json?sql=select+1+as+one&_extra=request", + "host": "localhost", + "args": { + "sql": [ + "select 1 as one" + ], + "_extra": [ + "request" + ] + } + } + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query`` + + .. code-block:: json + + { + "sql": "select 1 as one", + "params": {} + } + + ``GET /fixtures/neighborhood_search.json?text=town&_extra=query`` + + .. code-block:: json + + { + "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities\n on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n", + "params": { + "text": "town" + } + } + +``metadata`` + Metadata about the table, database or stored query + + ``GET /fixtures/neighborhood_search.json?text=town&_extra=metadata`` + + .. code-block:: json + + { + "database": "fixtures", + "name": "neighborhood_search", + "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities\n on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n", + "title": "Search neighborhoods", + "description": null, + "description_html": null, + "hide_sql": false, + "fragment": null, + "params": [], + "parameters": [], + "is_write": false, + "is_private": false, + "is_trusted": true, + "owner_id": null, + "on_success_message": null, + "on_success_message_sql": null, + "on_success_redirect": null, + "on_error_message": null, + "on_error_redirect": null + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database`` + + .. code-block:: json + + "fixtures" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``private`` + Whether this resource is private to the current actor + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=private`` + + .. code-block:: json + + false + .. [[[end]]] .. _table_arguments: diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index 69ec6e5e..44ef4a42 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -9,39 +9,80 @@ def table_extras(cog): from datasette.extras import ExtraScope from datasette.views.table_extras import table_extra_registry - classes = table_extra_registry.public_classes_for_scope(ExtraScope.TABLE) + scopes = [ + ( + ExtraScope.TABLE, + "Table JSON responses", + "The available table extras are listed below.", + ), + ( + ExtraScope.ROW, + "Row JSON responses", + "The following extras are available for row JSON responses.", + ), + ( + ExtraScope.QUERY, + "Query JSON responses", + ( + "The following extras are available for arbitrary SQL query " + "responses and stored, named query responses." + ), + ), + ] + classes_by_scope = [ + (scope, heading, intro, table_extra_registry.public_classes_for_scope(scope)) + for scope, heading, intro in scopes + ] - live_examples = asyncio.run(_fetch_live_examples(classes)) + live_examples = asyncio.run( + _fetch_live_examples( + [ + (scope, cls) + for scope, _, _, classes in classes_by_scope + for cls in classes + ] + ) + ) cog.out("\n") - for cls in classes: - example = cls.example - description = cls.description or "" - notes = [] - if cls.expensive: - notes.append("May execute additional queries.") - if cls.docs_note: - notes.append(cls.docs_note) - if notes: - description = "{} ({})".format(description, " ".join(notes)).strip() + for scope, heading, intro, classes in classes_by_scope: + cog.out("{}\n{}\n\n".format(heading, "~" * len(heading))) + cog.out("{}\n\n".format(intro)) + for cls in classes: + examples = _examples_for_scope(cls, scope) + description = cls.description or "" + notes = [] + if cls.expensive: + notes.append("May execute additional queries.") + if cls.docs_note: + notes.append(cls.docs_note) + if notes: + description = "{} ({})".format(description, " ".join(notes)).strip() - cog.out("``{}``\n".format(cls.key())) - cog.out(" {}\n\n".format(description)) - if example is None: - continue - - if example.path: - value = live_examples[(example.path, example.key or cls.key())] - cog.out(" ``GET {}``\n\n".format(example.path)) - else: - value = example.value - if example.note: - cog.out(" {}\n\n".format(example.note)) - cog.out(" .. code-block:: json\n\n") - cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) - cog.out("\n\n") + cog.out("``{}``\n".format(cls.key())) + cog.out(" {}\n\n".format(description)) + for example in examples: + if example.path: + value = live_examples[(example.path, example.key or cls.key())] + cog.out(" ``GET {}``\n\n".format(example.path)) + else: + value = example.value + if example.note: + cog.out(" {}\n\n".format(example.note)) + cog.out(" .. code-block:: json\n\n") + cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) + cog.out("\n\n") -async def _fetch_live_examples(classes): +def _examples_for_scope(cls, scope): + examples = cls.example_for_scope(scope) + if examples is None: + return [] + if isinstance(examples, list): + return examples + return [examples] + + +async def _fetch_live_examples(scoped_classes): from datasette.app import Datasette from datasette.fixtures import write_fixture_database @@ -49,18 +90,40 @@ async def _fetch_live_examples(classes): with tempfile.TemporaryDirectory() as tmpdir: db_path = pathlib.Path(tmpdir) / "fixtures.db" write_fixture_database(db_path) - datasette = Datasette([str(db_path)], settings={"num_sql_threads": 1}) + datasette = Datasette( + [str(db_path)], + settings={"num_sql_threads": 1}, + config={ + "databases": { + "fixtures": { + "queries": { + "neighborhood_search": { + "sql": textwrap.dedent(""" + select _neighborhood, facet_cities.name, state + from facetable + join facet_cities + on facetable._city_id = facet_cities.id + where _neighborhood like '%' || :text || '%' + order by _neighborhood; + """), + "title": "Search neighborhoods", + } + } + } + } + }, + ) try: - for cls in classes: - example = cls.example - if example is None or not example.path: - continue - key = example.key or cls.key() - response = await datasette.client.get(example.path) - assert response.status_code == 200, example.path - data = response.json() - assert key in data, "{} missing from {}".format(key, example.path) - examples[(example.path, key)] = data[key] + for scope, cls in scoped_classes: + for example in _examples_for_scope(cls, scope): + if not example.path: + continue + key = example.key or cls.key() + response = await datasette.client.get(example.path) + assert response.status_code == 200, example.path + data = response.json() + assert key in data, "{} missing from {}".format(key, example.path) + examples[(example.path, key)] = data[key] finally: for db in datasette.databases.values(): if not db.is_memory: diff --git a/tests/test_api.py b/tests/test_api.py index f6187529..e1385b6f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -426,6 +426,28 @@ async def test_row_foreign_key_tables(ds_client): ] +@pytest.mark.asyncio +async def test_row_extras(ds_client): + response = await ds_client.get( + "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables" + ) + assert response.status_code == 200 + data = response.json() + assert data["database"] == "fixtures" + assert data["table"] == "simple_primary_key" + assert data["primary_keys"] == ["id"] + assert data["query"]["sql"] == 'select * from simple_primary_key where "id"=:p0' + assert data["query"]["params"] == {"p0": "1"} + assert data["request"]["path"] == "/fixtures/simple_primary_key/1.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "table": "simple_primary_key", + "pks": "1", + "format": "json", + } + assert len(data["foreign_key_tables"]) == 5 + + @pytest.mark.asyncio async def test_row_extra_render_cell(): """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook on row pages""" diff --git a/tests/test_docs.py b/tests/test_docs.py index 3aa67730..13b3a549 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -132,7 +132,7 @@ def test_render_cell_extra_example_explains_row_and_column_mapping(): def test_debug_and_request_extra_examples_are_documented(): content = (docs_path / "json_api.rst").read_text() - section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + section = content.split("Table JSON responses")[-1].split("Row JSON responses")[0] debug_section = section.split("``debug``")[-1].split("``request``")[0] assert "GET /fixtures/facetable.json?_extra=debug" in debug_section @@ -143,6 +143,20 @@ def test_debug_and_request_extra_examples_are_documented(): assert '"full_path":' in request_section +def test_row_and_query_extra_sections_are_documented(): + content = (docs_path / "json_api.rst").read_text() + assert "Row JSON responses" in content + assert ( + "``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables``" + in content + ) + assert "Query JSON responses" in content + assert "``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query``" in content + assert ( + "``GET /fixtures/neighborhood_search.json?text=town&_extra=query``" in content + ) + + @pytest.fixture(scope="session") def documented_labels(): labels = set() diff --git a/tests/test_table_api.py b/tests/test_table_api.py index eeb3dc8b..388e3979 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -68,6 +68,55 @@ async def test_table_shape_arrayfirst(ds_client): ] +@pytest.mark.asyncio +async def test_query_extras_for_arbitrary_sql(ds_client): + response = await ds_client.get( + "/fixtures/-/query.json?" + + urllib.parse.urlencode( + { + "sql": "select 1 as one", + "_extra": "columns,database,query,request,debug", + } + ) + ) + assert response.status_code == 200 + data = response.json() + assert data["rows"] == [{"one": 1}] + assert data["columns"] == ["one"] + assert data["database"] == "fixtures" + assert data["query"]["sql"] == "select 1 as one" + assert data["request"]["path"] == "/fixtures/-/query.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "format": "json", + } + + +@pytest.mark.asyncio +async def test_query_extras_for_stored_query(ds_client): + response = await ds_client.get( + "/fixtures/neighborhood_search.json?" + + urllib.parse.urlencode( + { + "text": "town", + "_extra": "columns,database,query,request,debug", + } + ) + ) + assert response.status_code == 200 + data = response.json() + assert data["columns"] == ["_neighborhood", "name", "state"] + assert data["database"] == "fixtures" + assert data["query"]["sql"].strip().startswith("select _neighborhood") + assert data["query"]["params"]["text"] == "town" + assert data["request"]["path"] == "/fixtures/neighborhood_search.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "table": "neighborhood_search", + "format": "json", + } + + @pytest.mark.asyncio async def test_table_shape_objects(ds_client): response = await ds_client.get("/fixtures/simple_primary_key.json?_shape=objects") From d8605ef4c2c054610d2f4fbf1c00d182afa617e4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 19:58:00 -0700 Subject: [PATCH 1050/1075] Fix execute_isolated_fn() against immutable databases execute_isolated_fn() always opened its temporary connection with write=True, which is not allowed for immutable databases - so APIs that rely on it, like SQL analysis when storing a query, failed. An immutable database can never receive writes, so there is no write queue to block: in that case the function now opens a read-only connection and runs it on the executor, bypassing the write thread entirely. Mutable databases keep the existing write-thread behavior. Also fixed a latent bug in the write thread where a connect() failure for an isolated task would crash the thread instead of delivering the exception back to the caller. Closes #2768 Co-Authored-By: Claude Fable 5 --- datasette/database.py | 46 +++++++++++++++++++------------- tests/test_internals_database.py | 33 +++++++++++++++++++++++ tests/test_queries.py | 34 ++++++++++++++++++++++- 3 files changed, 94 insertions(+), 19 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 0a32442c..6cd5d11e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -298,13 +298,14 @@ class Database: async def execute_isolated_fn(self, fn): self._check_not_closed() - # Open a new connection just for the duration of this function + # Open a new connection just for the duration of this function, # blocking the write queue to avoid any writes occurring during it - if self.ds.executor is None: - # non-threaded mode - isolated_connection = self.connect(write=True) + write = self.is_mutable + + def _run(): + isolated_connection = self.connect(write=write) try: - result = fn(isolated_connection) + return fn(isolated_connection) finally: isolated_connection.close() try: @@ -312,10 +313,18 @@ class Database: except ValueError: # Was probably a memory connection pass - return result - else: - # Threaded mode - send to write thread - return await self._send_to_write_thread(fn, isolated_connection=True) + + if self.ds.executor is None: + # non-threaded mode + return _run() + if not write: + # Immutable database - no writes can ever occur, so there is no + # write queue to block; run against a fresh read-only connection + return await asyncio.get_running_loop().run_in_executor( + self.ds.executor, _run + ) + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) async def analyze_sql(self, sql, params=None) -> SQLAnalysis: self._check_not_closed() @@ -449,20 +458,21 @@ class Database: if conn_exception is not None: exception = conn_exception elif task.isolated_connection: - isolated_connection = self.connect(write=True) try: - result = task.fn(isolated_connection) + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass except Exception as e: sys.stderr.write("{}\n".format(e)) sys.stderr.flush() exception = e - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass else: try: if task.transaction: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index bb209649..bad4e8ca 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -863,6 +863,39 @@ async def test_execute_isolated(db, disable_threads): assert not await db.execute_isolated_fn(table_exists_checker("created_by_isolated")) +@pytest.mark.asyncio +async def test_execute_isolated_connect_failure_does_not_kill_write_thread(): + # A connect() failure for an isolated task should be returned to the + # caller as an exception, not crash the write thread + class ConnectError(Exception): + pass + + ds = Datasette(memory=True) + db = ds.add_memory_database("test_isolated_connect_failure") + # Start the write thread with a healthy dedicated write connection + await db.execute_write("create table dogs (id integer primary key)") + + original_connect = db.connect + + def broken_connect(write=False): + raise ConnectError("Could not connect") + + db.connect = broken_connect + try: + with pytest.raises(ConnectError): + await asyncio.wait_for(db.execute_isolated_fn(lambda conn: None), timeout=2) + finally: + db.connect = original_connect + + # Write thread should still be alive and processing tasks + assert db._write_thread.is_alive() + await db.execute_write("insert into dogs (id) values (1)") + count = await db.execute_isolated_fn( + lambda conn: conn.execute("select count(*) from dogs").fetchone()[0] + ) + assert count == 1 + + @pytest.mark.asyncio async def test_analyze_sql(): ds = Datasette(memory=True) diff --git a/tests/test_queries.py b/tests/test_queries.py index 6e9bcbdb..0354f73a 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -9,7 +9,7 @@ from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, StoredQueryPage from datasette.utils.asgi import Forbidden -from datasette.utils.sqlite import supports_returning +from datasette.utils.sqlite import sqlite3, supports_returning requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" @@ -593,6 +593,38 @@ async def test_query_store_api_creates_read_only_query(): assert data["query"]["owner_id"] == "root" +@pytest.mark.asyncio +async def test_query_store_api_creates_query_for_immutable_database(tmp_path): + db_path = tmp_path / "immutable.db" + conn = sqlite3.connect(str(db_path)) + conn.execute("create table dogs (id integer primary key, name text)") + conn.commit() + conn.close() + + ds = Datasette([], immutables=[str(db_path)], default_deny=True) + ds.root_enabled = True + await ds.invoke_startup() + + response = await ds.client.post( + "/immutable/-/queries/store", + actor={"id": "root"}, + json={ + "query": { + "name": "by_name", + "sql": "select * from dogs where name = :name", + } + }, + ) + + ds.close() + assert response.status_code == 201 + data = response.json() + assert data["ok"] is True + assert data["query"]["name"] == "by_name" + assert data["query"]["parameters"] == ["name"] + assert data["query"]["is_write"] is False + + @pytest.mark.asyncio async def test_query_list_and_definition_api(): ds = Datasette(memory=True) From 3c1012dcc2995d184ea24fe70e8ccd6580592aff Mon Sep 17 00:00:00 2001 From: Viraat Das Date: Wed, 10 Jun 2026 20:15:03 -0700 Subject: [PATCH 1051/1075] Fix write query failing when a named parameter is called :sql (#2765) Closes #2761 --- .../templates/_sql_parameter_scripts.html | 34 ++++++++++++----- datasette/templates/_sql_parameters.html | 5 ++- datasette/views/execute_write.py | 2 + datasette/views/query_helpers.py | 14 ++++--- tests/test_api_write.py | 38 +++++++++++++++++++ tests/test_html.py | 4 +- tests/test_stored_queries.py | 2 +- 7 files changed, 79 insertions(+), 20 deletions(-) diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html index 159a141c..9b83889e 100644 --- a/datasette/templates/_sql_parameter_scripts.html +++ b/datasette/templates/_sql_parameter_scripts.html @@ -27,16 +27,20 @@ window.datasetteSqlParameters = (() => { manager.section .querySelectorAll("[data-parameter-control]") .forEach((control) => { - manager.parameterState.set(control.name, controlState(control)); + manager.parameterState.set( + control.dataset.parameterName, + controlState(control) + ); }); } - function createControl(parameter, id, state) { + function createControl(parameter, id, state, namePrefix) { const control = document.createElement(state.expanded ? "textarea" : "input"); control.id = id; - control.name = parameter; + control.name = `${namePrefix || ""}${parameter}`; control.value = state.value; control.setAttribute("data-parameter-control", ""); + control.dataset.parameterName = parameter; if (state.expanded) { control.rows = 5; } else { @@ -53,10 +57,16 @@ window.datasetteSqlParameters = (() => { value, selectionStart ) { - const replacement = createControl(control.name, control.id, { - value: value === undefined ? control.value : value, - expanded: expand, - }); + const parameter = control.dataset.parameterName; + const replacement = createControl( + parameter, + control.id, + { + value: value === undefined ? control.value : value, + expanded: expand, + }, + manager.namePrefix + ); button.textContent = expand ? "Collapse" : "Expand"; button.setAttribute("aria-expanded", expand ? "true" : "false"); control.replaceWith(replacement); @@ -64,7 +74,7 @@ window.datasetteSqlParameters = (() => { if (selectionStart !== undefined && replacement.setSelectionRange) { replacement.setSelectionRange(selectionStart, selectionStart); } - manager.parameterState.set(replacement.name, controlState(replacement)); + manager.parameterState.set(parameter, controlState(replacement)); } function renderParameters(manager, parameters) { @@ -99,7 +109,7 @@ window.datasetteSqlParameters = (() => { label.htmlFor = id; label.textContent = parameter; - const control = createControl(parameter, id, state); + const control = createControl(parameter, id, state, manager.namePrefix); row.append(label, control); if (manager.allowExpand) { @@ -124,7 +134,10 @@ window.datasetteSqlParameters = (() => { if (!control.matches || !control.matches("[data-parameter-control]")) { return; } - manager.parameterState.set(control.name, controlState(control)); + manager.parameterState.set( + control.dataset.parameterName, + controlState(control) + ); }); if (!manager.allowExpand) { @@ -230,6 +243,7 @@ window.datasetteSqlParameters = (() => { ? section.dataset.allowExpand === "1" : false : options.allowExpand, + namePrefix: section ? section.dataset.parameterNamePrefix || "" : "", parameterState: new Map(), }; if (section) { diff --git a/datasette/templates/_sql_parameters.html b/datasette/templates/_sql_parameters.html index 58801d40..b5c1bde8 100644 --- a/datasette/templates/_sql_parameters.html +++ b/datasette/templates/_sql_parameters.html @@ -1,9 +1,10 @@ -
+{% set sql_parameter_name_prefix = sql_parameter_name_prefix|default("") %} +
{% if parameter_names %}

Parameters

{% for parameter in parameter_names %} {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %} -

{% if sql_parameters_allow_expand|default(false) %} {% endif %}

+

{% if sql_parameters_allow_expand|default(false) %} {% endif %}

{% endfor %} {% endif %}
diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index c5d55b80..2817f56e 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -9,6 +9,7 @@ from .base import BaseView, _error from .database import display_rows as display_query_rows from .query_helpers import ( QueryValidationError, + SQL_PARAMETER_FORM_PREFIX, _analysis_is_write, _analysis_rows, _analysis_rows_with_permissions, @@ -295,6 +296,7 @@ class ExecuteWriteView(BaseView): "execute_write_columns": execute_write_columns, "execute_write_display_rows": execute_write_display_rows, "execute_write_truncated": execute_write_truncated, + "sql_parameter_name_prefix": SQL_PARAMETER_FORM_PREFIX, "execute_disabled": bool(execute_disabled_reason), "execute_disabled_reason": execute_disabled_reason, "table_columns": table_columns, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 9efe3f81..026a999f 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -49,6 +49,8 @@ _query_write_fields = { "on_error_redirect", } +SQL_PARAMETER_FORM_PREFIX = "_sql_param_" + class QueryValidationError(Exception): def __init__(self, message, status=400, *, flash=False): @@ -289,11 +291,13 @@ def _coerce_execute_write_payload(data, is_json): ) params = data.get("params") or {} else: - params = { - key: value - for key, value in data.items() - if key not in {"sql", "csrftoken", "_json"} - } + params = {} + for key, value in data.items(): + if key in {"sql", "csrftoken", "_json"}: + continue + if key.startswith(SQL_PARAMETER_FORM_PREFIX): + key = key[len(SQL_PARAMETER_FORM_PREFIX) :] + params[key] = value if not isinstance(params, dict): raise QueryValidationError("params must be a dictionary") return data.get("sql"), params diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 64f91701..b7ceb6b2 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -794,6 +794,44 @@ async def test_update_row_alter(ds_write): assert response.json() == {"ok": True} +@pytest.mark.asyncio +async def test_execute_write_form_parameter_called_sql(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_parameter_sql", name="data") + await db.execute_write("create table docs (id integer primary key, title text)") + await db.execute_write("insert into docs (id, title) values (1, 'Initial')") + await ds.invoke_startup() + + form_response = await ds.client.get( + "/data/-/execute-write", + actor={"id": "root"}, + params={"sql": "update docs set title = :sql where id = :id"}, + ) + assert form_response.status_code == 200 + assert 'data-parameter-name-prefix="_sql_param_"' in form_response.text + assert '' in form_response.text + assert 'name="_sql_param_sql"' in form_response.text + assert 'data-parameter-name="sql"' in form_response.text + assert 'name="_sql_param_id"' in form_response.text + + response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "update docs set title = :sql where id = :id", + "_sql_param_sql": "Updated", + "_sql_param_id": "1", + }, + ) + + assert response.status_code == 200 + assert "Query executed, 1 row affected" in response.text + assert (await db.execute("select title from docs where id = 1")).first()[ + 0 + ] == "Updated" + + @pytest.mark.asyncio @pytest.mark.parametrize( "input,expected_errors", diff --git a/tests/test_html.py b/tests/test_html.py index bb7f612e..20ab22bc 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -342,7 +342,7 @@ async def test_query_parameter_form_fields(ds_client): response = await ds_client.get("/fixtures/-/query?sql=select+:name") assert response.status_code == 200 assert ( - ' ' + ' ' in response.text ) assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text @@ -351,7 +351,7 @@ async def test_query_parameter_form_fields(ds_client): response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") assert response2.status_code == 200 assert ( - ' ' + ' ' in response2.text ) diff --git a/tests/test_stored_queries.py b/tests/test_stored_queries.py index 2c648d5f..46420749 100644 --- a/tests/test_stored_queries.py +++ b/tests/test_stored_queries.py @@ -201,7 +201,7 @@ def test_error_in_on_success_message_sql(stored_write_client): def test_custom_params(stored_write_client): response = stored_write_client.get("/data/update_name?extra=foo") assert ( - '' + '' in response.text ) From f4b450603559b6a6412ed67e9eb170255dd1ab6b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:49:23 -0700 Subject: [PATCH 1052/1075] Remove legacy ?_extras= row parameter The pre-1.0 ?_extras= (plural) parameter was kept for backwards compatibility with the old row JSON API. ?_extra= is the documented mechanism now that row pages share the extras registry. Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 5 ----- tests/test_api.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 3fe213d7..ce15a822 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -165,13 +165,8 @@ class RowView(DataView): "primary_key_values": pk_values, } - # Handle _extra parameter (new style) extras = _get_extras(request) - # Also support legacy _extras parameter for backward compatibility - if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): - extras.add("foreign_key_tables") - # Process extras row_extra_context = RowExtraContext( datasette=self.ds, diff --git a/tests/test_api.py b/tests/test_api.py index e1385b6f..f57d0206 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -383,7 +383,7 @@ async def test_row_strange_table_name(ds_client): @pytest.mark.asyncio async def test_row_foreign_key_tables(ds_client): response = await ds_client.get( - "/fixtures/simple_primary_key/1.json?_extras=foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" ) assert response.status_code == 200 # Foreign keys are sorted by (other_table, column, other_column) From d825d8c4f38d980356abc50c739a440585253062 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:53:41 -0700 Subject: [PATCH 1053/1075] Remove _get_extras() shim in favor of extra_names_from_request() Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 5 +++-- datasette/views/table.py | 6 +----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index ce15a822..c6721ca0 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -14,7 +14,8 @@ from datasette.plugins import pm import json import markupsafe import sqlite_utils -from .table import display_columns_and_rows, _get_extras +from datasette.extras import extra_names_from_request +from .table import display_columns_and_rows from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry @@ -165,7 +166,7 @@ class RowView(DataView): "primary_key_values": pk_values, } - extras = _get_extras(request) + extras = extra_names_from_request(request) # Process extras row_extra_context = RowExtraContext( diff --git a/datasette/views/table.py b/datasette/views/table.py index c2d520f8..1b298c50 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -854,10 +854,6 @@ class TableDropView(BaseView): return Response.json({"ok": True}, status=200) -def _get_extras(request): - return extra_names_from_request(request) - - async def _columns_to_select(table_columns, pks, request): columns = list(table_columns) if "_col" in request.args: @@ -1461,7 +1457,7 @@ async def table_view_data( rows = rows[:page_size] # Resolve extras - extras = _get_extras(request) + extras = extra_names_from_request(request) if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")): extras.add("facet_results") if request.args.get("_shape") == "object": From df8a61450b478e66c458b3f05c286daeb2c2a6b0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:57:02 -0700 Subject: [PATCH 1054/1075] Remove hasattr/getattr probing from multi-scope extras TableExtraContext, RowExtraContext and QueryExtraContext now share normalized table_name, is_view, pks and query_name fields (defaulting to None/False where inapplicable) so DebugExtra, RenderCellExtra and RenderersExtra can read them directly. RenderCellExtra uses context.columns in every scope - the table and row views both derive columns from results.description so output is unchanged. Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 1 - datasette/views/table_extras.py | 30 +++++++++++++----------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index c6721ca0..e15dfce9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -179,7 +179,6 @@ class RowView(DataView): private=private, rows=rows, columns=columns, - results_description=results.description, pks=pks, pk_values=pk_values, sql=resolved.sql, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index ec104be3..63c87a6f 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -53,6 +53,7 @@ class TableExtraContext: extra_registry: ExtraRegistry display_columns_and_rows: object run_sequential: object + query_name: str | None = None scope: ExtraScope = ExtraScope.TABLE @@ -67,7 +68,6 @@ class RowExtraContext: private: bool rows: list columns: list - results_description: list pks: list pk_values: list sql: str @@ -75,6 +75,7 @@ class RowExtraContext: extras: set extra_registry: ExtraRegistry foreign_key_tables: object + is_view: bool = False scope: ExtraScope = ExtraScope.ROW @@ -96,6 +97,9 @@ class QueryExtraContext: metadata: dict extras: set extra_registry: ExtraRegistry + table_name: str | None = None + is_view: bool = False + pks: list | None = None scope: ExtraScope = ExtraScope.QUERY @@ -383,6 +387,8 @@ class DebugExtra(Extra): } if context.scope == ExtraScope.TABLE: debug["resolved"] = repr(context.resolved) + debug["nofacet"] = context.nofacet + debug["nosuggest"] = context.nosuggest elif context.scope == ExtraScope.ROW: debug["resolved"] = { "table": context.table_name, @@ -391,10 +397,6 @@ class DebugExtra(Extra): "pks": context.pks, "pk_values": context.pk_values, } - if hasattr(context, "nofacet"): - debug["nofacet"] = context.nofacet - if hasattr(context, "nosuggest"): - debug["nosuggest"] = context.nosuggest return debug @@ -527,16 +529,10 @@ class RenderCellExtra(Extra): scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - table_name = getattr(context, "table_name", None) - is_view = getattr(context, "is_view", False) - pks = getattr(context, "pks", []) - pks_for_display = ( - pks if pks else (["rowid"] if table_name and not is_view else []) + table_name = context.table_name + pks_for_display = context.pks or ( + ["rowid"] if table_name and not context.is_view else [] ) - if hasattr(context, "results_description"): - col_names = [col[0] for col in context.results_description] - else: - col_names = context.columns ct_map = ( await context.datasette.get_column_types(context.database_name, table_name) if table_name @@ -545,7 +541,7 @@ class RenderCellExtra(Extra): rendered_rows = [] for row in context.rows: rendered_row = {} - for value, column in zip(row, col_names): + for value, column in zip(row, context.columns): ct = ct_map.get(column) plugin_display_value = None if ct: @@ -869,7 +865,7 @@ class RenderersExtra(Extra): url_labels_extra = {} if expandable_columns: url_labels_extra = {"_labels": "on"} - table_name = getattr(context, "table_name", None) + table_name = context.table_name view_name = "table" if context.scope == ExtraScope.TABLE else "database" for key, (_, can_render) in context.datasette.renderers.items(): it_can_render = call_with_supported_arguments( @@ -878,7 +874,7 @@ class RenderersExtra(Extra): columns=context.columns or [], rows=context.rows or [], sql=query.get("sql", None), - query_name=getattr(context, "query_name", None), + query_name=context.query_name, database=context.database_name, table=table_name, request=context.request, From ab62ec96d187fa05f7d672d48a0b3f962fb8c228 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:45:13 -0700 Subject: [PATCH 1055/1075] Fix _extra=private for arbitrary SQL query pages QueryView hardcoded private=False unless the request was for a stored query, so /db/-/query.json?_extra=private reported false even when execute-sql was restricted to the authenticated actor. Use check_visibility() like the table and row views do. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 6 ++++-- tests/test_table_api.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 96a58758..e6efddea 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -612,11 +612,13 @@ class QueryView(View): ) else: - await datasette.ensure_permission( + visible, private = await datasette.check_visibility( + request.actor, action="execute-sql", resource=DatabaseResource(database=database), - actor=request.actor, ) + if not visible: + raise Forbidden("execute-sql") # Flattened because of ?sql=&name1=value1&name2=value2 feature params = {key: request.args.get(key) for key in request.args} diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 388e3979..4ab2f596 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -117,6 +117,29 @@ async def test_query_extras_for_stored_query(ds_client): } +def test_query_extra_private_for_arbitrary_sql(): + with make_app_client(config={"allow_sql": {"id": "root"}}) as client: + cookies = {"ds_actor": client.actor_cookie({"id": "root"})} + response = client.get( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private", + cookies=cookies, + ) + assert response.status == 200 + assert response.json["private"] is True + # Anonymous users cannot execute SQL at all here + anon = client.get("/fixtures/-/query.json?sql=select+1+as+one") + assert anon.status == 403 + + +def test_query_extra_private_false_when_sql_is_public(): + with make_app_client() as client: + response = client.get( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" + ) + assert response.status == 200 + assert response.json["private"] is False + + @pytest.mark.asyncio async def test_table_shape_objects(ds_client): response = await ds_client.get("/fixtures/simple_primary_key.json?_shape=objects") From 8f888515b618bc0eb18e23c861b938b6bbbbf5d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:47:26 -0700 Subject: [PATCH 1056/1075] Fix _extra=query to report the params that were actually bound QueryExtra re-derived named parameters from the SQL with a regex, which missed parameters declared in a stored query's params list, reported magic _-prefixed parameters with raw querystring values that were never bound, and echoed the entire querystring when no SQL was present. QueryView now passes its named_parameter_values dict - the parameters it actually bound - through QueryExtraContext. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 2 +- datasette/views/table_extras.py | 11 +--------- tests/test_table_api.py | 38 +++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index e6efddea..a719fa4f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -717,7 +717,7 @@ class QueryView(View): rows=rows, columns=columns, sql=sql, - params=params_for_query, + params=named_parameter_values, query_name=stored_query.name if stored_query else None, stored_query=stored_query, stored_query_write=stored_query_write, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 63c87a6f..21a908a0 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -8,7 +8,6 @@ from datasette.resources import TableResource from datasette.utils import ( await_me_maybe, call_with_supported_arguments, - named_parameters as derive_named_parameters, path_with_added_args, path_with_format, path_with_removed_args, @@ -592,17 +591,9 @@ class QueryExtra(Extra): scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - params = context.params - if context.scope == ExtraScope.QUERY and context.sql: - parameter_names = set(derive_named_parameters(context.sql)) - params = { - key: value - for key, value in dict(context.params).items() - if key in parameter_names - } return { "sql": context.sql, - "params": params, + "params": context.params, } diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 4ab2f596..cfa3b512 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -131,6 +131,44 @@ def test_query_extra_private_for_arbitrary_sql(): assert anon.status == 403 +def test_query_extra_query_reports_bound_params(): + config = { + "databases": { + "fixtures": { + "queries": { + "declared_params": { + "sql": "select 1 as one", + "params": ["foo"], + }, + "magic_host": { + "sql": "select :_header_host as h", + }, + } + } + } + } + with make_app_client(config=config) as client: + # Declared parameters are reported even when the regex cannot find them + response = client.get("/fixtures/declared_params.json?foo=bar&_extra=query") + assert response.status == 200 + assert response.json["query"]["params"] == {"foo": "bar"} + # Magic parameters are bound internally and should not be reported, + # especially not as a value taken from the querystring + response = client.get( + "/fixtures/magic_host.json?_extra=query&_header_host=spoofed" + ) + assert response.status == 200 + assert response.json["rows"] == [{"h": "localhost"}] + assert response.json["query"]["params"] == {} + + +def test_query_extra_query_does_not_echo_querystring_without_sql(): + with make_app_client() as client: + response = client.get("/fixtures/-/query.json?_extra=query&foo=bar") + assert response.status == 200 + assert response.json["query"]["params"] == {} + + def test_query_extra_private_false_when_sql_is_public(): with make_app_client() as client: response = client.get( From b635dc53f42e06908c7510d743e85100a6488f22 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:50:44 -0700 Subject: [PATCH 1057/1075] Make filters, actions and display_rows extras internal These three extras return values that exist for the HTML templates - a Filters instance, an async function and markupsafe/sqlite3.Row data - so requesting them on a .json page returned a 500 serialization error, while the generated documentation and ?_extra=extras both advertised them as API surface. They are now public=False: ignored like any unknown name on JSON requests, omitted from the docs and the extras list, and still resolved for the HTML view via the new include_internal flag on ExtraRegistry.resolve(). Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 17 ++++++++--------- datasette/views/table.py | 9 ++++++++- datasette/views/table_extras.py | 12 ++++++++++-- docs/json_api.rst | 9 --------- tests/test_table_api.py | 18 ++++++++++++++++++ 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index f655e517..d5847937 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -89,7 +89,7 @@ class ExtraRegistry: def public_classes_for_scope(self, scope): return self.classes_for_scope(scope, include_internal=False) - async def resolve(self, requested, context, scope): + async def resolve(self, requested, context, scope, include_internal=False): registry = Registry() async def context_provider(): @@ -100,15 +100,14 @@ class ExtraRegistry: for cls in self.classes_for_scope(scope): registry.register(cls().resolve, name=cls.key()) - public_names = {cls.key() for cls in self.public_classes_for_scope(scope)} - requested_public_names = [ - name - for name in requested - if name in public_names and name in registry._registry - ] - resolved = await registry.resolve_multi(requested_public_names) + allowed_names = { + cls.key() + for cls in self.classes_for_scope(scope, include_internal=include_internal) + } + requested_names = [name for name in requested if name in allowed_names] + resolved = await registry.resolve_multi(requested_names) return { - name: resolved[name] for name in requested_public_names if name in resolved + name: resolved[name] for name in requested_names if name in resolved } diff --git a/datasette/views/table.py b/datasette/views/table.py index 1b298c50..3cf8e6c6 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1518,7 +1518,14 @@ async def table_view_data( "ok": True, "next": next_value and str(next_value) or None, } - data.update(await resolve_table_extras(extras, table_extra_context)) + data.update( + await resolve_table_extras( + extras, + table_extra_context, + # The HTML view needs extras that are not JSON serializable + include_internal=bool(extra_extras), + ) + ) raw_sqlite_rows = rows[:page_size] # Apply transform_value for columns with assigned types ct_map = await datasette.get_column_types(database_name, table_name) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 21a908a0..c98ae22c 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -333,6 +333,8 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): description = "Table or view actions made available by plugin hooks" scopes = frozenset({ExtraScope.TABLE}) + # Returns an async function for the HTML templates - not JSON serializable + public = False async def resolve(self, context): async def actions(): @@ -476,6 +478,8 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): description = "Row data formatted for the HTML table display" scopes = frozenset({ExtraScope.TABLE}) + # Contains markupsafe/sqlite3.Row values - not JSON serializable + public = False async def resolve(self, context, display_columns_and_rows): return display_columns_and_rows["rows"] @@ -772,6 +776,8 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): description = "Filters object used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) + # Returns a Filters instance for the HTML templates - not JSON serializable + public = False async def resolve(self, context): return context.filters @@ -1034,8 +1040,10 @@ TABLE_EXTRA_CLASSES = [ table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) -async def resolve_table_extras(extras, context): - return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) +async def resolve_table_extras(extras, context, include_internal=False): + return await table_extra_registry.resolve( + extras, context, ExtraScope.TABLE, include_internal=include_internal + ) async def resolve_row_extras(extras, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 379d26a0..6b595577 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -425,9 +425,6 @@ The available table extras are listed below. } ] -``display_rows`` - Row data formatted for the HTML table display - ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook @@ -554,12 +551,6 @@ The available table extras are listed below. "9403e5" -``actions`` - Table or view actions made available by plugin hooks - -``filters`` - Filters object used by the HTML table interface - ``renderers`` Alternative output renderers available for this table diff --git a/tests/test_table_api.py b/tests/test_table_api.py index cfa3b512..0cb67164 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -117,6 +117,24 @@ async def test_query_extras_for_stored_query(ds_client): } +@pytest.mark.parametrize("extra", ["filters", "actions", "display_rows"]) +@pytest.mark.asyncio +async def test_html_only_extras_are_not_available_via_json(ds_client, extra): + # These extras exist for the HTML view; their values are not JSON + # serializable so they are internal, not part of the JSON API + response = await ds_client.get(f"/fixtures/facetable.json?_extra={extra}") + assert response.status_code == 200 + assert extra not in response.json() + + +@pytest.mark.asyncio +async def test_html_only_extras_are_not_advertised(ds_client): + response = await ds_client.get("/fixtures/facetable.json?_extra=extras") + assert response.status_code == 200 + names = {e["name"] for e in response.json()["extras"]} + assert {"filters", "actions", "display_rows"}.isdisjoint(names) + + def test_query_extra_private_for_arbitrary_sql(): with make_app_client(config={"allow_sql": {"id": "root"}}) as client: cookies = {"ds_actor": client.actor_cookie({"id": "root"})} From bbf0424c4519441715f73ee6468e0c53cc959861 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:51:25 -0700 Subject: [PATCH 1058/1075] Changelog for row/query extras and related fixes Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 75e4f3e8..19089dd1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,13 @@ Changelog ------------------- - Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) +- Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. See :ref:`json_api_extra` for the full list. +- New generated reference documentation for every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra`. +- ``?_extra=`` values can be separated by commas as well as repeated, e.g. ``?_extra=count,next_url``. Previously a comma-separated value that included ``columns`` failed to include the ``columns`` key in the response. +- The ``?_extra=private`` extra on arbitrary SQL query pages now correctly reflects whether the SQL execution permission is private to the current actor - it previously always returned ``false``. +- The ``?_extra=query`` extra on query pages now reports the named parameters that were actually bound when the query executed, including parameters declared in a stored query's ``params`` list. Magic ``_``-prefixed parameters are no longer echoed back with unbound values taken from the querystring. +- Extras that exist to serve the HTML interface (``filters``, ``actions``, ``display_rows``) are no longer advertised or reachable through the JSON API, where requesting them previously returned a 500 serialization error. +- The pre-1.0 ``?_extras=`` (plural) parameter on row pages has been removed - use ``?_extra=foreign_key_tables`` instead. .. _v1_0_a32: From 6babd23cec9c41edd3d0ba2fab1c319905446b21 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:53:00 -0700 Subject: [PATCH 1059/1075] QueryView: only resolve extras for renderer formats, single metadata path Extras were resolved before the format dispatch, so a .csv request carrying ?_extra= parameters paid for extras (including per-cell render_cell plugin calls) whose results were then discarded, and the HTML path duplicated the stored-query metadata derivation. Extras now resolve inside the renderer-dispatch branch only, and both consumers share a query_metadata() helper that no longer fetches database metadata just to throw it away for stored queries. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 55 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index a719fa4f..ad3fb843 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -700,33 +700,12 @@ class QueryView(View): except DatasetteError: raise - extras = extra_names_from_request(request) - metadata = None - data = {"ok": True, "rows": rows, "columns": columns} - if extras: - metadata = await datasette.get_database_metadata(database) + async def query_metadata(): if stored_query: metadata = stored_query_to_dict(stored_query) metadata.pop("source", None) - query_extra_context = QueryExtraContext( - datasette=datasette, - request=request, - db=db, - database_name=database, - private=private, - rows=rows, - columns=columns, - sql=sql, - params=named_parameter_values, - query_name=stored_query.name if stored_query else None, - stored_query=stored_query, - stored_query_write=stored_query_write, - error=query_error, - metadata=metadata, - extras=extras, - extra_registry=table_extra_registry, - ) - data.update(await resolve_query_extras(extras, query_extra_context)) + return metadata + return await datasette.get_database_metadata(database) # Handle formats from plugins if format_ == "csv": @@ -740,6 +719,28 @@ class QueryView(View): return await stream_csv(datasette, fetch_data_for_csv, request, db.name) elif format_ in datasette.renderers.keys(): + data = {"ok": True, "rows": rows, "columns": columns} + extras = extra_names_from_request(request) + if extras: + query_extra_context = QueryExtraContext( + datasette=datasette, + request=request, + db=db, + database_name=database, + private=private, + rows=rows, + columns=columns, + sql=sql, + params=named_parameter_values, + query_name=stored_query.name if stored_query else None, + stored_query=stored_query, + stored_query_write=stored_query_write, + error=query_error, + metadata=await query_metadata(), + extras=extras, + extra_registry=table_extra_registry, + ) + data.update(await resolve_query_extras(extras, query_extra_context)) # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) result = call_with_supported_arguments( @@ -806,11 +807,7 @@ class QueryView(View): ) } ) - if metadata is None: - metadata = await datasette.get_database_metadata(database) - if stored_query: - metadata = stored_query_to_dict(stored_query) - metadata.pop("source", None) + metadata = await query_metadata() renderers = {} for key, (_, can_render) in datasette.renderers.items(): it_can_render = call_with_supported_arguments( From a1b6a6976d0ddafba6b927ce9fb83e62cb9091c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:55:28 -0700 Subject: [PATCH 1060/1075] Remove dead weight from the extras machinery - TableExtraContext.next_value, RowExtraContext.resolved and QueryExtraContext.stored_query/stored_query_write/error had no readers - drop the fields and the arguments that populated them - Extra.documentation() and the stable classvar were unused parallel descriptions of what the docs generator reads directly - ExtraRegistry.resolve no longer carries an always-true membership guard (resolve_multi returns every requested registered name) Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 19 +------------------ datasette/views/database.py | 3 --- datasette/views/row.py | 1 - datasette/views/table.py | 1 - datasette/views/table_extras.py | 5 ----- 5 files changed, 1 insertion(+), 28 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index d5847937..4aa93057 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -50,7 +50,6 @@ class Extra(Provider): example: ClassVar[ExtraExample | None] = None examples: ClassVar[dict[ExtraScope, ExtraExample | list[ExtraExample]]] = {} public: ClassVar[bool] = True - stable: ClassVar[bool] = True expensive: ClassVar[bool] = False docs_note: ClassVar[str | None] = None @@ -58,20 +57,6 @@ class Extra(Provider): def example_for_scope(cls, scope): return cls.examples.get(scope, cls.example) - @classmethod - def documentation(cls): - return { - "name": cls.key(), - "description": cls.description, - "scopes": [ - scope.value for scope in sorted(cls.scopes, key=lambda s: s.value) - ], - "stable": cls.stable, - "expensive": cls.expensive, - "docs_note": cls.docs_note, - "example": cls.example, - } - class ExtraRegistry: def __init__(self, classes): @@ -106,9 +91,7 @@ class ExtraRegistry: } requested_names = [name for name in requested if name in allowed_names] resolved = await registry.resolve_multi(requested_names) - return { - name: resolved[name] for name in requested_names if name in resolved - } + return {name: resolved[name] for name in requested_names} def _camel_to_snake(name): diff --git a/datasette/views/database.py b/datasette/views/database.py index ad3fb843..46e26496 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -733,9 +733,6 @@ class QueryView(View): sql=sql, params=named_parameter_values, query_name=stored_query.name if stored_query else None, - stored_query=stored_query, - stored_query_write=stored_query_write, - error=query_error, metadata=await query_metadata(), extras=extras, extra_registry=table_extra_registry, diff --git a/datasette/views/row.py b/datasette/views/row.py index e15dfce9..c300758b 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -172,7 +172,6 @@ class RowView(DataView): row_extra_context = RowExtraContext( datasette=self.ds, request=request, - resolved=resolved, db=db, database_name=database, table_name=table, diff --git a/datasette/views/table.py b/datasette/views/table.py index 3cf8e6c6..65388c9c 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1497,7 +1497,6 @@ async def table_view_data( nofacet=nofacet, nosuggest=nosuggest, next_arg=request.args.get("_next"), - next_value=next_value, next_url=next_url, sql=sql, sql_no_order_no_limit=sql_no_order_no_limit, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index c98ae22c..493135f3 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -37,7 +37,6 @@ class TableExtraContext: nofacet: object nosuggest: object next_arg: object - next_value: object next_url: str | None sql: str sql_no_order_no_limit: str @@ -60,7 +59,6 @@ class TableExtraContext: class RowExtraContext: datasette: object request: object - resolved: object db: object database_name: str table_name: str @@ -90,9 +88,6 @@ class QueryExtraContext: sql: str | None params: dict query_name: str | None - stored_query: object - stored_query_write: bool - error: str | None metadata: dict extras: set extra_registry: ExtraRegistry From cfafa5b37f5350303600e912955c43e210a113b7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:56:40 -0700 Subject: [PATCH 1061/1075] Use plain set literals for Extra scopes frozenset({...}) was immutability ceremony for class attributes that nothing mutates. scopes = {ExtraScope.TABLE} reads cleaner. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 2 +- datasette/views/table_extras.py | 74 ++++++++++++++++----------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 4aa93057..fee92939 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -30,7 +30,7 @@ class ExtraExample: class Provider: name: ClassVar[str | None] = None - scopes: ClassVar[frozenset[ExtraScope]] = frozenset() + scopes: ClassVar[set[ExtraScope]] = set() public: ClassVar[bool] = False @classmethod diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 493135f3..ce1d7bdf 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -100,7 +100,7 @@ class QueryExtraContext: class CountSqlExtra(Extra): description = "SQL query used to calculate the total count" example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.count_sql @@ -109,7 +109,7 @@ class CountSqlExtra(Extra): class CountExtra(Extra): description = "Total count of rows matching these filters" example = ExtraExample("/fixtures/facetable.json?_extra=count") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context): @@ -141,7 +141,7 @@ class CountExtra(Extra): class FacetInstancesProvider(Provider): - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, count): facet_instances = [] @@ -182,7 +182,7 @@ class FacetResultsExtra(Extra): }, note="Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context, facet_instances): @@ -217,7 +217,7 @@ class FacetsTimedOutExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, facet_results): return facet_results["timed_out"] @@ -234,7 +234,7 @@ class SuggestedFacetsExtra(Extra): ], note="Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context, facet_instances): @@ -259,7 +259,7 @@ class HumanDescriptionEnExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): human_description_en = context.filters.human_description_en( @@ -279,7 +279,7 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.next_url @@ -296,7 +296,7 @@ class ColumnsExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=columns" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.columns @@ -305,7 +305,7 @@ class ColumnsExtra(Extra): class AllColumnsExtra(Extra): description = "All columns in the table, regardless of _col/_nocol filtering" example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return list(context.table_columns) @@ -319,7 +319,7 @@ class PrimaryKeysExtra(Extra): "/fixtures/simple_primary_key/1.json?_extra=primary_keys" ) } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): return context.pks @@ -327,7 +327,7 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): description = "Table or view actions made available by plugin hooks" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Returns an async function for the HTML templates - not JSON serializable public = False @@ -358,7 +358,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): description = "Whether this resource is a view instead of a table" example = ExtraExample("/fixtures/simple_view.json?_extra=is_view") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.is_view @@ -375,7 +375,7 @@ class DebugExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=debug" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): debug = { @@ -407,7 +407,7 @@ class RequestExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=request" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return { @@ -420,7 +420,7 @@ class RequestExtra(Extra): class DisplayColumnsAndRowsProvider(Provider): - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): display_columns, display_rows = await context.display_columns_and_rows( @@ -464,7 +464,7 @@ class DisplayColumnsExtra(Extra): ], note="Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, display_columns_and_rows): return display_columns_and_rows["columns"] @@ -472,7 +472,7 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): description = "Row data formatted for the HTML table display" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Contains markupsafe/sqlite3.Row values - not JSON serializable public = False @@ -524,7 +524,7 @@ class RenderCellExtra(Extra): ), ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): table_name = context.table_name @@ -587,7 +587,7 @@ class QueryExtra(Extra): ExtraExample("/fixtures/neighborhood_search.json?text=town&_extra=query"), ], } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return { @@ -599,7 +599,7 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" example = ExtraExample(value={}) - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): ct_map = await context.datasette.get_column_types( @@ -616,7 +616,7 @@ class ColumnTypesExtra(Extra): class SetColumnTypeUiExtra(Extra): description = "Column type UI metadata for this table" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): if context.is_view: @@ -676,7 +676,7 @@ class MetadataExtra(Extra): "/fixtures/neighborhood_search.json?text=town&_extra=metadata" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): if context.scope == ExtraScope.QUERY: @@ -713,7 +713,7 @@ class DatabaseExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=database" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.database_name @@ -725,7 +725,7 @@ class TableExtra(Extra): examples = { ExtraScope.ROW: ExtraExample("/fixtures/simple_primary_key/1.json?_extra=table") } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): return context.table_name @@ -742,7 +742,7 @@ class DatabaseColorExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=database_color" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.db.color @@ -753,7 +753,7 @@ class FormHiddenArgsExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): form_hidden_args = [] @@ -770,7 +770,7 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): description = "Filters object used by the HTML table interface" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Returns a Filters instance for the HTML templates - not JSON serializable public = False @@ -781,7 +781,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return [ @@ -796,7 +796,7 @@ class SortedFacetResultsExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, facet_results): facet_configs = context.table_metadata.get("facets", []) @@ -832,7 +832,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): description = "SQL definition for this table" example = ExtraExample("/fixtures/facetable.json?_extra=table_definition") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return await context.db.get_table_definition(context.table_name) @@ -841,7 +841,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): description = "SQL definition for this view" example = ExtraExample("/fixtures/simple_view.json?_extra=view_definition") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return await context.db.get_view_definition(context.table_name) @@ -850,7 +850,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" example = ExtraExample("/fixtures/facetable.json?_extra=renderers") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, expandable_columns, query): renderers = {} @@ -896,7 +896,7 @@ class PrivateExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.private @@ -905,7 +905,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): expandables = [] @@ -921,7 +921,7 @@ class ForeignKeyTablesExtra(Extra): example = ExtraExample( "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" ) - scopes = frozenset({ExtraScope.ROW}) + scopes = {ExtraScope.ROW} async def resolve(self, context): return await context.foreign_key_tables( @@ -931,7 +931,7 @@ class ForeignKeyTablesExtra(Extra): class ExtrasExtra(Extra): description = "Available ?_extra= blocks" - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): all_extras = [ From 4edea3ad2637f4bf275f3a322e4c7747b964d907 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:04:12 -0700 Subject: [PATCH 1062/1075] Build extras registries once per scope instead of per request ExtraRegistry.resolve() previously constructed a fresh asyncinject Registry on every table, row and query request - instantiating all ~37 Extra classes and re-running inspect.signature reflection over each resolve method every time. The Extra classes are stateless, so the asyncinject Registry for each scope is now built lazily once and shared, along with the allowed-name sets. The per-request context reaches the shared registry through a contextvars.ContextVar provider rather than resolve_multi(results=...) seeding: asyncinject's parallel executor never schedules anything when the only initially-ready node is an unregistered pre-seeded value, so seeding would have stalled every resolution. asyncio tasks copy the caller's context, which keeps concurrent resolves isolated - covered by a new test. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 63 ++++++++++++++++++++++++++++++++---------- tests/test_extras.py | 65 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 tests/test_extras.py diff --git a/datasette/extras.py b/datasette/extras.py index fee92939..2c3450b2 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,3 +1,4 @@ +import contextvars import re from dataclasses import dataclass from enum import Enum @@ -5,6 +6,11 @@ from typing import ClassVar from asyncinject import Registry +# Per-request context for Extra.resolve(), so the asyncinject registries can +# be shared across requests. asyncio tasks copy the caller's context, so +# concurrent resolve() calls each see their own value. +_resolve_context = contextvars.ContextVar("datasette_extras_context") + def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") @@ -62,6 +68,13 @@ class ExtraRegistry: def __init__(self, classes): self.classes = list(classes) self.classes_by_name = {cls.key(): cls for cls in self.classes} + # Lazily-built shared state, keyed by scope. Safe to share across + # requests because Extra instances are stateless and asyncinject's + # Registry keeps per-call state local to each resolve_multi() call. + # If extras classes ever become registerable at runtime (e.g. via a + # plugin hook) these caches will need invalidating. + self._scope_registries = {} + self._allowed_names = {} def classes_for_scope(self, scope, include_internal=True): classes = [ @@ -74,23 +87,43 @@ class ExtraRegistry: def public_classes_for_scope(self, scope): return self.classes_for_scope(scope, include_internal=False) + def _registry_for_scope(self, scope): + registry = self._scope_registries.get(scope) + if registry is None: + registry = Registry() + + async def context_provider(): + return _resolve_context.get() + + registry.register(context_provider, name="context") + for cls in self.classes_for_scope(scope): + registry.register(cls().resolve, name=cls.key()) + self._scope_registries[scope] = registry + return registry + + def _allowed_names_for_scope(self, scope, include_internal): + key = (scope, include_internal) + names = self._allowed_names.get(key) + if names is None: + names = { + cls.key() + for cls in self.classes_for_scope( + scope, include_internal=include_internal + ) + } + self._allowed_names[key] = names + return names + async def resolve(self, requested, context, scope, include_internal=False): - registry = Registry() - - async def context_provider(): - return context - - registry.register(context_provider, name="context") - - for cls in self.classes_for_scope(scope): - registry.register(cls().resolve, name=cls.key()) - - allowed_names = { - cls.key() - for cls in self.classes_for_scope(scope, include_internal=include_internal) - } + allowed_names = self._allowed_names_for_scope(scope, include_internal) requested_names = [name for name in requested if name in allowed_names] - resolved = await registry.resolve_multi(requested_names) + token = _resolve_context.set(context) + try: + resolved = await self._registry_for_scope(scope).resolve_multi( + requested_names + ) + finally: + _resolve_context.reset(token) return {name: resolved[name] for name in requested_names} diff --git a/tests/test_extras.py b/tests/test_extras.py new file mode 100644 index 00000000..ad8a9f00 --- /dev/null +++ b/tests/test_extras.py @@ -0,0 +1,65 @@ +import asyncio + +import pytest + +from datasette.extras import Extra, ExtraRegistry, ExtraScope + + +class SlowValueExtra(Extra): + description = "Returns context['value'], optionally slowly" + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + if context["slow"]: + await asyncio.sleep(0.05) + return context["value"] + + +class DependentExtra(Extra): + description = "Depends on slow_value" + scopes = {ExtraScope.TABLE} + + async def resolve(self, context, slow_value): + return slow_value + 1 + + +def test_registry_is_built_once_per_scope(): + registry = ExtraRegistry([SlowValueExtra, DependentExtra]) + first = registry._registry_for_scope(ExtraScope.TABLE) + second = registry._registry_for_scope(ExtraScope.TABLE) + assert first is second + + +@pytest.mark.asyncio +async def test_concurrent_resolves_do_not_share_state(): + # The asyncinject registry is shared across requests - resolved values + # must not leak between concurrent resolve() calls with different contexts + registry = ExtraRegistry([SlowValueExtra, DependentExtra]) + slow, fast = await asyncio.gather( + registry.resolve( + {"slow_value", "dependent"}, + {"value": 100, "slow": True}, + ExtraScope.TABLE, + ), + registry.resolve( + {"slow_value", "dependent"}, + {"value": 200, "slow": False}, + ExtraScope.TABLE, + ), + ) + assert slow == {"slow_value": 100, "dependent": 101} + assert fast == {"slow_value": 200, "dependent": 201} + + +@pytest.mark.asyncio +async def test_table_row_and_query_scopes_use_separate_registries(): + from datasette.views.table_extras import table_extra_registry + + registries = { + scope: table_extra_registry._registry_for_scope(scope) for scope in ExtraScope + } + assert len(set(map(id, registries.values()))) == 3 + # Scope-specific extras only registered where they belong + assert "count" in registries[ExtraScope.TABLE]._registry + assert "count" not in registries[ExtraScope.QUERY]._registry + assert "foreign_key_tables" in registries[ExtraScope.ROW]._registry From 96226621325c5aa19e6a700efcd1b441006958fa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:15:18 -0700 Subject: [PATCH 1063/1075] Fix SQL injection via bracket escape bypass in escape_sqlite() (#2677) escape_sqlite() wrapped identifiers in [brackets] without escaping any ] characters inside the string. Since SQLite does not support escaping ] within bracket quoting, an identifier containing ] could break out and inject arbitrary SQL. Fall back to double-quote quoting (doubling any embedded ") when the identifier contains ]. Co-Authored-By: Claude Fable 5 --- datasette/utils/__init__.py | 4 ++++ tests/test_utils.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 2dff9667..55e539b9 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -410,6 +410,10 @@ def escape_css_string(s): def escape_sqlite(s): if _boring_keyword_re.match(s) and (s.lower() not in reserved_words): return s + elif "]" in s: + # SQLite does not support escaping ] inside [bracket] quoting, so fall + # back to double-quote quoting (doubling any embedded ") - #2677 + return '"{}"'.format(s.replace('"', '""')) else: return f"[{s}]" diff --git a/tests/test_utils.py b/tests/test_utils.py index 64607244..74f1963f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -216,6 +216,38 @@ def test_detect_fts(open_quote, close_quote): conn.close() +@pytest.mark.parametrize( + "identifier,expected", + ( + ("plain", "plain"), + ("select", "[select]"), + ("has space", "[has space]"), + ("has'quote", "[has'quote]"), + # Identifiers containing ] must fall back to double-quote quoting + # (SQLite does not support escaping ] inside [brackets]) - #2677 + ("has]bracket", '"has]bracket"'), + ('has"dquote]', '"has""dquote]"'), + ), +) +def test_escape_sqlite(identifier, expected): + assert utils.escape_sqlite(identifier) == expected + + +def test_escape_sqlite_prevents_injection(): + # https://github.com/simonw/datasette/issues/2677 + conn = utils.sqlite3.connect(":memory:") + conn.execute("CREATE TABLE users (id INTEGER, password TEXT)") + conn.execute("INSERT INTO users VALUES (1, 'super_secret_password')") + malicious = "users] UNION SELECT password FROM users--" + conn.execute('CREATE TABLE "{}" (id INTEGER)'.format(malicious)) + sql = "select count(*) from {}".format(utils.escape_sqlite(malicious)) + results = conn.execute(sql).fetchall() + conn.close() + # The injected UNION must not execute - only the empty malicious table + # is queried, so we get a single count row and no leaked password + assert results == [(0,)] + + @pytest.mark.parametrize("table", ("regular", "has'single quote")) def test_detect_fts_different_table_names(table): sql = """ From 1c514d69f6cc09c820c119e7bbf4dc75235e90cc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:17:16 -0700 Subject: [PATCH 1064/1075] Prevent open redirect via backslash in path (#2680) asgi_send_redirect() only collapsed leading forward slashes, so a path like /\example.com/ produced a Location of /\example.com. Browsers normalise backslashes to forward slashes, turning that into the protocol-relative //example.com and redirecting off-site. Collapse any run of leading slashes and backslashes to a single slash. Co-Authored-By: Claude Fable 5 --- datasette/utils/asgi.py | 8 +++++--- tests/test_custom_pages.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 35f243b6..55eba1bb 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -330,9 +330,11 @@ async def asgi_send_html(send, html, status=200, headers=None): async def asgi_send_redirect(send, location, status=302): - # Prevent open redirect vulnerability: strip multiple leading slashes - # //example.com would be interpreted as a protocol-relative URL (e.g., https://example.com/) - location = re.sub(r"^/+", "/", location) + # Prevent open redirect vulnerability: collapse leading slashes and + # backslashes down to a single slash. //example.com is a protocol-relative + # URL, and browsers normalise backslashes to slashes so /\example.com would + # be treated as //example.com - https://github.com/simonw/datasette/issues/2680 + location = re.sub(r"^[/\\]+", "/", location) await asgi_send( send, "", diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 39a4c06b..86cdcc6b 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -104,3 +104,24 @@ def test_custom_route_pattern_with_slash_slash_302(custom_pages_client): response = custom_pages_client.get("//example.com/") assert response.status == 302 assert response.headers["location"] == "/example.com" + + +@pytest.mark.parametrize( + "path", + ( + "/\\example.com/", + "/\\\\example.com/", + "/\\/example.com/", + ), +) +def test_redirect_does_not_allow_backslash_open_redirect(custom_pages_client, path): + # https://github.com/simonw/datasette/issues/2680 + # Browsers normalise backslashes to forward slashes, so a Location of + # /\example.com would be treated as the protocol-relative //example.com + response = custom_pages_client.get(path) + assert response.status == 302 + location = response.headers["location"] + assert location == "/example.com" + # Must not start with anything a browser reads as protocol-relative + assert not location.startswith("//") + assert not location.startswith("/\\") From c31bb55011567d13f39d8096da4aef5b5a8a720a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:24:21 -0700 Subject: [PATCH 1065/1075] Add regression test for --default-deny index 500 (#2644) datasette --default-deny --root with no config file previously 500'd on the instance and database index pages: rendering them computes is_private (include_is_private=True), which references the anon_rules CTE, but that CTE was only defined when anonymous permission rules existed. This was fixed by the empty-anon_rules fallback added in 4b5fac9c; this commit adds a regression test that fails without that fallback (SQLite "no such table: anon_rules" -> 500). Co-Authored-By: Claude Fable 5 --- tests/test_default_deny.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_default_deny.py b/tests/test_default_deny.py index 81e95b84..f1e43064 100644 --- a/tests/test_default_deny.py +++ b/tests/test_default_deny.py @@ -127,3 +127,23 @@ async def test_default_deny_basic_permissions(): # Authenticated user without explicit permission should also be denied assert await ds.allowed(action="view-instance", actor={"id": "user"}) is False + + +@pytest.mark.asyncio +async def test_default_deny_root_no_config_index_does_not_500(): + # https://github.com/simonw/datasette/issues/2644 + # --default-deny --root with no config file must not 500 on the index + # pages. Rendering those pages computes is_private (include_is_private), + # which references the anon_rules CTE - that CTE must still be defined + # even when there are no anonymous permission rules at all. + ds = Datasette(default_deny=True) + ds.root_enabled = True + await ds.invoke_startup() + db = ds.add_memory_database("test_db_2644") + await db.execute_write("create table test_table (id integer primary key)") + await ds._refresh_schemas() + + cookie = ds.sign({"a": {"id": "root"}}, "actor") + for path in ("/", "/test_db_2644", "/test_db_2644/test_table"): + response = await ds.client.get(path, cookies={"ds_actor": cookie}) + assert response.status_code == 200, f"{path} returned {response.status_code}" From d5141a5778ac5ce6a6f4cfda990b2ab556b7f9f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:27:13 -0700 Subject: [PATCH 1066/1075] Fix /-/check 500 for query actions (#2756) _check_permission_for_actor() constructed child resources with resource_class(database=parent, table=child), but QueryResource takes a "query" argument, not "table", so /-/check?action=delete-query (and view-query / update-query) raised TypeError. Construct the resource positionally so it works for any child resource class. Co-Authored-By: Claude Fable 5 --- datasette/views/special.py | 8 +++++--- tests/test_permissions.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/datasette/views/special.py b/datasette/views/special.py index 75c54c3c..aa063ad6 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -497,11 +497,13 @@ async def _check_permission_for_actor(ds, action, parent, child, actor): if action_obj.resource_class is None: resource_obj = None elif action_obj.takes_parent and action_obj.takes_child: - # Child-level resource (e.g., TableResource, QueryResource) - resource_obj = action_obj.resource_class(database=parent, table=child) + # Child-level resource (e.g., TableResource, QueryResource). The child + # argument is named differently per resource class (table, query, ...), + # so pass positionally - https://github.com/simonw/datasette/issues/2756 + resource_obj = action_obj.resource_class(parent, child) elif action_obj.takes_parent: # Parent-level resource (e.g., DatabaseResource) - resource_obj = action_obj.resource_class(database=parent) + resource_obj = action_obj.resource_class(parent) else: # This shouldn't happen given validation in Action.__post_init__ return {"error": f"Invalid action configuration: {action}"}, 500 diff --git a/tests/test_permissions.py b/tests/test_permissions.py index e5e75432..8323fe92 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1733,6 +1733,29 @@ async def test_permission_check_view_requires_debug_permission(): assert data["allowed"] is True +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ("view-query", "update-query", "delete-query")) +async def test_permission_check_view_query_actions(action): + # https://github.com/simonw/datasette/issues/2756 + # QueryResource takes a "query" argument, not "table", so /-/check must + # not assume every child resource class accepts table= + ds = Datasette() + ds.root_enabled = True + root_token = await ds.create_token("root", handler="signed") + response = await ds.client.get( + f"/-/check.json?action={action}&parent=mydb&child=myquery", + headers={"Authorization": f"Bearer {root_token}"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["action"] == action + assert data["resource"] == { + "parent": "mydb", + "child": "myquery", + "path": "/mydb/myquery", + } + + @pytest.mark.asyncio async def test_root_allow_block_with_table_restricted_actor(): """ From 154ea483eaba7a636289a6972baeced7163acd60 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:30:12 -0700 Subject: [PATCH 1067/1075] Pass columns and rows to can_render for canned queries (#2711) The HTML branch of QueryView built an empty data dict before looping over register_output_renderer can_render callbacks, so renderers that depend on the result columns or rows (e.g. datasette-atom, datasette-ics) never appeared as export options for canned queries. Populate data with the executed query's rows, columns, SQL and query name. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 10 +++++++++- tests/test_plugins.py | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index a1647ca9..66887f9b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -762,7 +762,15 @@ class QueryView(View): ) ), ) - data = {} + data = { + "ok": query_error is None, + "rows": rows, + "columns": columns, + "query": {"sql": sql, "params": params}, + "query_name": stored_query.name if stored_query else None, + "database": database, + "table": None, + } headers.update( { "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 32276437..cf753c9e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -626,6 +626,31 @@ async def test_hook_register_output_renderer_can_render(ds_client): }.items() <= ds_client.ds._can_render_saw.items() +@pytest.mark.asyncio +async def test_hook_register_output_renderer_can_render_canned_query(ds_client): + # https://github.com/simonw/datasette/issues/2711 + # can_render for a canned query must be passed the query's columns, rows + # and SQL - previously it received an empty data dict, so renderers that + # depend on the columns (datasette-atom, datasette-ics) never showed up. + response = await ds_client.get("/fixtures/pragma_cache_size") + assert response.status_code == 200 + saw = ds_client.ds._can_render_saw + assert saw["columns"] == ["cache_size"] + assert len(saw["rows"]) == 1 + assert saw["sql"] == "PRAGMA cache_size;" + assert saw["query_name"] == "pragma_cache_size" + # The renderer's export link should therefore be offered + links = ( + Soup(response.text, "html.parser") + .find("p", {"class": "export-links"}) + .find_all("a") + ) + actual = [link["href"] for link in links] + assert any( + href.startswith("/fixtures/pragma_cache_size.testall") for href in actual + ) + + @pytest.mark.asyncio async def test_hook_prepare_jinja2_environment(ds_client): ds_client.ds._HELLO = "HI" From 92848c06b8cf5b8bb6b93088bb547810f9fce8b6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:43:32 -0700 Subject: [PATCH 1068/1075] Stop facet counts from wrapping (#2754) ul.tight-bullets li uses word-break: break-all so long facet labels can wrap, but that also let the count number break across lines. Wrap each count in a span.facet-count with white-space: nowrap so the label can still wrap while the count stays on one line. Co-Authored-By: Claude Fable 5 --- datasette/static/app.css | 5 +++++ datasette/templates/_facet_results.html | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 815f6db8..6d675d9f 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -706,6 +706,11 @@ button.core[type=button] { color: #666; padding-right: 0.25em; } +/* The label may wrap (word-break: break-all on the li) but the count should + stay on one line - https://github.com/simonw/datasette/issues/2754 */ +.facet-count { + white-space: nowrap; +} .facet-info li, .facet-info ul { margin: 0; diff --git a/datasette/templates/_facet_results.html b/datasette/templates/_facet_results.html index 034e9678..570bb37e 100644 --- a/datasette/templates/_facet_results.html +++ b/datasette/templates/_facet_results.html @@ -12,9 +12,9 @@
    {% for facet_value in facet_info.results %} {% if not facet_value.selected %} -
  • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
    • +
  • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
    • {% else %} -
  • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
    • +
  • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
    • {% endif %} {% endfor %} {% if facet_info.truncated %} From 9adb5416743a7312758e986c254baa1758228ad0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 06:42:08 -0700 Subject: [PATCH 1069/1075] Use asyncinject 0.7 results= seeding for per-request extras context asyncinject 0.7 fixed the parallel executor stalling when every initially-ready node is a seeded value, and made seeded values take precedence over registered functions. That lets the shared per-scope registries receive the per-request context directly via resolve_multi(results={'context': ...}) instead of the contextvars.ContextVar workaround. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 21 +++------------------ pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 2c3450b2..5cab52a4 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,4 +1,3 @@ -import contextvars import re from dataclasses import dataclass from enum import Enum @@ -6,11 +5,6 @@ from typing import ClassVar from asyncinject import Registry -# Per-request context for Extra.resolve(), so the asyncinject registries can -# be shared across requests. asyncio tasks copy the caller's context, so -# concurrent resolve() calls each see their own value. -_resolve_context = contextvars.ContextVar("datasette_extras_context") - def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") @@ -91,11 +85,6 @@ class ExtraRegistry: registry = self._scope_registries.get(scope) if registry is None: registry = Registry() - - async def context_provider(): - return _resolve_context.get() - - registry.register(context_provider, name="context") for cls in self.classes_for_scope(scope): registry.register(cls().resolve, name=cls.key()) self._scope_registries[scope] = registry @@ -117,13 +106,9 @@ class ExtraRegistry: async def resolve(self, requested, context, scope, include_internal=False): allowed_names = self._allowed_names_for_scope(scope, include_internal) requested_names = [name for name in requested if name in allowed_names] - token = _resolve_context.set(context) - try: - resolved = await self._registry_for_scope(scope).resolve_multi( - requested_names - ) - finally: - _resolve_context.reset(token) + resolved = await self._registry_for_scope(scope).resolve_multi( + requested_names, results={"context": context} + ) return {name: resolved[name] for name in requested_names} diff --git a/pyproject.toml b/pyproject.toml index 38085476..0d136d60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "mergedeep>=1.1.1", "itsdangerous>=1.1", "sqlite-utils>=3.30", - "asyncinject>=0.6.1", + "asyncinject>=0.7", "setuptools", "pip", ] From 648a34ce8196ecf02504c0daed594bd1cd540210 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:13:07 -0700 Subject: [PATCH 1070/1075] Fix for test I broke in 92848c06 refs #2754 --- tests/test_table_html.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 2e671d55..63e233fa 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -349,7 +349,11 @@ async def test_facet_display(ds_client): { "name": a.text, "qs": a["href"].split("?")[-1], - "count": int(str(a.parent).split("")[1].split("<")[0]), + "count": int( + a.parent.find( + "span", {"class": "facet-count"} + ).text.replace(",", "") + ), } for a in div.find("ul").find_all("a") ], @@ -695,7 +699,7 @@ async def test_table_html_foreign_key_facets(ds_client): assert response.status_code == 200 assert ( '
  • - 1
    • ' + ' data-facet-value="3">- 1' ) in response.text From 26f3b20e58bd2ad582d5fae326acf22b42627eb1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:29:27 -0700 Subject: [PATCH 1071/1075] Fix to our pytest plugin to better support pytest-cov Refs https://github.com/simonw/datasette/pulls#issuecomment-4681621052 --- datasette/_pytest_plugin.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 5fb6b473..103c616d 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -19,23 +19,38 @@ import weakref import pytest -from datasette.app import Datasette - _active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( "datasette_active_instances", default=None ) -_original_init = Datasette.__init__ +_original_init = None -def _tracking_init(self, *args, **kwargs): - _original_init(self, *args, **kwargs) - instances = _active_instances.get() - if instances is not None: - instances.append(weakref.ref(self)) +def _install_tracking(): + # datasette.app is imported lazily here rather than at module level: + # as a pytest11 entry point this module is imported during pytest + # startup, before pytest-cov starts measuring, so a module-level + # import would drag in all of datasette and make every import-time + # line in the package invisible to coverage + global _original_init + if _original_init is not None: + return + from datasette.app import Datasette + + _original_init = Datasette.__init__ + + def _tracking_init(self, *args, **kwargs): + _original_init(self, *args, **kwargs) + instances = _active_instances.get() + if instances is not None: + instances.append(weakref.ref(self)) + + Datasette.__init__ = _tracking_init -Datasette.__init__ = _tracking_init +def pytest_configure(config): + if _enabled(config): + _install_tracking() def pytest_addoption(parser): From 993169ae496aa0fa30271b6cb4dfc50202f6e7c1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 08:24:37 -0700 Subject: [PATCH 1072/1075] Release 1.0a33 Refs #2735, #2677, #2680, #2711, #2756, #2761, #2768, #2754 --- datasette/version.py | 2 +- docs/changelog.rst | 40 ++++++++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index 1e8c61d5..9536d459 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a32" +__version__ = "1.0a33" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19089dd1..48bef0bf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,15 +9,43 @@ Changelog 1.0a33 (unreleased) ------------------- -- Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) -- Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. See :ref:`json_api_extra` for the full list. -- New generated reference documentation for every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra`. -- ``?_extra=`` values can be separated by commas as well as repeated, e.g. ``?_extra=count,next_url``. Previously a comma-separated value that included ``columns`` failed to include the ``columns`` key in the response. -- The ``?_extra=private`` extra on arbitrary SQL query pages now correctly reflects whether the SQL execution permission is private to the current actor - it previously always returned ``false``. -- The ``?_extra=query`` extra on query pages now reports the named parameters that were actually bound when the query executed, including parameters declared in a stored query's ``params`` list. Magic ``_``-prefixed parameters are no longer echoed back with unbound values taken from the querystring. +Stored queries can now be edited and deleted through the web interface, and the JSON API ``?_extra=`` mechanism has been extended to cover row and query pages in addition to tables. This release also fixes two security issues: an identifier-quoting bug involving table and column names that contain ``]``, and an open redirect. + +Editing and deleting stored queries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) + +``?_extra=`` support for row and query pages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. + +New generated reference documentation describes every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra` for the full list. + +You can explore the new extras using this `Datasette extras API explorer tool `__. + +Other improvements and fixes to the extras mechanism: + - Extras that exist to serve the HTML interface (``filters``, ``actions``, ``display_rows``) are no longer advertised or reachable through the JSON API, where requesting them previously returned a 500 serialization error. - The pre-1.0 ``?_extras=`` (plural) parameter on row pages has been removed - use ``?_extra=foreign_key_tables`` instead. +Security fixes +~~~~~~~~~~~~~~ + +- Fixed an identifier-quoting bug in ``datasette.utils.escape_sqlite()``. Datasette uses this helper when constructing SQL around table and column names; identifiers containing ``]`` could break out of SQLite bracket quoting and alter the generated SQL, for example by adding a ``UNION SELECT``. Identifiers containing ``]`` are now quoted using double quotes instead. (:issue:`2677`) +- Fixed an open redirect vulnerability. Requesting a path such as ``/\example.com/`` produced a redirect with a ``Location: /\example.com`` header - browsers normalize backslashes to forward slashes, turning that into the protocol-relative URL ``//example.com`` and redirecting the user off-site. Any run of leading slashes and backslashes in a redirect path is now collapsed to a single slash. (:issue:`2680`) + +Bug fixes +~~~~~~~~~ + +- ``can_render()`` callbacks registered by the :ref:`register_output_renderer() ` plugin hook now receive the result ``rows`` and ``columns`` for stored queries. Previously renderers that inspect the available columns - such as `datasette-atom `__ and `datasette-ics `__ - never appeared as export options on stored query pages. (:issue:`2711`) +- Fixed a 500 error from the :ref:`/-/check ` permission debugging endpoint when checking query actions such as ``view-query``, ``update-query`` and ``delete-query``. (:issue:`2756`) +- Write queries that use a named parameter called ``:sql`` no longer fail with an error. (:issue:`2761`) +- :ref:`db.execute_isolated_fn() ` now works against immutable databases, using a read-only connection that bypasses the write thread. It previously always attempted to open a writable connection, which would fail - breaking features built on top of it, such as the SQL analysis step used when storing a query. An exception raised while opening the connection for an isolated function no longer crashes the write thread. (:issue:`2768`) +- Facet counts are now displayed on the same line as the facet value instead of wrapping onto a second line. (:issue:`2754`) +- Datasette's pytest plugin no longer imports the rest of Datasette at pytest startup time. This means plugin test suites using ``pytest-cov`` now correctly record coverage of code that runs when ``datasette`` modules are first imported. + .. _v1_0_a32: 1.0a32 (2026-05-31) From 1d4212122e5597f2e13625193fb7d45b25928447 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 10:36:16 -0700 Subject: [PATCH 1073/1075] Add release date for 1.0a33 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 48bef0bf..c0bd7e6b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changelog .. _v1_0_a33: -1.0a33 (unreleased) +1.0a33 (2026-06-11) ------------------- Stored queries can now be edited and deleted through the web interface, and the JSON API ``?_extra=`` mechanism has been extended to cover row and query pages in addition to tables. This release also fixes two security issues: an identifier-quoting bug involving table and column names that contain ``]``, and an open redirect. From fa86ac7b11c44ef80146db6eed25d88c954ee37a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 19:41:24 -0700 Subject: [PATCH 1074/1075] Clearer examples and descriptions for JSON API extras (#2773) Review of the generated ?_extra= documentation found several extras with no example output or with examples that needed explanation: - extras: now shows an abbreviated example of the toggle list and has a clearer description (which also improves the live API output) - set_column_type_ui: example of the shape seen with set-column-type permission, plus a note that it is null otherwise - column_types: live example generated from a table with an assigned column type instead of an empty {} - metadata: live table example now demonstrates a table description and column descriptions; row and query examples gained explanatory notes - expandable_columns, foreign_key_tables, facets_timed_out, next_url, renderers: notes explaining the shape of their output Also added docs_note cross-references to the relevant documentation: facets, pagination, render_cell and register_output_renderer plugin hooks, column type configuration and API, metadata, custom templates, permissions and foreign key label expansion. foreign_key_tables is now flagged as potentially executing additional queries. https://claude.ai/code/session_01EfjBe6E817m9XNFW7EX3Vm Co-authored-by: Claude --- datasette/views/table_extras.py | 182 +++++++++++++++++++++++++-- docs/json_api.rst | 215 ++++++++++++++++++++++++++------ docs/json_api_doc.py | 19 ++- 3 files changed, 367 insertions(+), 49 deletions(-) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index ce1d7bdf..948f3daa 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -184,6 +184,7 @@ class FacetResultsExtra(Extra): ) scopes = {ExtraScope.TABLE} expensive = True + docs_note = "See :ref:`facets` for details of how facets work." async def resolve(self, context, facet_instances): facet_results = {} @@ -215,7 +216,12 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): description = "Facet calculations that timed out" example = ExtraExample( - "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" + "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out", + note=( + "A list of the names of any facets that exceeded the " + ":ref:`setting_facet_time_limit_ms` time limit - an empty list " + "if every facet calculation completed." + ), ) scopes = {ExtraScope.TABLE} @@ -236,6 +242,9 @@ class SuggestedFacetsExtra(Extra): ) scopes = {ExtraScope.TABLE} expensive = True + docs_note = ( + "Suggestions are controlled by the :ref:`setting_suggest_facets` setting." + ) async def resolve(self, context, facet_instances): suggested_facets = [] @@ -278,7 +287,13 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" - example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") + example = ExtraExample( + "/fixtures/facetable.json?_size=1&_extra=next_url", + note=( + "``null`` if there are no more pages of results. " + "See :ref:`json_api_pagination`." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -366,6 +381,10 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" + docs_note = ( + "The contents of this block are not a stable part of the Datasette " + "API and may change without warning." + ) example = ExtraExample("/fixtures/facetable.json?_extra=debug") examples = { ExtraScope.ROW: ExtraExample( @@ -482,6 +501,10 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" + docs_note = ( + "See the :ref:`render_cell() plugin hook ` " + "documentation." + ) example = ExtraExample( value={ "rows": [ @@ -598,7 +621,28 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" - example = ExtraExample(value={}) + docs_note = ( + "An empty object if no column types have been assigned. Column types " + "can be assigned in :ref:`configuration " + "` or using the :ref:`set column " + "type API `." + ) + example = ExtraExample( + "/fixtures/facetable.json?_size=0&_extra=column_types", + note=( + "This example is from an instance where the ``tags`` column has " + "been assigned the ``json`` column type." + ), + ) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/facetable/1.json?_extra=column_types", + note=( + "This example is from an instance where the ``tags`` column " + "has been assigned the ``json`` column type." + ), + ) + } scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): @@ -615,7 +659,40 @@ class ColumnTypesExtra(Extra): class SetColumnTypeUiExtra(Extra): - description = "Column type UI metadata for this table" + description = "Information needed to build an interface for assigning column types" + docs_note = ( + "``null`` unless the current actor is allowed to use the :ref:`set " + "column type API ` for this table." + ) + example = ExtraExample( + value={ + "path": "/fixtures/facetable/-/set-column-type", + "columns": { + "created": { + "current": None, + "options": [ + {"name": "email", "description": "Email address"}, + {"name": "json", "description": "JSON data"}, + {"name": "url", "description": "URL"}, + ], + }, + "tags": { + "current": {"type": "json", "config": None}, + "options": [ + {"name": "email", "description": "Email address"}, + {"name": "json", "description": "JSON data"}, + {"name": "url", "description": "URL"}, + ], + }, + }, + }, + note=( + "Shape abbreviated to two columns, as seen by an actor with " + "``set-column-type`` permission. ``current`` is the column type " + "currently assigned to each column and ``options`` lists the " + "types that could be assigned to it." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -667,13 +744,33 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): description = "Metadata about the table, database or stored query" - example = ExtraExample("/fixtures/facetable.json?_extra=metadata") + docs_note = "See :ref:`metadata` for how to attach metadata to tables." + example = ExtraExample( + "/fixtures/facetable.json?_extra=metadata", + note=( + "This example is from an instance where the ``facetable`` table " + "has a metadata ``description`` and a :ref:`column description " + "` for its ``state`` column. The " + "``columns`` object is empty for tables with no column " + "descriptions." + ), + ) examples = { ExtraScope.ROW: ExtraExample( - "/fixtures/simple_primary_key/1.json?_extra=metadata" + "/fixtures/simple_primary_key/1.json?_extra=metadata", + note=( + "This table has no metadata, so only an empty ``columns`` " + "object is returned." + ), ), ExtraScope.QUERY: ExtraExample( - "/fixtures/neighborhood_search.json?text=town&_extra=metadata" + "/fixtures/neighborhood_search.json?text=town&_extra=metadata", + note=( + "For stored queries this returns the full configuration of " + "the query, including the :ref:`stored query options " + "`. For ``?sql=`` queries it returns an " + "empty object." + ), ), } scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} @@ -733,6 +830,10 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" + docs_note = ( + "A six character hex color, without the leading ``#``, derived from " + "a hash of the database name and used in the Datasette interface." + ) example = ExtraExample("/fixtures/facetable.json?_extra=database_color") examples = { ExtraScope.ROW: ExtraExample( @@ -780,6 +881,11 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" + docs_note = ( + "The first template in this list that exists will be used to render " + "the table on the HTML version of this page. See " + ":ref:`customization_custom_templates`." + ) example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") scopes = {ExtraScope.TABLE} @@ -793,6 +899,12 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): description = "Facet results sorted for display" + docs_note = ( + "The same data as ``facet_results``, as a list in the order used by " + "the HTML interface: facets from :ref:`facet configuration " + "` first, then other facets ordered by their number " + "of results." + ) example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" ) @@ -849,7 +961,15 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" - example = ExtraExample("/fixtures/facetable.json?_extra=renderers") + example = ExtraExample( + "/fixtures/facetable.json?_extra=renderers", + note=( + "Each key is the name of an output format, each value the URL " + "for this data in that format. Plugins can add additional " + "formats using the :ref:`register_output_renderer() plugin hook " + "`." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context, expandable_columns, query): @@ -887,6 +1007,10 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): description = "Whether this resource is private to the current actor" + docs_note = ( + "``true`` if the current actor can see this resource but an " + "anonymous user could not. See :ref:`authentication_permissions`." + ) example = ExtraExample("/fixtures/facetable.json?_extra=private") examples = { ExtraScope.ROW: ExtraExample( @@ -904,7 +1028,15 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" - example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") + docs_note = "See :ref:`expand_foreign_keys` for how to expand these labels." + example = ExtraExample( + "/fixtures/facetable.json?_extra=expandable_columns", + note=( + "Each item is a ``[foreign_key, label_column]`` pair: the " + "foreign key relationship, then the column in the other table " + "that would be used as the label for each expanded value." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -919,9 +1051,14 @@ class ExpandableColumnsExtra(Extra): class ForeignKeyTablesExtra(Extra): description = "Tables that link to this row using foreign keys" example = ExtraExample( - "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables", + note=( + "``count`` is the number of rows in the other table that " + "reference this row, and ``link`` is a URL to browse those rows." + ), ) scopes = {ExtraScope.ROW} + expensive = True async def resolve(self, context): return await context.foreign_key_tables( @@ -930,7 +1067,30 @@ class ForeignKeyTablesExtra(Extra): class ExtrasExtra(Extra): - description = "Available ?_extra= blocks" + description = "List of ?_extra= blocks that can be used on this page" + example = ExtraExample( + value=[ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": False, + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": True, + }, + ], + note=( + "Shape abbreviated from /fixtures/facetable.json?_extra=extras - " + "the full response lists every extra described on this page. " + "``toggle_url`` is the current URL with that extra added or " + "removed, and ``selected`` is ``true`` for extras included in " + "the current request." + ), + ) scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 6b595577..fbc3cf60 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -276,7 +276,7 @@ The available table extras are listed below. "select count(*) from facetable " ``facet_results`` - Results of facets calculated against this data (May execute additional queries.) + Results of facets calculated against this data (May execute additional queries. See :ref:`facets` for details of how facets work.) Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results. @@ -309,12 +309,14 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out`` + A list of the names of any facets that exceeded the :ref:`setting_facet_time_limit_ms` time limit - an empty list if every facet calculation completed. + .. code-block:: json [] ``suggested_facets`` - Suggestions for facets that might return interesting results (May execute additional queries.) + Suggestions for facets that might return interesting results (May execute additional queries. Suggestions are controlled by the :ref:`setting_suggest_facets` setting.) Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets. @@ -341,6 +343,8 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` + ``null`` if there are no more pages of results. See :ref:`json_api_pagination`. + .. code-block:: json "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" @@ -426,7 +430,7 @@ The available table extras are listed below. ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item per row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -452,7 +456,7 @@ The available table extras are listed below. } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/facetable.json?_extra=debug`` @@ -501,28 +505,108 @@ The available table extras are listed below. } ``column_types`` - Column type assignments for this table + Column type assignments for this table (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) - .. code-block:: json + ``GET /fixtures/facetable.json?_size=0&_extra=column_types`` - {} - -``set_column_type_ui`` - Column type UI metadata for this table - -``metadata`` - Metadata about the table, database or stored query - - ``GET /fixtures/facetable.json?_extra=metadata`` + This example is from an instance where the ``tags`` column has been assigned the ``json`` column type. .. code-block:: json { - "columns": {} + "tags": { + "type": "json", + "config": null + } + } + +``set_column_type_ui`` + Information needed to build an interface for assigning column types (``null`` unless the current actor is allowed to use the :ref:`set column type API ` for this table.) + + Shape abbreviated to two columns, as seen by an actor with ``set-column-type`` permission. ``current`` is the column type currently assigned to each column and ``options`` lists the types that could be assigned to it. + + .. code-block:: json + + { + "path": "/fixtures/facetable/-/set-column-type", + "columns": { + "created": { + "current": null, + "options": [ + { + "name": "email", + "description": "Email address" + }, + { + "name": "json", + "description": "JSON data" + }, + { + "name": "url", + "description": "URL" + } + ] + }, + "tags": { + "current": { + "type": "json", + "config": null + }, + "options": [ + { + "name": "email", + "description": "Email address" + }, + { + "name": "json", + "description": "JSON data" + }, + { + "name": "url", + "description": "URL" + } + ] + } + } + } + +``metadata`` + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) + + ``GET /fixtures/facetable.json?_extra=metadata`` + + This example is from an instance where the ``facetable`` table has a metadata ``description`` and a :ref:`column description ` for its ``state`` column. The ``columns`` object is empty for tables with no column descriptions. + + .. code-block:: json + + { + "description": "A demo table of places, used to demonstrate facets", + "columns": { + "state": "Two letter US state code" + } } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -543,7 +627,7 @@ The available table extras are listed below. "facetable" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/facetable.json?_extra=database_color`` @@ -556,6 +640,8 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_extra=renderers`` + Each key is the name of an output format, each value the URL for this data in that format. Plugins can add additional formats using the :ref:`register_output_renderer() plugin hook `. + .. code-block:: json { @@ -563,7 +649,7 @@ The available table extras are listed below. } ``custom_table_templates`` - Custom template names considered for this table + Custom template names considered for this table (The first template in this list that exists will be used to render the table on the HTML version of this page. See :ref:`customization_custom_templates`.) ``GET /fixtures/facetable.json?_extra=custom_table_templates`` @@ -576,7 +662,7 @@ The available table extras are listed below. ] ``sorted_facet_results`` - Facet results sorted for display + Facet results sorted for display (The same data as ``facet_results``, as a list in the order used by the HTML interface: facets from :ref:`facet configuration ` first, then other facets ordered by their number of results.) ``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results`` @@ -643,7 +729,7 @@ The available table extras are listed below. true ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/facetable.json?_extra=private`` @@ -652,10 +738,12 @@ The available table extras are listed below. false ``expandable_columns`` - Foreign key columns that can be expanded with labels + Foreign key columns that can be expanded with labels (See :ref:`expand_foreign_keys` for how to expand these labels.) ``GET /fixtures/facetable.json?_extra=expandable_columns`` + Each item is a ``[foreign_key, label_column]`` pair: the foreign key relationship, then the column in the other table that would be used as the label for each expanded value. + .. code-block:: json [ @@ -720,7 +808,7 @@ The following extras are available for row JSON responses. ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item for the requested row. The object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -741,7 +829,7 @@ The following extras are available for row JSON responses. } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/simple_primary_key/1.json?_extra=debug`` @@ -803,17 +891,28 @@ The following extras are available for row JSON responses. } ``column_types`` - Column type assignments for this table + Column type assignments for this table (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) + + ``GET /fixtures/facetable/1.json?_extra=column_types`` + + This example is from an instance where the ``tags`` column has been assigned the ``json`` column type. .. code-block:: json - {} + { + "tags": { + "type": "json", + "config": null + } + } ``metadata`` - Metadata about the table, database or stored query + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) ``GET /fixtures/simple_primary_key/1.json?_extra=metadata`` + This table has no metadata, so only an empty ``columns`` object is returned. + .. code-block:: json { @@ -821,7 +920,26 @@ The following extras are available for row JSON responses. } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -842,7 +960,7 @@ The following extras are available for row JSON responses. "simple_primary_key" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/simple_primary_key/1.json?_extra=database_color`` @@ -851,7 +969,7 @@ The following extras are available for row JSON responses. "9403e5" ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/simple_primary_key/1.json?_extra=private`` @@ -860,10 +978,12 @@ The following extras are available for row JSON responses. false ``foreign_key_tables`` - Tables that link to this row using foreign keys + Tables that link to this row using foreign keys (May execute additional queries.) ``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables`` + ``count`` is the number of rows in the other table that reference this row, and ``link`` is a URL to browse those rows. + .. code-block:: json [ @@ -921,7 +1041,7 @@ The following extras are available for arbitrary SQL query responses and stored, ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item per query result row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -941,7 +1061,7 @@ The following extras are available for arbitrary SQL query responses and stored, } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=debug`` @@ -1000,10 +1120,12 @@ The following extras are available for arbitrary SQL query responses and stored, } ``metadata`` - Metadata about the table, database or stored query + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) ``GET /fixtures/neighborhood_search.json?text=town&_extra=metadata`` + For stored queries this returns the full configuration of the query, including the :ref:`stored query options `. For ``?sql=`` queries it returns an empty object. + .. code-block:: json { @@ -1029,7 +1151,26 @@ The following extras are available for arbitrary SQL query responses and stored, } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -1041,7 +1182,7 @@ The following extras are available for arbitrary SQL query responses and stored, "fixtures" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database_color`` @@ -1050,7 +1191,7 @@ The following extras are available for arbitrary SQL query responses and stored, "9403e5" ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=private`` diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index 44ef4a42..422e67f4 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -93,9 +93,26 @@ async def _fetch_live_examples(scoped_classes): datasette = Datasette( [str(db_path)], settings={"num_sql_threads": 1}, + metadata={ + "databases": { + "fixtures": { + "tables": { + "facetable": { + "description": "A demo table of places, used to demonstrate facets", + "columns": {"state": "Two letter US state code"}, + } + } + } + } + }, config={ "databases": { "fixtures": { + "tables": { + "facetable": { + "column_types": {"tags": "json"}, + } + }, "queries": { "neighborhood_search": { "sql": textwrap.dedent(""" @@ -108,7 +125,7 @@ async def _fetch_live_examples(scoped_classes): """), "title": "Search neighborhoods", } - } + }, } } }, From d4cb8b464bf1cbe69a8921fc8c9315e04a5f49cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jun 2026 13:21:58 -0700 Subject: [PATCH 1075/1075] Fix for trace_child_tasks exception handling I had Claude Fable 5 review our use of contextvar and it spotted this place where exceptions were not correctly handled. --- datasette/tracer.py | 6 ++++-- tests/test_tracer.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/datasette/tracer.py b/datasette/tracer.py index 9e66613b..28f3cc09 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -27,8 +27,10 @@ def get_task_id(): @contextmanager def trace_child_tasks(): token = trace_task_id.set(get_task_id()) - yield - trace_task_id.reset(token) + try: + yield + finally: + trace_task_id.reset(token) @contextmanager diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 6cc80fc4..9db211d3 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -70,6 +70,19 @@ def test_trace_query_errors(): assert trace_info["traces"][-1]["error"] == "no such table: non_existent_table" +@pytest.mark.asyncio +async def test_trace_child_tasks_resets_contextvar_on_exception(): + from datasette import tracer + + before = tracer.trace_task_id.get() + with pytest.raises(ValueError): + with tracer.trace_child_tasks(): + assert tracer.trace_task_id.get() is not None + raise ValueError("simulated error") + # The contextvar must be reset even though the block raised + assert tracer.trace_task_id.get() == before + + def test_trace_parallel_queries(): with make_app_client(settings={"trace_debug": True}) as client: response = client.get("/parallel-queries?_trace=1")