From a107e3a028923c1ab3911c0f880011283f93f368 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 14 Aug 2022 16:07:46 -0700 Subject: [PATCH 0001/1116] datasette-sentry is an example of handle_exception --- docs/plugin_hooks.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index aec1df56..c6f35d06 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1261,6 +1261,8 @@ This example logs an error to `Sentry `__ and then renders a return inner +Example: `datasette-sentry `_ + .. _plugin_hook_menu_links: menu_links(datasette, actor, request) From 481eb96d85291cdfa5767a83884a1525dfc382d8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 15 Aug 2022 13:17:28 -0700 Subject: [PATCH 0002/1116] https://datasette.io/tutorials/clean-data tutorial Refs #1783 --- docs/getting_started.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 571540cf..a9eaa404 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -20,6 +20,7 @@ Datasette has several `tutorials `__ to help you - `Exploring a database with Datasette `__ shows how to use the Datasette web interface to explore a new database. - `Learn SQL with Datasette `__ introduces SQL, and shows how to use that query language to ask questions of your data. +- `Cleaning data with sqlite-utils and Datasette `__ guides you through using `sqlite-utils `__ to turn a CSV file into a database that you can explore using Datasette. .. _getting_started_datasette_lite: From a3e6f1b16757fb2d39e7ddba4e09eda2362508bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Aug 2022 09:06:02 -0700 Subject: [PATCH 0003/1116] Increase height of non-JS textarea to fit query Closes #1786 --- datasette/templates/query.html | 3 ++- tests/test_html.py | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/datasette/templates/query.html b/datasette/templates/query.html index cee779fc..a35e3afe 100644 --- a/datasette/templates/query.html +++ b/datasette/templates/query.html @@ -45,7 +45,8 @@ {% endif %} {% if not hide_sql %} {% if editable and allow_execute_sql %} -

+

{% else %} 
{% if query %}{{ query.sql }}{% endif %}
{% endif %} diff --git a/tests/test_html.py b/tests/test_html.py index 409fec68..be21bd84 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -695,10 +695,8 @@ def test_query_error(app_client): response = app_client.get("/fixtures?sql=select+*+from+notatable") html = response.text assert '

no such table: notatable

' in html - assert ( - '' - in html - ) + assert '" in html assert "0 results" not in html From 09a41662e70b788469157bb58ed9ca4acdf2f904 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Aug 2022 09:10:48 -0700 Subject: [PATCH 0004/1116] Fix typo --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index c6f35d06..30bd75b7 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -874,7 +874,7 @@ canned_queries(datasette, database, actor) ``actor`` - dictionary or None The currently authenticated :ref:`actor `. -Ues this hook to return a dictionary of additional :ref:`canned query ` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query ` documentation. +Use this hook to return a dictionary of additional :ref:`canned query ` definitions for the specified database. The return value should be the same shape as the JSON described in the :ref:`canned query ` documentation. .. code-block:: python From 6c0ba7c00c2ae3ecbb5309efa59079cea1c850b3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Aug 2022 14:52:04 -0700 Subject: [PATCH 0005/1116] Improved CLI reference documentation, refs #1787 --- datasette/cli.py | 2 +- docs/changelog.rst | 2 +- docs/cli-reference.rst | 325 ++++++++++++++++++++++++++++++--------- docs/getting_started.rst | 50 ------ docs/index.rst | 2 +- docs/publish.rst | 2 + 6 files changed, 259 insertions(+), 124 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 8781747c..f2a03d53 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -282,7 +282,7 @@ def package( port, **extra_metadata, ): - """Package specified SQLite files into a new datasette Docker container""" + """Package SQLite files into a Datasette Docker container""" if not shutil.which("docker"): click.secho( ' The package command requires "docker" to be installed and configured ', diff --git a/docs/changelog.rst b/docs/changelog.rst index 1225c63f..f9dcc980 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -621,7 +621,7 @@ See also `Datasette 0.49: The annotated release notes `__ for conversations about the project that go beyond just bug reports and issues. - Datasette can now be installed on macOS using Homebrew! Run ``brew install simonw/datasette/datasette``. See :ref:`installation_homebrew`. (:issue:`335`) - Two new commands: ``datasette install name-of-plugin`` and ``datasette uninstall name-of-plugin``. These are equivalent to ``pip install`` and ``pip uninstall`` but automatically run in the same virtual environment as Datasette, so users don't have to figure out where that virtual environment is - useful for installations created using Homebrew or ``pipx``. See :ref:`plugins_installing`. (:issue:`925`) -- A new command-line option, ``datasette --get``, accepts a path to a URL within the Datasette instance. It will run that request through Datasette (without starting a web server) and print out the response. See :ref:`getting_started_datasette_get` for an example. (:issue:`926`) +- A new command-line option, ``datasette --get``, accepts a path to a URL within the Datasette instance. It will run that request through Datasette (without starting a web server) and print out the response. See :ref:`cli_datasette_get` for an example. (:issue:`926`) .. _v0_46: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 415af13c..a1e56774 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -4,44 +4,34 @@ CLI reference =============== -This page lists the ``--help`` for every ``datasette`` CLI command. +The ``datasette`` CLI tool provides a number of commands. + +Running ``datasette`` without specifying a command runs the default command, ``datasette serve``. See :ref:`cli_help_serve___help` for the full list of options for that command. .. [[[cog from datasette import cli from click.testing import CliRunner import textwrap - commands = [ - ["--help"], - ["serve", "--help"], - ["serve", "--help-settings"], - ["plugins", "--help"], - ["publish", "--help"], - ["publish", "cloudrun", "--help"], - ["publish", "heroku", "--help"], - ["package", "--help"], - ["inspect", "--help"], - ["install", "--help"], - ["uninstall", "--help"], - ] - cog.out("\n") - for command in commands: - title = "datasette " + " ".join(command) - ref = "_cli_help_" + ("_".join(command).replace("-", "_")) - cog.out(".. {}:\n\n".format(ref)) - cog.out(title + "\n") - cog.out(("=" * len(title)) + "\n\n") + def help(args): + title = "datasette " + " ".join(args) cog.out("::\n\n") - result = CliRunner().invoke(cli.cli, command) + result = CliRunner().invoke(cli.cli, args) output = result.output.replace("Usage: cli ", "Usage: datasette ") cog.out(textwrap.indent(output, ' ')) cog.out("\n\n") .. ]]] +.. [[[end]]] .. _cli_help___help: datasette --help ================ +Running ``datasette --help`` shows a list of all of the available commands. + +.. [[[cog + help(["--help"]) +.. ]]] :: Usage: datasette [OPTIONS] COMMAND [ARGS]... @@ -59,17 +49,34 @@ datasette --help serve* Serve up specified SQLite database files with a web UI inspect Generate JSON summary of provided database files install Install plugins and packages from PyPI into the same... - package Package specified SQLite files into a new datasette Docker... + package Package SQLite files into a Datasette Docker container plugins List currently installed plugins publish Publish specified SQLite database files to the internet along... uninstall Uninstall plugins and Python packages from the Datasette... +.. [[[end]]] + +Additional commands added by plugins that use the :ref:`plugin_hook_register_commands` hook will be listed here as well. + .. _cli_help_serve___help: -datasette serve --help -====================== +datasette serve +=============== +This command starts the Datasette web application running on your machine:: + + datasette serve mydatabase.db + +Or since this is the default command you can run this instead:: + + datasette mydatabase.db + +Once started you can access it at ``http://localhost:8001`` + +.. [[[cog + help(["serve", "--help"]) +.. ]]] :: Usage: datasette serve [OPTIONS] [FILES]... @@ -121,11 +128,75 @@ datasette serve --help --help Show this message and exit. +.. [[[end]]] + + +.. _cli_datasette_get: + +datasette --get +--------------- + +The ``--get`` option to ``datasette serve`` (or just ``datasette``) specifies the path to a page within Datasette and causes Datasette to output the content from that path without starting the web server. + +This means that all of Datasette's functionality can be accessed directly from the command-line. + +For example:: + + $ datasette --get '/-/versions.json' | jq . + { + "python": { + "version": "3.8.5", + "full": "3.8.5 (default, Jul 21 2020, 10:48:26) \n[Clang 11.0.3 (clang-1103.0.32.62)]" + }, + "datasette": { + "version": "0.46+15.g222a84a.dirty" + }, + "asgi": "3.0", + "uvicorn": "0.11.8", + "sqlite": { + "version": "3.32.3", + "fts_versions": [ + "FTS5", + "FTS4", + "FTS3" + ], + "extensions": { + "json1": null + }, + "compile_options": [ + "COMPILER=clang-11.0.3", + "ENABLE_COLUMN_METADATA", + "ENABLE_FTS3", + "ENABLE_FTS3_PARENTHESIS", + "ENABLE_FTS4", + "ENABLE_FTS5", + "ENABLE_GEOPOLY", + "ENABLE_JSON1", + "ENABLE_PREUPDATE_HOOK", + "ENABLE_RTREE", + "ENABLE_SESSION", + "MAX_VARIABLE_NUMBER=250000", + "THREADSAFE=1" + ] + } + } + +The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. + +This lets you use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions. + .. _cli_help_serve___help_settings: datasette serve --help-settings -=============================== +------------------------------- +This command outputs all of the available Datasette :ref:`settings `. + +These can be passed to ``datasette serve`` using ``datasette serve --setting name value``. + +.. [[[cog + help(["--help-settings"]) +.. ]]] :: Settings: @@ -170,11 +241,18 @@ datasette serve --help-settings +.. [[[end]]] + .. _cli_help_plugins___help: -datasette plugins --help -======================== +datasette plugins +================= +Output JSON showing all currently installed plugins, their versions, whether they include static files or templates and which :ref:`plugin_hooks` they use. + +.. [[[cog + help(["plugins", "--help"]) +.. ]]] :: Usage: datasette plugins [OPTIONS] @@ -187,11 +265,110 @@ datasette plugins --help --help Show this message and exit. +.. [[[end]]] + +Example output: + +.. code-block:: json + + [ + { + "name": "datasette-geojson", + "static": false, + "templates": false, + "version": "0.3.1", + "hooks": [ + "register_output_renderer" + ] + }, + { + "name": "datasette-geojson-map", + "static": true, + "templates": false, + "version": "0.4.0", + "hooks": [ + "extra_body_script", + "extra_css_urls", + "extra_js_urls" + ] + }, + { + "name": "datasette-leaflet", + "static": true, + "templates": false, + "version": "0.2.2", + "hooks": [ + "extra_body_script", + "extra_template_vars" + ] + } + ] + + +.. _cli_help_install___help: + +datasette install +================= + +Install new Datasette plugins. This command works like ``pip install`` but ensures that your plugins will be installed into the same environment as Datasette. + +This command:: + + datasette install datasette-cluster-map + +Would install the `datasette-cluster-map `__ plugin. + +.. [[[cog + help(["install", "--help"]) +.. ]]] +:: + + Usage: datasette install [OPTIONS] PACKAGES... + + Install plugins and packages from PyPI into the same environment as Datasette + + Options: + -U, --upgrade Upgrade packages to latest version + --help Show this message and exit. + + +.. [[[end]]] + +.. _cli_help_uninstall___help: + +datasette uninstall +=================== + +Uninstall one or more plugins. + +.. [[[cog + help(["uninstall", "--help"]) +.. ]]] +:: + + Usage: datasette uninstall [OPTIONS] PACKAGES... + + Uninstall plugins and Python packages from the Datasette environment + + Options: + -y, --yes Don't ask for confirmation + --help Show this message and exit. + + +.. [[[end]]] + .. _cli_help_publish___help: -datasette publish --help -======================== +datasette publish +================= +Shows a list of available deployment targets for :ref:`publishing data ` with Datasette. + +Additional deployment targets can be added by plugins that use the :ref:`plugin_hook_publish_subcommand` hook. + +.. [[[cog + help(["publish", "--help"]) +.. ]]] :: Usage: datasette publish [OPTIONS] COMMAND [ARGS]... @@ -207,11 +384,19 @@ datasette publish --help heroku Publish databases to Datasette running on Heroku +.. [[[end]]] + + .. _cli_help_publish_cloudrun___help: -datasette publish cloudrun --help -================================= +datasette publish cloudrun +========================== +See :ref:`publish_cloud_run`. + +.. [[[cog + help(["publish", "cloudrun", "--help"]) +.. ]]] :: Usage: datasette publish cloudrun [OPTIONS] [FILES]... @@ -256,11 +441,19 @@ datasette publish cloudrun --help --help Show this message and exit. +.. [[[end]]] + + .. _cli_help_publish_heroku___help: -datasette publish heroku --help -=============================== +datasette publish heroku +======================== +See :ref:`publish_heroku`. + +.. [[[cog + help(["publish", "heroku", "--help"]) +.. ]]] :: Usage: datasette publish heroku [OPTIONS] [FILES]... @@ -297,16 +490,23 @@ datasette publish heroku --help --help Show this message and exit. +.. [[[end]]] + .. _cli_help_package___help: -datasette package --help -======================== +datasette package +================= +Package SQLite files into a Datasette Docker container, see :ref:`cli_package`. + +.. [[[cog + help(["package", "--help"]) +.. ]]] :: Usage: datasette package [OPTIONS] FILES... - Package specified SQLite files into a new datasette Docker container + Package SQLite files into a Datasette Docker container Options: -t, --tag TEXT Name for the resulting Docker container, can @@ -335,11 +535,26 @@ datasette package --help --help Show this message and exit. +.. [[[end]]] + + .. _cli_help_inspect___help: -datasette inspect --help -======================== +datasette inspect +================= +Outputs JSON representing introspected data about one or more SQLite database files. + +If you are opening an immutable database, you can pass this file to the ``--inspect-data`` option to improve Datasette's performance by allowing it to skip running row counts against the database when it first starts running:: + + datasette inspect mydatabase.db > inspect-data.json + datasette serve -i mydatabase.db --inspect-file inspect-data.json + +This performance optimization is used automatically by some of the ``datasette publish`` commands. You are unlikely to need to apply this optimization manually. + +.. [[[cog + help(["inspect", "--help"]) +.. ]]] :: Usage: datasette inspect [OPTIONS] [FILES]... @@ -355,36 +570,4 @@ datasette inspect --help --help Show this message and exit. -.. _cli_help_install___help: - -datasette install --help -======================== - -:: - - Usage: datasette install [OPTIONS] PACKAGES... - - Install plugins and packages from PyPI into the same environment as Datasette - - Options: - -U, --upgrade Upgrade packages to latest version - --help Show this message and exit. - - -.. _cli_help_uninstall___help: - -datasette uninstall --help -========================== - -:: - - Usage: datasette uninstall [OPTIONS] PACKAGES... - - Uninstall plugins and Python packages from the Datasette environment - - Options: - -y, --yes Don't ask for confirmation - --help Show this message and exit. - - .. [[[end]]] diff --git a/docs/getting_started.rst b/docs/getting_started.rst index a9eaa404..6515ef8d 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -138,53 +138,3 @@ JSON in a more convenient format: } ] } - -.. _getting_started_datasette_get: - -datasette --get ---------------- - -The ``--get`` option can specify the path to a page within Datasette and cause Datasette to output the content from that path without starting the web server. This means that all of Datasette's functionality can be accessed directly from the command-line. For example:: - - $ datasette --get '/-/versions.json' | jq . - { - "python": { - "version": "3.8.5", - "full": "3.8.5 (default, Jul 21 2020, 10:48:26) \n[Clang 11.0.3 (clang-1103.0.32.62)]" - }, - "datasette": { - "version": "0.46+15.g222a84a.dirty" - }, - "asgi": "3.0", - "uvicorn": "0.11.8", - "sqlite": { - "version": "3.32.3", - "fts_versions": [ - "FTS5", - "FTS4", - "FTS3" - ], - "extensions": { - "json1": null - }, - "compile_options": [ - "COMPILER=clang-11.0.3", - "ENABLE_COLUMN_METADATA", - "ENABLE_FTS3", - "ENABLE_FTS3_PARENTHESIS", - "ENABLE_FTS4", - "ENABLE_FTS5", - "ENABLE_GEOPOLY", - "ENABLE_JSON1", - "ENABLE_PREUPDATE_HOOK", - "ENABLE_RTREE", - "ENABLE_SESSION", - "MAX_VARIABLE_NUMBER=250000", - "THREADSAFE=1" - ] - } - } - -The exit code will be 0 if the request succeeds and 1 if the request produced an HTTP status code other than 200 - e.g. a 404 or 500 error. This means you can use ``datasette --get /`` to run tests against a Datasette application in a continuous integration environment such as GitHub Actions. - -Running ``datasette`` without specifying a command runs the default command, ``datasette serve``. See :ref:`cli_help_serve___help` for the full list of options for that command. diff --git a/docs/index.rst b/docs/index.rst index efe196b3..5a9cc7ed 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,6 +40,7 @@ Contents getting_started installation ecosystem + cli-reference pages publish deploying @@ -61,6 +62,5 @@ Contents plugin_hooks testing_plugins internals - cli-reference contributing changelog diff --git a/docs/publish.rst b/docs/publish.rst index 9c7c99cc..dd8566ed 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -56,6 +56,8 @@ Cloud Run provides a URL on the ``.run.app`` domain, but you can also point your See :ref:`cli_help_publish_cloudrun___help` for the full list of options for this command. +.. _publish_heroku: + Publishing to Heroku -------------------- From aff3df03d4fe0806ce432d1818f6643cdb2a854e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Aug 2022 14:55:08 -0700 Subject: [PATCH 0006/1116] Ignore ro which stands for read only Refs #1787 where it caused tests to break --- docs/codespell-ignore-words.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/codespell-ignore-words.txt b/docs/codespell-ignore-words.txt index a625cde5..d6744d05 100644 --- a/docs/codespell-ignore-words.txt +++ b/docs/codespell-ignore-words.txt @@ -1 +1 @@ -AddWordsToIgnoreHere +ro From 0d9d33955b503c88a2c712144d97f094baa5d46d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 18 Aug 2022 16:06:12 -0700 Subject: [PATCH 0007/1116] Clarify you can publish multiple files, closes #1788 --- docs/publish.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/publish.rst b/docs/publish.rst index dd8566ed..d817ed31 100644 --- a/docs/publish.rst +++ b/docs/publish.rst @@ -31,7 +31,7 @@ Publishing to Google Cloud Run You will first need to install and configure the Google Cloud CLI tools by following `these instructions `__. -You can then publish a database to Google Cloud Run using the following command:: +You can then publish one or more SQLite database files to Google Cloud Run using the following command:: datasette publish cloudrun mydatabase.db --service=my-database @@ -63,7 +63,7 @@ Publishing to Heroku To publish your data using `Heroku `__, first create an account there and install and configure the `Heroku CLI tool `_. -You can publish a database to Heroku using the following command:: +You can publish one or more databases to Heroku using the following command:: datasette publish heroku mydatabase.db @@ -138,7 +138,7 @@ If a plugin has any :ref:`plugins_configuration_secret` you can use the ``--plug datasette package ================= -If you have docker installed (e.g. using `Docker for Mac `_) you can use the ``datasette package`` command to create a new Docker image in your local repository containing the datasette app bundled together with your selected SQLite databases:: +If you have docker installed (e.g. using `Docker for Mac `_) you can use the ``datasette package`` command to create a new Docker image in your local repository containing the datasette app bundled together with one or more SQLite databases:: datasette package mydatabase.db From 663ac431fe7202c85967568d82b2034f92b9aa43 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sat, 20 Aug 2022 02:04:16 +0200 Subject: [PATCH 0008/1116] Use Read the Docs action v1 (#1778) Read the Docs repository was renamed from `readthedocs/readthedocs-preview` to `readthedocs/actions/`. Now, the `preview` action is under `readthedocs/actions/preview` and is tagged as `v1` --- .github/workflows/documentation-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index e7062a46..a54bd83a 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -11,6 +11,6 @@ jobs: documentation-links: runs-on: ubuntu-latest steps: - - uses: readthedocs/readthedocs-preview@main + - uses: readthedocs/actions/preview@v1 with: project-slug: "datasette" From 1d64c9a8dac45b9a3452acf8e76dfadea2b0bc49 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Tue, 23 Aug 2022 11:34:30 -0700 Subject: [PATCH 0009/1116] Add new entrypoint option to --load-extensions. (#1789) Thanks, @asg017 --- .gitignore | 6 ++++ datasette/app.py | 8 ++++- datasette/cli.py | 4 ++- datasette/utils/__init__.py | 11 ++++++ tests/ext.c | 48 ++++++++++++++++++++++++++ tests/test_load_extensions.py | 65 +++++++++++++++++++++++++++++++++++ 6 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 tests/ext.c create mode 100644 tests/test_load_extensions.py diff --git a/.gitignore b/.gitignore index 066009f0..277ff653 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,9 @@ ENV/ .DS_Store node_modules .*.swp + +# In case someone compiled tests/ext.c for test_load_extensions, don't +# include it in source control. +tests/*.dylib +tests/*.so +tests/*.dll \ No newline at end of file diff --git a/datasette/app.py b/datasette/app.py index 1a9afc10..bb9232c9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -559,7 +559,13 @@ class Datasette: if self.sqlite_extensions: conn.enable_load_extension(True) for extension in self.sqlite_extensions: - conn.execute("SELECT load_extension(?)", [extension]) + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. + if isinstance(extension, tuple): + path, entrypoint = extension + conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) + else: + conn.execute("SELECT load_extension(?)", [extension]) if self.setting("cache_size_kb"): conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}") # pylint: disable=no-member diff --git a/datasette/cli.py b/datasette/cli.py index f2a03d53..6eb42712 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -21,6 +21,7 @@ from .app import ( pm, ) from .utils import ( + LoadExtension, StartupError, check_connection, find_spatialite, @@ -128,9 +129,10 @@ def sqlite_extensions(fn): return click.option( "sqlite_extensions", "--load-extension", + type=LoadExtension(), envvar="SQLITE_EXTENSIONS", multiple=True, - help="Path to a SQLite extension to load", + help="Path to a SQLite extension to load, and optional entrypoint", )(fn) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index d148cc2c..0fc87d51 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -833,6 +833,17 @@ class StaticMount(click.ParamType): self.fail(f"{value} is not a valid directory path", param, ctx) return path, dirpath +# The --load-extension parameter can optionally include a specific entrypoint. +# This is done by appending ":entrypoint_name" after supplying the path to the extension +class LoadExtension(click.ParamType): + name = "path:entrypoint?" + + def convert(self, value, param, ctx): + if ":" not in value: + return value + path, entrypoint = value.split(":", 1) + return path, entrypoint + def format_bytes(bytes): current = float(bytes) diff --git a/tests/ext.c b/tests/ext.c new file mode 100644 index 00000000..5fe970d9 --- /dev/null +++ b/tests/ext.c @@ -0,0 +1,48 @@ +/* +** This file implements a SQLite extension with multiple entrypoints. +** +** The default entrypoint, sqlite3_ext_init, has a single function "a". +** The 1st alternate entrypoint, sqlite3_ext_b_init, has a single function "b". +** The 2nd alternate entrypoint, sqlite3_ext_c_init, has a single function "c". +** +** Compiling instructions: +** https://www.sqlite.org/loadext.html#compiling_a_loadable_extension +** +*/ + +#include "sqlite3ext.h" + +SQLITE_EXTENSION_INIT1 + +// SQL function that returns back the value supplied during sqlite3_create_function() +static void func(sqlite3_context *context, int argc, sqlite3_value **argv) { + sqlite3_result_text(context, (char *) sqlite3_user_data(context), -1, SQLITE_STATIC); +} + + +// The default entrypoint, since it matches the "ext.dylib"/"ext.so" name +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "a", 0, 0, "a", func, 0, 0); +} + +// Alternate entrypoint #1 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_b_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "b", 0, 0, "b", func, 0, 0); +} + +// Alternate entrypoint #2 +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ext_c_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3_create_function(db, "c", 0, 0, "c", func, 0, 0); +} diff --git a/tests/test_load_extensions.py b/tests/test_load_extensions.py new file mode 100644 index 00000000..360bc8f3 --- /dev/null +++ b/tests/test_load_extensions.py @@ -0,0 +1,65 @@ +from datasette.app import Datasette +import pytest +from pathlib import Path + +# not necessarily a full path - the full compiled path looks like "ext.dylib" +# or another suffix, but sqlite will, under the hood, decide which file +# extension to use based on the operating system (apple=dylib, windows=dll etc) +# this resolves to "./ext", which is enough for SQLite to calculate the rest +COMPILED_EXTENSION_PATH = str(Path(__file__).parent / "ext") + +# See if ext.c has been compiled, based off the different possible suffixes. +def has_compiled_ext(): + for ext in ["dylib", "so", "dll"]: + path = Path(__file__).parent / f"ext.{ext}" + if path.is_file(): + return True + return False + + +@pytest.mark.asyncio +@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c") +async def test_load_extension_default_entrypoint(): + + # The default entrypoint only loads a() and NOT b() or c(), so those + # should fail. + ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH]) + + response = await ds.client.get("/_memory.json?sql=select+a()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "a" + + response = await ds.client.get("/_memory.json?sql=select+b()") + assert response.status_code == 400 + assert response.json()["error"] == "no such function: b" + + response = await ds.client.get("/_memory.json?sql=select+c()") + assert response.status_code == 400 + assert response.json()["error"] == "no such function: c" + + +@pytest.mark.asyncio +@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c") +async def test_load_extension_multiple_entrypoints(): + + # Load in the default entrypoint and the other 2 custom entrypoints, now + # all a(), b(), and c() should run successfully. + ds = Datasette( + sqlite_extensions=[ + COMPILED_EXTENSION_PATH, + (COMPILED_EXTENSION_PATH, "sqlite3_ext_b_init"), + (COMPILED_EXTENSION_PATH, "sqlite3_ext_c_init"), + ] + ) + + response = await ds.client.get("/_memory.json?sql=select+a()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "a" + + response = await ds.client.get("/_memory.json?sql=select+b()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "b" + + response = await ds.client.get("/_memory.json?sql=select+c()") + assert response.status_code == 200 + assert response.json()["rows"][0][0] == "c" From fd1086c6867f3e3582b1eca456e4ea95f6cecf8b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Aug 2022 11:35:41 -0700 Subject: [PATCH 0010/1116] Applied Black, refs #1789 --- datasette/app.py | 4 ++-- datasette/utils/__init__.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index bb9232c9..f2a6763a 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -559,8 +559,8 @@ class Datasette: if self.sqlite_extensions: conn.enable_load_extension(True) for extension in self.sqlite_extensions: - # "extension" is either a string path to the extension - # or a 2-item tuple that specifies which entrypoint to load. + # "extension" is either a string path to the extension + # or a 2-item tuple that specifies which entrypoint to load. if isinstance(extension, tuple): path, entrypoint = extension conn.execute("SELECT load_extension(?, ?)", [path, entrypoint]) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 0fc87d51..bbaa0510 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -833,6 +833,7 @@ class StaticMount(click.ParamType): self.fail(f"{value} is not a valid directory path", param, ctx) return path, dirpath + # The --load-extension parameter can optionally include a specific entrypoint. # This is done by appending ":entrypoint_name" after supplying the path to the extension class LoadExtension(click.ParamType): From 456dc155d491a009942ace71a4e1827cddc6b93d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Aug 2022 11:40:36 -0700 Subject: [PATCH 0011/1116] Ran cog, refs #1789 --- docs/cli-reference.rst | 95 +++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index a1e56774..f8419d58 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -84,48 +84,53 @@ Once started you can access it at ``http://localhost:8001`` Serve up specified SQLite database files with a web UI Options: - -i, --immutable PATH Database files to open in immutable mode - -h, --host TEXT Host for server. Defaults to 127.0.0.1 which means - only connections from the local machine will be - allowed. Use 0.0.0.0 to listen to all IPs and allow - access from other machines. - -p, --port INTEGER RANGE Port for server, defaults to 8001. Use -p 0 to - automatically assign an available port. - [0<=x<=65535] - --uds TEXT Bind to a Unix domain socket - --reload Automatically reload if code or metadata change - detected - useful for development - --cors Enable CORS by serving Access-Control-Allow-Origin: - * - --load-extension TEXT Path to a SQLite extension to load - --inspect-file TEXT Path to JSON file created using "datasette inspect" - -m, --metadata FILENAME Path to JSON/YAML file containing license/source - metadata - --template-dir DIRECTORY Path to directory containing custom templates - --plugins-dir DIRECTORY Path to directory containing custom plugins - --static MOUNT:DIRECTORY Serve static files from this directory at /MOUNT/... - --memory Make /_memory database available - --config CONFIG Deprecated: set config option using - configname:value. Use --setting instead. - --setting SETTING... Setting, see - docs.datasette.io/en/stable/settings.html - --secret TEXT Secret used for signing secure values, such as - signed cookies - --root Output URL that sets a cookie authenticating the - root user - --get TEXT Run an HTTP GET request against this path, print - results and exit - --version-note TEXT Additional note to show on /-/versions - --help-settings Show available settings - --pdb Launch debugger on any errors - -o, --open Open Datasette in your web browser - --create Create database files if they do not exist - --crossdb Enable cross-database joins using the /_memory - database - --nolock Ignore locking, open locked files in read-only mode - --ssl-keyfile TEXT SSL key file - --ssl-certfile TEXT SSL certificate file - --help Show this message and exit. + -i, --immutable PATH Database files to open in immutable mode + -h, --host TEXT Host for server. Defaults to 127.0.0.1 which + means only connections from the local machine + will be allowed. Use 0.0.0.0 to listen to all + IPs and allow access from other machines. + -p, --port INTEGER RANGE Port for server, defaults to 8001. Use -p 0 to + automatically assign an available port. + [0<=x<=65535] + --uds TEXT Bind to a Unix domain socket + --reload Automatically reload if code or metadata + change detected - useful for development + --cors Enable CORS by serving Access-Control-Allow- + Origin: * + --load-extension PATH:ENTRYPOINT? + Path to a SQLite extension to load, and + optional entrypoint + --inspect-file TEXT Path to JSON file created using "datasette + inspect" + -m, --metadata FILENAME Path to JSON/YAML file containing + license/source metadata + --template-dir DIRECTORY Path to directory containing custom templates + --plugins-dir DIRECTORY Path to directory containing custom plugins + --static MOUNT:DIRECTORY Serve static files from this directory at + /MOUNT/... + --memory Make /_memory database available + --config CONFIG Deprecated: set config option using + configname:value. Use --setting instead. + --setting SETTING... Setting, see + docs.datasette.io/en/stable/settings.html + --secret TEXT Secret used for signing secure values, such as + signed cookies + --root Output URL that sets a cookie authenticating + the root user + --get TEXT Run an HTTP GET request against this path, + print results and exit + --version-note TEXT Additional note to show on /-/versions + --help-settings Show available settings + --pdb Launch debugger on any errors + -o, --open Open Datasette in your web browser + --create Create database files if they do not exist + --crossdb Enable cross-database joins using the /_memory + database + --nolock Ignore locking, open locked files in read-only + mode + --ssl-keyfile TEXT SSL key file + --ssl-certfile TEXT SSL certificate file + --help Show this message and exit. .. [[[end]]] @@ -566,8 +571,10 @@ This performance optimization is used automatically by some of the ``datasette p Options: --inspect-file TEXT - --load-extension TEXT Path to a SQLite extension to load - --help Show this message and exit. + --load-extension PATH:ENTRYPOINT? + Path to a SQLite extension to load, and + optional entrypoint + --help Show this message and exit. .. [[[end]]] From ba35105eee2d3ba620e4f230028a02b2e2571df2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 23 Aug 2022 17:11:45 -0700 Subject: [PATCH 0012/1116] Test `--load-extension` in GitHub Actions (#1792) * Run the --load-extension test, refs #1789 * Ran cog, refs #1789 --- .github/workflows/test.yml | 3 +++ tests/test_api.py | 2 +- tests/test_html.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90b6555e..e38d5ee9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,9 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- + - name: Build extension for --load-extension test + run: |- + (cd tests && gcc ext.c -fPIC -shared -o ext.so) - name: Install dependencies run: | pip install -e '.[test]' diff --git a/tests/test_api.py b/tests/test_api.py index 253c1718..f6db2f9d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -36,7 +36,7 @@ def test_homepage(app_client): # 4 hidden FTS tables + no_primary_key (hidden in metadata) assert d["hidden_tables_count"] == 6 # 201 in no_primary_key, plus 6 in other hidden tables: - assert d["hidden_table_rows_sum"] == 207 + assert d["hidden_table_rows_sum"] == 207, response.json assert d["views_count"] == 4 diff --git a/tests/test_html.py b/tests/test_html.py index be21bd84..d6e969ad 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -115,7 +115,7 @@ def test_database_page(app_client): assert fragment in response.text # And views - views_ul = soup.find("h2", text="Views").find_next_sibling("ul") + views_ul = soup.find("h2", string="Views").find_next_sibling("ul") assert views_ul is not None assert [ ("/fixtures/paginated_view", "paginated_view"), @@ -128,7 +128,7 @@ def test_database_page(app_client): ] == sorted([(a["href"], a.text) for a in views_ul.find_all("a")]) # And a list of canned queries - queries_ul = soup.find("h2", text="Queries").find_next_sibling("ul") + queries_ul = soup.find("h2", string="Queries").find_next_sibling("ul") assert queries_ul is not None assert [ ("/fixtures/from_async_hook", "from_async_hook"), From 51030df1869b3b574dd3584d1563415776b9cd4e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 11:35:40 -0700 Subject: [PATCH 0013/1116] Don't use upper bound dependencies any more See https://iscinumpy.dev/post/bound-version-constraints/ for the rationale behind this change. Closes #1800 --- setup.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/setup.py b/setup.py index a1c51d0b..b2e50b38 100644 --- a/setup.py +++ b/setup.py @@ -42,21 +42,21 @@ setup( include_package_data=True, python_requires=">=3.7", install_requires=[ - "asgiref>=3.2.10,<3.6.0", - "click>=7.1.1,<8.2.0", + "asgiref>=3.2.10", + "click>=7.1.1", "click-default-group-wheel>=1.2.2", - "Jinja2>=2.10.3,<3.1.0", - "hupper~=1.9", + "Jinja2>=2.10.3", + "hupper>=1.9", "httpx>=0.20", - "pint~=0.9", - "pluggy>=1.0,<1.1", - "uvicorn~=0.11", - "aiofiles>=0.4,<0.9", - "janus>=0.6.2,<1.1", + "pint>=0.9", + "pluggy>=1.0", + "uvicorn>=0.11", + "aiofiles>=0.4", + "janus>=0.6.2", "asgi-csrf>=0.9", - "PyYAML>=5.3,<7.0", - "mergedeep>=1.1.1,<1.4.0", - "itsdangerous>=1.1,<3.0", + "PyYAML>=5.3", + "mergedeep>=1.1.1", + "itsdangerous>=1.1", ], entry_points=""" [console_scripts] @@ -72,14 +72,14 @@ setup( "sphinx-copybutton", ], "test": [ - "pytest>=5.2.2,<7.2.0", - "pytest-xdist>=2.2.1,<2.6", - "pytest-asyncio>=0.17,<0.20", - "beautifulsoup4>=4.8.1,<4.12.0", + "pytest>=5.2.2", + "pytest-xdist>=2.2.1", + "pytest-asyncio>=0.17", + "beautifulsoup4>=4.8.1", "black==22.6.0", "blacken-docs==1.12.1", - "pytest-timeout>=1.4.2,<2.2", - "trustme>=0.7,<0.10", + "pytest-timeout>=1.4.2", + "trustme>=0.7", "cogapp>=3.3.0", ], "rich": ["rich"], From 294ecd45f7801971dbeef383d0c5456ee95ab839 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Sep 2022 11:51:51 -0700 Subject: [PATCH 0014/1116] Bump black from 22.6.0 to 22.8.0 (#1797) Bumps [black](https://github.com/psf/black) from 22.6.0 to 22.8.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/22.6.0...22.8.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b2e50b38..92fa60d0 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( "pytest-xdist>=2.2.1", "pytest-asyncio>=0.17", "beautifulsoup4>=4.8.1", - "black==22.6.0", + "black==22.8.0", "blacken-docs==1.12.1", "pytest-timeout>=1.4.2", "trustme>=0.7", From b91e17280c05bbb9cf97432081bdcea8665879f9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 16:50:53 -0700 Subject: [PATCH 0015/1116] Run tests in serial, refs #1802 --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e38d5ee9..9c8c48ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,8 +33,7 @@ jobs: pip freeze - name: Run tests run: | - pytest -n auto -m "not serial" - pytest -m "serial" + pytest - name: Check if cog needs to be run run: | cog --check docs/*.rst From b2b901e8c4b939e50ee1117ffcd2881ed8a8e3bf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 17:05:23 -0700 Subject: [PATCH 0016/1116] Skip SpatiaLite test if no conn.enable_load_extension() Ran into this problem while working on #1802 --- tests/test_spatialite.py | 2 ++ tests/utils.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/tests/test_spatialite.py b/tests/test_spatialite.py index 8b98c5d6..c07a30e8 100644 --- a/tests/test_spatialite.py +++ b/tests/test_spatialite.py @@ -1,5 +1,6 @@ from datasette.app import Datasette from datasette.utils import find_spatialite, SpatialiteNotFound, SPATIALITE_FUNCTIONS +from .utils import has_load_extension import pytest @@ -13,6 +14,7 @@ def has_spatialite(): @pytest.mark.asyncio @pytest.mark.skipif(not has_spatialite(), reason="Requires SpatiaLite") +@pytest.mark.skipif(not has_load_extension(), reason="Requires enable_load_extension") async def test_spatialite_version_info(): ds = Datasette(sqlite_extensions=["spatialite"]) response = await ds.client.get("/-/versions.json") diff --git a/tests/utils.py b/tests/utils.py index 972300db..191ead9b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,3 +1,6 @@ +from datasette.utils.sqlite import sqlite3 + + def assert_footer_links(soup): footer_links = soup.find("footer").findAll("a") assert 4 == len(footer_links) @@ -22,3 +25,8 @@ def inner_html(soup): # This includes the parent tag - so remove that inner_html = html.split(">", 1)[1].rsplit("<", 1)[0] return inner_html.strip() + + +def has_load_extension(): + conn = sqlite3.connect(":memory:") + return hasattr(conn, "enable_load_extension") From 1c29b925d300d1ee17047504473f2517767aa05b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 17:10:52 -0700 Subject: [PATCH 0017/1116] Run tests in serial again Because this didn't fix the issue I'm seeing in #1802 Revert "Run tests in serial, refs #1802" This reverts commit b91e17280c05bbb9cf97432081bdcea8665879f9. --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c8c48ef..e38d5ee9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,8 @@ jobs: pip freeze - name: Run tests run: | - pytest + pytest -n auto -m "not serial" + pytest -m "serial" - name: Check if cog needs to be run run: | cog --check docs/*.rst From 64288d827f7ff97f825e10f714da3f781ecf9345 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 17:40:19 -0700 Subject: [PATCH 0018/1116] Workaround for test failure: RuntimeError: There is no current event loop (#1803) * Remove ensure_eventloop hack * Hack to recover from intermittent RuntimeError calling asyncio.Lock() --- datasette/app.py | 10 +++++++++- tests/test_cli.py | 27 ++++++++++----------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index f2a6763a..c6bbdaf0 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -231,7 +231,15 @@ class Datasette: self.inspect_data = inspect_data self.immutables = set(immutables or []) self.databases = collections.OrderedDict() - self._refresh_schemas_lock = asyncio.Lock() + try: + self._refresh_schemas_lock = asyncio.Lock() + except RuntimeError as rex: + # Workaround for intermittent test failure, see: + # https://github.com/simonw/datasette/issues/1802 + if "There is no current event loop in thread" in str(rex): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + self._refresh_schemas_lock = asyncio.Lock() self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: diff --git a/tests/test_cli.py b/tests/test_cli.py index d0f6e26c..f0d28037 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -22,13 +22,6 @@ from unittest import mock import urllib -@pytest.fixture -def ensure_eventloop(): - # Workaround for "Event loop is closed" error - if asyncio.get_event_loop().is_closed(): - asyncio.set_event_loop(asyncio.new_event_loop()) - - def test_inspect_cli(app_client): runner = CliRunner() result = runner.invoke(cli, ["inspect", "fixtures.db"]) @@ -72,7 +65,7 @@ def test_serve_with_inspect_file_prepopulates_table_counts_cache(): ), ) def test_spatialite_error_if_attempt_to_open_spatialite( - ensure_eventloop, spatialite_paths, should_suggest_load_extension + spatialite_paths, should_suggest_load_extension ): with mock.patch("datasette.utils.SPATIALITE_PATHS", spatialite_paths): runner = CliRunner() @@ -199,14 +192,14 @@ def test_version(): @pytest.mark.parametrize("invalid_port", ["-1", "0.5", "dog", "65536"]) -def test_serve_invalid_ports(ensure_eventloop, invalid_port): +def test_serve_invalid_ports(invalid_port): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--port", invalid_port]) assert result.exit_code == 2 assert "Invalid value for '-p'" in result.stderr -def test_setting(ensure_eventloop): +def test_setting(): runner = CliRunner() result = runner.invoke( cli, ["--setting", "default_page_size", "5", "--get", "/-/settings.json"] @@ -215,14 +208,14 @@ def test_setting(ensure_eventloop): assert json.loads(result.output)["default_page_size"] == 5 -def test_setting_type_validation(ensure_eventloop): +def test_setting_type_validation(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--setting", "default_page_size", "dog"]) assert result.exit_code == 2 assert '"default_page_size" should be an integer' in result.stderr -def test_config_deprecated(ensure_eventloop): +def test_config_deprecated(): # The --config option should show a deprecation message runner = CliRunner(mix_stderr=False) result = runner.invoke( @@ -233,14 +226,14 @@ def test_config_deprecated(ensure_eventloop): assert "will be deprecated in" in result.stderr -def test_sql_errors_logged_to_stderr(ensure_eventloop): +def test_sql_errors_logged_to_stderr(): runner = CliRunner(mix_stderr=False) result = runner.invoke(cli, ["--get", "/_memory.json?sql=select+blah"]) assert result.exit_code == 1 assert "sql = 'select blah', params = {}: no such column: blah\n" in result.stderr -def test_serve_create(ensure_eventloop, tmpdir): +def test_serve_create(tmpdir): runner = CliRunner() db_path = tmpdir / "does_not_exist_yet.db" assert not db_path.exists() @@ -258,7 +251,7 @@ def test_serve_create(ensure_eventloop, tmpdir): assert db_path.exists() -def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): +def test_serve_duplicate_database_names(tmpdir): "'datasette db.db nested/db.db' should attach two databases, /db and /db_2" runner = CliRunner() db_1_path = str(tmpdir / "db.db") @@ -273,7 +266,7 @@ def test_serve_duplicate_database_names(ensure_eventloop, tmpdir): assert {db["name"] for db in databases} == {"db", "db_2"} -def test_serve_deduplicate_same_database_path(ensure_eventloop, tmpdir): +def test_serve_deduplicate_same_database_path(tmpdir): "'datasette db.db db.db' should only attach one database, /db" runner = CliRunner() db_path = str(tmpdir / "db.db") @@ -287,7 +280,7 @@ def test_serve_deduplicate_same_database_path(ensure_eventloop, tmpdir): @pytest.mark.parametrize( "filename", ["test-database (1).sqlite", "database (1).sqlite"] ) -def test_weird_database_names(ensure_eventloop, tmpdir, filename): +def test_weird_database_names(tmpdir, filename): # https://github.com/simonw/datasette/issues/1181 runner = CliRunner() db_path = str(tmpdir / filename) From c9d1943aede436fa3413fd49bc56335cbda4ad07 Mon Sep 17 00:00:00 2001 From: Daniel Rech Date: Tue, 6 Sep 2022 02:45:41 +0200 Subject: [PATCH 0019/1116] Fix word break in facets by adding ul.tight-bullets li word-break: break-all (#1794) Thanks, @dmr --- datasette/static/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index af3e14d5..712b9925 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -260,6 +260,7 @@ ul.bullets li { ul.tight-bullets li { list-style-type: disc; margin-bottom: 0; + word-break: break-all; } a.not-underlined { text-decoration: none; From d80775a48d20917633792fdc9525f075d3bc2c7a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 5 Sep 2022 17:44:44 -0700 Subject: [PATCH 0020/1116] Raise error if it's not about loops, refs #1802 --- datasette/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index c6bbdaf0..aeb81687 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -240,6 +240,8 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() + else: + raise self.crossdb = crossdb self.nolock = nolock if memory or crossdb or not self.files: From 8430c3bc7dd22b173c1a8c6cd7180e3b31240cd1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 08:59:19 -0700 Subject: [PATCH 0021/1116] table facet_size in metadata, refs #1804 --- datasette/facets.py | 14 +++++++++++--- tests/test_facets.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index b15a758c..e70d42df 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -102,11 +102,19 @@ class Facet: def get_facet_size(self): facet_size = self.ds.setting("default_facet_size") max_returned_rows = self.ds.setting("max_returned_rows") + table_facet_size = None + if self.table: + tables_metadata = self.ds.metadata("tables", database=self.database) or {} + table_metadata = tables_metadata.get(self.table) or {} + if table_metadata: + table_facet_size = table_metadata.get("facet_size") custom_facet_size = self.request.args.get("_facet_size") - if custom_facet_size == "max": - facet_size = max_returned_rows - elif custom_facet_size and custom_facet_size.isdigit(): + if custom_facet_size and custom_facet_size.isdigit(): facet_size = int(custom_facet_size) + elif table_facet_size: + facet_size = table_facet_size + if facet_size == "max": + facet_size = max_returned_rows return min(facet_size, max_returned_rows) async def suggest(self): diff --git a/tests/test_facets.py b/tests/test_facets.py index c28dc43c..cbee23b0 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -581,6 +581,23 @@ async def test_facet_size(): ) data5 = response5.json() assert len(data5["facet_results"]["city"]["results"]) == 20 + # Now try messing with facet_size in the table metadata + ds._metadata_local = { + "databases": { + "test_facet_size": {"tables": {"neighbourhoods": {"facet_size": 6}}} + } + } + response6 = await ds.client.get("/test_facet_size/neighbourhoods.json?_facet=city") + data6 = response6.json() + assert len(data6["facet_results"]["city"]["results"]) == 6 + # Setting it to max bumps it up to 50 again + ds._metadata_local["databases"]["test_facet_size"]["tables"]["neighbourhoods"][ + "facet_size" + ] = "max" + data7 = ( + await ds.client.get("/test_facet_size/neighbourhoods.json?_facet=city") + ).json() + assert len(data7["facet_results"]["city"]["results"]) == 20 def test_other_types_of_facet_in_metadata(): From 303c6c733d95a6133558ec1b468f5bea5827d0d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 11:05:00 -0700 Subject: [PATCH 0022/1116] Fix for incorrectly handled _facet_size=max, refs #1804 --- datasette/facets.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/datasette/facets.py b/datasette/facets.py index e70d42df..7fb0c68b 100644 --- a/datasette/facets.py +++ b/datasette/facets.py @@ -109,12 +109,19 @@ class Facet: if table_metadata: table_facet_size = table_metadata.get("facet_size") custom_facet_size = self.request.args.get("_facet_size") - if custom_facet_size and custom_facet_size.isdigit(): - facet_size = int(custom_facet_size) - elif table_facet_size: - facet_size = table_facet_size - if facet_size == "max": - facet_size = max_returned_rows + if custom_facet_size: + if custom_facet_size == "max": + facet_size = max_returned_rows + elif custom_facet_size.isdigit(): + facet_size = int(custom_facet_size) + else: + # Invalid value, ignore it + custom_facet_size = None + if table_facet_size and not custom_facet_size: + if table_facet_size == "max": + facet_size = max_returned_rows + else: + facet_size = table_facet_size return min(facet_size, max_returned_rows) async def suggest(self): From 0a7815d2038255a0834c955066a2a16c01f707b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 11:06:49 -0700 Subject: [PATCH 0023/1116] Documentation for facet_size in metadata, closes #1804 --- docs/facets.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/facets.rst b/docs/facets.rst index 2a2eb039..6c9d99bd 100644 --- a/docs/facets.rst +++ b/docs/facets.rst @@ -129,6 +129,22 @@ You can specify :ref:`array ` or :ref:`date ] } +You can change the default facet size (the number of results shown for each facet) for a table using ``facet_size``: + +.. code-block:: json + + { + "databases": { + "sf-trees": { + "tables": { + "Street_Tree_List": { + "facets": ["qLegalStatus"], + "facet_size": 10 + } + } + } + } + } Suggested facets ---------------- From d0476897e10249bb4867473722270d02491c2c1f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 11:24:30 -0700 Subject: [PATCH 0024/1116] Fixed Sphinx warning about language = None --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 4ef6b768..8965974a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -71,7 +71,7 @@ release = "" # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From ff9c87197dde8b09f9787ee878804cb6842ea5dc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 11:26:21 -0700 Subject: [PATCH 0025/1116] Fixed Sphinx warnings on cli-reference page --- docs/cli-reference.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index f8419d58..4a8465cb 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -14,7 +14,7 @@ Running ``datasette`` without specifying a command runs the default command, ``d import textwrap def help(args): title = "datasette " + " ".join(args) - cog.out("::\n\n") + cog.out("\n::\n\n") result = CliRunner().invoke(cli.cli, args) output = result.output.replace("Usage: cli ", "Usage: datasette ") cog.out(textwrap.indent(output, ' ')) @@ -32,6 +32,7 @@ Running ``datasette --help`` shows a list of all of the available commands. .. [[[cog help(["--help"]) .. ]]] + :: Usage: datasette [OPTIONS] COMMAND [ARGS]... @@ -77,6 +78,7 @@ Once started you can access it at ``http://localhost:8001`` .. [[[cog help(["serve", "--help"]) .. ]]] + :: Usage: datasette serve [OPTIONS] [FILES]... @@ -202,6 +204,7 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam .. [[[cog help(["--help-settings"]) .. ]]] + :: Settings: @@ -258,6 +261,7 @@ Output JSON showing all currently installed plugins, their versions, whether the .. [[[cog help(["plugins", "--help"]) .. ]]] + :: Usage: datasette plugins [OPTIONS] @@ -326,6 +330,7 @@ Would install the `datasette-cluster-map Date: Tue, 6 Sep 2022 16:50:43 -0700 Subject: [PATCH 0026/1116] truncate_cells_html now affects URLs too, refs #1805 --- datasette/utils/__init__.py | 10 ++++++++++ datasette/views/database.py | 11 ++++++++--- datasette/views/table.py | 8 ++++++-- tests/fixtures.py | 9 +++++---- tests/test_api.py | 2 +- tests/test_table_api.py | 11 +++++++---- tests/test_table_html.py | 11 +++++++++++ tests/test_utils.py | 20 ++++++++++++++++++++ 8 files changed, 68 insertions(+), 14 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index bbaa0510..2bdea673 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1167,3 +1167,13 @@ def resolve_routes(routes, path): if match is not None: return match, view return None, None + + +def truncate_url(url, length): + if (not length) or (len(url) <= length): + return url + bits = url.rsplit(".", 1) + if len(bits) == 2 and 1 <= len(bits[1]) <= 4 and "/" not in bits[1]: + rest, ext = bits + return rest[: length - 1 - len(ext)] + "…." + ext + return url[: length - 1] + "…" diff --git a/datasette/views/database.py b/datasette/views/database.py index 77632b9d..fc344245 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -20,6 +20,7 @@ from datasette.utils import ( path_with_format, path_with_removed_args, sqlite3, + truncate_url, InvalidSql, ) from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden @@ -371,6 +372,7 @@ class QueryView(DataView): async def extra_template(): display_rows = [] + truncate_cells = self.ds.setting("truncate_cells_html") for row in results.rows if results else []: display_row = [] for column, value in zip(results.columns, row): @@ -396,9 +398,12 @@ class QueryView(DataView): if value in ("", None): display_value = Markup(" ") elif is_url(str(display_value).strip()): - display_value = Markup( - '{url}'.format( - url=escape(value.strip()) + display_value = markupsafe.Markup( + '{truncated_url}'.format( + url=markupsafe.escape(value.strip()), + truncated_url=markupsafe.escape( + truncate_url(value.strip(), truncate_cells) + ), ) ) elif isinstance(display_value, bytes): diff --git a/datasette/views/table.py b/datasette/views/table.py index 49c30c9c..60c092f9 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -24,6 +24,7 @@ from datasette.utils import ( path_with_removed_args, path_with_replaced_args, to_css_class, + truncate_url, urlsafe_components, value_as_boolean, ) @@ -966,8 +967,11 @@ async def display_columns_and_rows( display_value = markupsafe.Markup(" ") elif is_url(str(value).strip()): display_value = markupsafe.Markup( - '{url}'.format( - url=markupsafe.escape(value.strip()) + '{truncated_url}'.format( + url=markupsafe.escape(value.strip()), + truncated_url=markupsafe.escape( + truncate_url(value.strip(), truncate_cells) + ), ) ) elif column in table_metadata.get("units", {}) and value != "": diff --git a/tests/fixtures.py b/tests/fixtures.py index c145ac78..82d8452e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -598,23 +598,24 @@ CREATE TABLE roadside_attractions ( pk integer primary key, name text, address text, + url text, latitude real, longitude real ); INSERT INTO roadside_attractions VALUES ( - 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", + 1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/", 37.0167, -122.0024 ); INSERT INTO roadside_attractions VALUES ( - 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", + 2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/", 37.3184, -121.9511 ); INSERT INTO roadside_attractions VALUES ( - 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", + 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null, 37.5793, -122.3442 ); INSERT INTO roadside_attractions VALUES ( - 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", + 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/", 37.0414, -122.0725 ); diff --git a/tests/test_api.py b/tests/test_api.py index f6db2f9d..7a2bf91f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -339,7 +339,7 @@ def test_database_page(app_client): }, { "name": "roadside_attractions", - "columns": ["pk", "name", "address", "latitude", "longitude"], + "columns": ["pk", "name", "address", "url", "latitude", "longitude"], "primary_keys": ["pk"], "count": 4, "hidden": False, diff --git a/tests/test_table_api.py b/tests/test_table_api.py index e56a72b5..0db04434 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -615,11 +615,12 @@ def test_table_through(app_client): response = app_client.get( '/fixtures/roadside_attractions.json?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}' ) - assert [ + assert response.json["rows"] == [ [ 3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", + None, 37.5793, -122.3442, ], @@ -627,13 +628,15 @@ def test_table_through(app_client): 4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", + "https://www.bigfootdiscoveryproject.com/", 37.0414, -122.0725, ], - ] == response.json["rows"] + ] + assert ( - 'where roadside_attraction_characteristics.characteristic_id = "1"' - == response.json["human_description_en"] + response.json["human_description_en"] + == 'where roadside_attraction_characteristics.characteristic_id = "1"' ) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index f3808ea3..8e37468f 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -69,6 +69,17 @@ def test_table_cell_truncation(): td.string for td in table.findAll("td", {"class": "col-neighborhood-b352a7"}) ] + # URLs should be truncated too + response2 = client.get("/fixtures/roadside_attractions") + assert response2.status == 200 + table = Soup(response2.body, "html.parser").find("table") + tds = table.findAll("td", {"class": "col-url"}) + assert [str(td) for td in tds] == [ + 'http…', + 'http…', + '\xa0', + 'http…', + ] def test_add_filter_redirects(app_client): diff --git a/tests/test_utils.py b/tests/test_utils.py index df788767..d71a612d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -626,3 +626,23 @@ def test_tilde_encoding(original, expected): assert actual == expected # And test round-trip assert original == utils.tilde_decode(actual) + + +@pytest.mark.parametrize( + "url,length,expected", + ( + ("https://example.com/", 5, "http…"), + ("https://example.com/foo/bar", 15, "https://exampl…"), + ("https://example.com/foo/bar/baz.jpg", 30, "https://example.com/foo/ba….jpg"), + # Extensions longer than 4 characters are not treated specially: + ("https://example.com/foo/bar/baz.jpeg2", 30, "https://example.com/foo/bar/b…"), + ( + "https://example.com/foo/bar/baz.jpeg2", + None, + "https://example.com/foo/bar/baz.jpeg2", + ), + ), +) +def test_truncate_url(url, length, expected): + actual = utils.truncate_url(url, length) + assert actual == expected From 5aa359b86907d11b3ee601510775a85a90224da8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 16:58:30 -0700 Subject: [PATCH 0027/1116] Apply cell truncation on query page too, refs #1805 --- datasette/views/database.py | 7 ++++++- tests/test_html.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index fc344245..affbc540 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -428,7 +428,12 @@ class QueryView(DataView): "" if len(value) == 1 else "s", ) ) - + else: + display_value = str(value) + if truncate_cells and len(display_value) > truncate_cells: + display_value = ( + display_value[:truncate_cells] + "\u2026" + ) display_row.append(display_value) display_rows.append(display_row) diff --git a/tests/test_html.py b/tests/test_html.py index d6e969ad..bf915247 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -186,6 +186,25 @@ def test_row_page_does_not_truncate(): ] +def test_query_page_truncates(): + with make_app_client(settings={"truncate_cells_html": 5}) as client: + response = client.get( + "/fixtures?" + + urllib.parse.urlencode( + { + "sql": "select 'this is longer than 5' as a, 'https://example.com/' as b" + } + ) + ) + assert response.status == 200 + table = Soup(response.body, "html.parser").find("table") + tds = table.findAll("td") + assert [str(td) for td in tds] == [ + 'this …', + 'http…', + ] + + @pytest.mark.parametrize( "path,expected_classes", [ From bf8d84af5422606597be893cedd375020cb2b369 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 6 Sep 2022 20:34:59 -0700 Subject: [PATCH 0028/1116] word-wrap: anywhere on links in cells, refs #1805 --- datasette/static/app.css | 1 + 1 file changed, 1 insertion(+) diff --git a/datasette/static/app.css b/datasette/static/app.css index 712b9925..08b724f6 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -446,6 +446,7 @@ th { } table a:link { text-decoration: none; + word-wrap: anywhere; } .rows-and-columns td:before { display: block; From fb7e70d5e72a951efe4b29ad999d8915c032d021 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 9 Sep 2022 09:19:20 -0700 Subject: [PATCH 0029/1116] Database(is_mutable=) now defaults to True, closes #1808 Refs https://github.com/simonw/datasette-upload-dbs/issues/6 --- datasette/database.py | 3 +-- docs/internals.rst | 9 +++++---- tests/test_internals_database.py | 1 + tests/test_internals_datasette.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index fa558045..44467370 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -28,7 +28,7 @@ AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) class Database: def __init__( - self, ds, path=None, is_mutable=False, is_memory=False, memory_name=None + self, ds, path=None, is_mutable=True, is_memory=False, memory_name=None ): self.name = None self.route = None @@ -39,7 +39,6 @@ class Database: self.memory_name = memory_name if memory_name is not None: self.is_memory = True - self.is_mutable = True self.hash = None self.cached_size = None self._cached_table_counts = None diff --git a/docs/internals.rst b/docs/internals.rst index 20797e98..adeec1d8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -426,12 +426,13 @@ The ``db`` parameter should be an instance of the ``datasette.database.Database` Database( datasette, path="path/to/my-new-database.db", - is_mutable=True, ) ) This will add a mutable database and serve it at ``/my-new-database``. +Use ``is_mutable=False`` to add an immutable database. + ``.add_database()`` returns the Database instance, with its name set as the ``database.name`` attribute. Any time you are working with a newly added database you should use the return value of ``.add_database()``, for example: .. code-block:: python @@ -671,8 +672,8 @@ Instances of the ``Database`` class can be used to execute queries against attac .. _database_constructor: -Database(ds, path=None, is_mutable=False, is_memory=False, memory_name=None) ----------------------------------------------------------------------------- +Database(ds, path=None, is_mutable=True, is_memory=False, memory_name=None) +--------------------------------------------------------------------------- The ``Database()`` constructor can be used by plugins, in conjunction with :ref:`datasette_add_database`, to create and register new databases. @@ -685,7 +686,7 @@ The arguments are as follows: Path to a SQLite database file on disk. ``is_mutable`` - boolean - Set this to ``True`` if it is possible that updates will be made to that database - otherwise Datasette will open it in immutable mode and any changes could cause undesired behavior. + Set this to ``False`` to cause Datasette to open the file in immutable mode. ``is_memory`` - boolean Use this to create non-shared memory connections. diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 551f67e1..9e81c1d6 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -499,6 +499,7 @@ def test_mtime_ns_is_none_for_memory(app_client): def test_is_mutable(app_client): + assert Database(app_client.ds, is_memory=True).is_mutable is True assert Database(app_client.ds, is_memory=True, is_mutable=True).is_mutable is True assert Database(app_client.ds, is_memory=True, is_mutable=False).is_mutable is False diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 1dc14cab..249920fe 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -58,7 +58,7 @@ async def test_datasette_constructor(): "route": "_memory", "path": None, "size": 0, - "is_mutable": False, + "is_mutable": True, "is_memory": True, "hash": None, } From 610425460b519e9c16d386cb81aa081c9d730ef0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 10 Sep 2022 14:24:26 -0700 Subject: [PATCH 0030/1116] Add --nolock to the README Chrome demo Refs #1744 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1af20129..af95b85e 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ This will start a web server on port 8001 - visit http://localhost:8001/ to acce Use Chrome on OS X? You can run datasette against your browser history like so: - datasette ~/Library/Application\ Support/Google/Chrome/Default/History + datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data: From b40872f5e5ae5dad331c58f75451e2d206565196 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 14 Sep 2022 14:31:54 -0700 Subject: [PATCH 0031/1116] prepare_jinja2_environment(datasette) argument, refs #1809 --- datasette/app.py | 2 +- datasette/hookspecs.py | 2 +- docs/plugin_hooks.rst | 9 +++++++-- tests/plugins/my_plugin.py | 3 ++- tests/test_plugins.py | 5 +++-- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index aeb81687..db686670 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -345,7 +345,7 @@ class Datasette: self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class # pylint: disable=no-member - pm.hook.prepare_jinja2_environment(env=self.jinja_env) + pm.hook.prepare_jinja2_environment(env=self.jinja_env, datasette=self) self._register_renderers() self._permission_checks = collections.deque(maxlen=200) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index a5fb536f..34e19664 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -26,7 +26,7 @@ def prepare_connection(conn, database, datasette): @hookspec -def prepare_jinja2_environment(env): +def prepare_jinja2_environment(env, datasette): """Modify Jinja2 template environment e.g. register custom template tags""" diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 30bd75b7..62ec5c90 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -61,12 +61,15 @@ Examples: `datasette-jellyfish `_, for @@ -85,6 +88,8 @@ You can now use this filter in your custom templates like so:: Table name: {{ table|uppercase }} +Examples: `datasette-edit-templates `_ + .. _plugin_hook_extra_template_vars: extra_template_vars(template, database, table, columns, view_name, request, datasette) diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index 53613b7d..d49a7a34 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -142,8 +142,9 @@ def extra_template_vars( @hookimpl -def prepare_jinja2_environment(env): +def prepare_jinja2_environment(env, datasette): env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}" + env.filters["to_hello"] = lambda s: datasette._HELLO @hookimpl diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 948a40b8..590d88f6 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -545,11 +545,12 @@ def test_hook_register_output_renderer_can_render(app_client): @pytest.mark.asyncio async def test_hook_prepare_jinja2_environment(app_client): + app_client.ds._HELLO = "HI" template = app_client.ds.jinja_env.from_string( - "Hello there, {{ a|format_numeric }}", {"a": 3412341} + "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}", {"a": 3412341} ) rendered = await app_client.ds.render_template(template) - assert "Hello there, 3,412,341" == rendered + assert "Hello there, 3,412,341, HI" == rendered def test_hook_publish_subcommand(): From 2ebcffe2226ece2a5a86722790d486a480338632 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Sep 2022 12:50:52 -0700 Subject: [PATCH 0032/1116] Bump furo from 2022.6.21 to 2022.9.15 (#1812) Bumps [furo](https://github.com/pradyunsg/furo) from 2022.6.21 to 2022.9.15. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2022.06.21...2022.09.15) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 92fa60d0..afcba1f0 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "furo==2022.6.21", + "furo==2022.9.15", "sphinx-autobuild", "codespell", "blacken-docs", From ddc999ad1296e8c69cffede3e367dda059b8adad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 16 Sep 2022 20:38:15 -0700 Subject: [PATCH 0033/1116] Async support for prepare_jinja2_environment, closes #1809 --- datasette/app.py | 22 ++++++++++++++--- datasette/utils/testing.py | 1 + docs/plugin_hooks.rst | 2 ++ docs/testing_plugins.rst | 30 ++++++++++++++++++++++++ tests/fixtures.py | 1 + tests/plugins/my_plugin.py | 10 ++++++-- tests/plugins/my_plugin_2.py | 6 +++++ tests/test_internals_datasette_client.py | 6 +++-- tests/test_plugins.py | 6 +++-- tests/test_routes.py | 1 + 10 files changed, 76 insertions(+), 9 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index db686670..ea3e7b43 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -208,6 +208,7 @@ class Datasette: crossdb=False, nolock=False, ): + self._startup_invoked = False assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" @@ -344,9 +345,6 @@ class Datasette: self.jinja_env.filters["quote_plus"] = urllib.parse.quote_plus self.jinja_env.filters["escape_sqlite"] = escape_sqlite self.jinja_env.filters["to_css_class"] = to_css_class - # pylint: disable=no-member - pm.hook.prepare_jinja2_environment(env=self.jinja_env, datasette=self) - self._register_renderers() self._permission_checks = collections.deque(maxlen=200) self._root_token = secrets.token_hex(32) @@ -389,8 +387,16 @@ class Datasette: return Urls(self) async def invoke_startup(self): + # This must be called for Datasette to be in a usable state + if self._startup_invoked: + return + for hook in pm.hook.prepare_jinja2_environment( + env=self.jinja_env, datasette=self + ): + await await_me_maybe(hook) for hook in pm.hook.startup(datasette=self): await await_me_maybe(hook) + self._startup_invoked = True def sign(self, value, namespace="default"): return URLSafeSerializer(self._secret, namespace).dumps(value) @@ -933,6 +939,8 @@ class Datasette: async def render_template( self, templates, context=None, request=None, view_name=None ): + if not self._startup_invoked: + raise Exception("render_template() called before await ds.invoke_startup()") context = context or {} if isinstance(templates, Template): template = templates @@ -1495,34 +1503,42 @@ class DatasetteClient: return path async def get(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.get(self._fix(path), **kwargs) async def options(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.options(self._fix(path), **kwargs) async def head(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.head(self._fix(path), **kwargs) async def post(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.post(self._fix(path), **kwargs) async def put(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.put(self._fix(path), **kwargs) async def patch(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.patch(self._fix(path), **kwargs) async def delete(self, path, **kwargs): + await self.ds.invoke_startup() async with httpx.AsyncClient(app=self.app) as client: return await client.delete(self._fix(path), **kwargs) async def request(self, method, path, **kwargs): + await self.ds.invoke_startup() avoid_path_rewrites = kwargs.pop("avoid_path_rewrites", None) async with httpx.AsyncClient(app=self.app) as client: return await client.request( diff --git a/datasette/utils/testing.py b/datasette/utils/testing.py index 640c94e6..b28fc575 100644 --- a/datasette/utils/testing.py +++ b/datasette/utils/testing.py @@ -147,6 +147,7 @@ class TestClient: content_type=None, if_none_match=None, ): + await self.ds.invoke_startup() headers = headers or {} if content_type: headers["content-type"] = content_type diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 62ec5c90..f208e727 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -88,6 +88,8 @@ You can now use this filter in your custom templates like so:: Table name: {{ table|uppercase }} +This function can return an awaitable function if it needs to run any async code. + Examples: `datasette-edit-templates `_ .. _plugin_hook_extra_template_vars: diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 992b4b0e..41f50e56 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -52,6 +52,36 @@ Then run the tests using pytest like so:: pytest +.. _testing_plugins_datasette_test_instance: + +Setting up a Datasette test instance +------------------------------------ + +The above example shows the easiest way to start writing tests against a Datasette instance: + +.. code-block:: python + + from datasette.app import Datasette + import pytest + + + @pytest.mark.asyncio + async def test_plugin_is_installed(): + datasette = Datasette(memory=True) + response = await datasette.client.get("/-/plugins.json") + assert response.status_code == 200 + +Creating a ``Datasette()`` instance like this as useful shortcut in tests, but there is one detail you need to be aware of. It's important to ensure that the async method ``.invoke_startup()`` is called on that instance. You can do that like this: + +.. code-block:: python + + datasette = Datasette(memory=True) + await datasette.invoke_startup() + +This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls. + +If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - those method calls ensure that ``.invoke_startup()`` has been called for you. + .. _testing_plugins_pdb: Using pdb for errors thrown inside Datasette diff --git a/tests/fixtures.py b/tests/fixtures.py index 82d8452e..5a875cd2 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -71,6 +71,7 @@ EXPECTED_PLUGINS = [ "handle_exception", "menu_links", "permission_allowed", + "prepare_jinja2_environment", "register_routes", "render_cell", "startup", diff --git a/tests/plugins/my_plugin.py b/tests/plugins/my_plugin.py index d49a7a34..1a41de38 100644 --- a/tests/plugins/my_plugin.py +++ b/tests/plugins/my_plugin.py @@ -143,8 +143,14 @@ def extra_template_vars( @hookimpl def prepare_jinja2_environment(env, datasette): - env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}" - env.filters["to_hello"] = lambda s: datasette._HELLO + async def select_times_three(s): + db = datasette.get_database() + return (await db.execute("select 3 * ?", [int(s)])).first()[0] + + async def inner(): + env.filters["select_times_three"] = select_times_three + + return inner @hookimpl diff --git a/tests/plugins/my_plugin_2.py b/tests/plugins/my_plugin_2.py index 4df02343..cee80703 100644 --- a/tests/plugins/my_plugin_2.py +++ b/tests/plugins/my_plugin_2.py @@ -126,6 +126,12 @@ def permission_allowed(datasette, actor, action): return inner +@hookimpl +def prepare_jinja2_environment(env, datasette): + env.filters["format_numeric"] = lambda s: f"{float(s):,.0f}" + env.filters["to_hello"] = lambda s: datasette._HELLO + + @hookimpl def startup(datasette): async def inner(): diff --git a/tests/test_internals_datasette_client.py b/tests/test_internals_datasette_client.py index 8c5b5bd3..497bf475 100644 --- a/tests/test_internals_datasette_client.py +++ b/tests/test_internals_datasette_client.py @@ -1,10 +1,12 @@ from .fixtures import app_client import httpx import pytest +import pytest_asyncio -@pytest.fixture -def datasette(app_client): +@pytest_asyncio.fixture +async def datasette(app_client): + await app_client.ds.invoke_startup() return app_client.ds diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 590d88f6..0ae3abf3 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -546,11 +546,13 @@ def test_hook_register_output_renderer_can_render(app_client): @pytest.mark.asyncio async def test_hook_prepare_jinja2_environment(app_client): app_client.ds._HELLO = "HI" + await app_client.ds.invoke_startup() template = app_client.ds.jinja_env.from_string( - "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}", {"a": 3412341} + "Hello there, {{ a|format_numeric }}, {{ a|to_hello }}, {{ b|select_times_three }}", + {"a": 3412341, "b": 5}, ) rendered = await app_client.ds.render_template(template) - assert "Hello there, 3,412,341, HI" == rendered + assert "Hello there, 3,412,341, HI, 15" == rendered def test_hook_publish_subcommand(): diff --git a/tests/test_routes.py b/tests/test_routes.py index 5ae55d21..d467abe1 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -59,6 +59,7 @@ def test_routes(routes, path, expected_class, expected_matches): @pytest_asyncio.fixture async def ds_with_route(): ds = Datasette() + await ds.invoke_startup() ds.remove_database("_memory") db = Database(ds, is_memory=True, memory_name="route-name-db") ds.add_database(db, name="original-name", route="custom-route-name") From df851c117db031dec50dd4ef1ca34745920ac77a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Sep 2022 16:46:39 -0700 Subject: [PATCH 0034/1116] Validate settings.json keys on startup, closes #1816 Refs #1814 --- datasette/app.py | 4 ++++ tests/test_config_dir.py | 20 ++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index ea3e7b43..8873ce28 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -292,6 +292,10 @@ class Datasette: raise StartupError("config.json should be renamed to settings.json") if config_dir and (config_dir / "settings.json").exists() and not settings: settings = json.loads((config_dir / "settings.json").read_text()) + # Validate those settings + for key in settings: + if key not in DEFAULT_SETTINGS: + raise StartupError("Invalid setting '{key}' in settings.json") self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index fe927c42..e365515b 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -5,6 +5,7 @@ import pytest from datasette.app import Datasette from datasette.cli import cli from datasette.utils.sqlite import sqlite3 +from datasette.utils import StartupError from .fixtures import TestClient as _TestClient from click.testing import CliRunner @@ -27,9 +28,8 @@ body { margin-top: 3em} @pytest.fixture(scope="session") -def config_dir_client(tmp_path_factory): +def config_dir(tmp_path_factory): config_dir = tmp_path_factory.mktemp("config-dir") - plugins_dir = config_dir / "plugins" plugins_dir.mkdir() (plugins_dir / "hooray.py").write_text(PLUGIN, "utf-8") @@ -77,7 +77,23 @@ def config_dir_client(tmp_path_factory): ), "utf-8", ) + return config_dir + +def test_invalid_settings(config_dir): + previous = (config_dir / "settings.json").read_text("utf-8") + (config_dir / "settings.json").write_text( + json.dumps({"invalid": "invalid-setting"}), "utf-8" + ) + try: + with pytest.raises(StartupError): + ds = Datasette([], config_dir=config_dir) + finally: + (config_dir / "settings.json").write_text(previous, "utf-8") + + +@pytest.fixture(scope="session") +def config_dir_client(config_dir): ds = Datasette([], config_dir=config_dir) yield _TestClient(ds) From cb1e093fd361b758120aefc1a444df02462389a3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 19 Sep 2022 18:15:40 -0700 Subject: [PATCH 0035/1116] Fixed error message, closes #1816 --- datasette/app.py | 4 +++- tests/test_config_dir.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 8873ce28..03d1dacc 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -295,7 +295,9 @@ class Datasette: # Validate those settings for key in settings: if key not in DEFAULT_SETTINGS: - raise StartupError("Invalid setting '{key}' in settings.json") + raise StartupError( + "Invalid setting '{}' in settings.json".format(key) + ) self._settings = dict(DEFAULT_SETTINGS, **(settings or {})) self.renderers = {} # File extension -> (renderer, can_render) functions self.version_note = version_note diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index e365515b..f5ecf0d6 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -86,8 +86,9 @@ def test_invalid_settings(config_dir): json.dumps({"invalid": "invalid-setting"}), "utf-8" ) try: - with pytest.raises(StartupError): + with pytest.raises(StartupError) as ex: ds = Datasette([], config_dir=config_dir) + assert ex.value.args[0] == "Invalid setting 'invalid' in settings.json" finally: (config_dir / "settings.json").write_text(previous, "utf-8") From 212137a90b4291db9605e039f198564dae59c5d0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 14:14:25 -0700 Subject: [PATCH 0036/1116] Release 0.63a0 Refs #1786, #1787, #1789, #1794, #1800, #1804, #1805, #1808, #1809, #1816 --- datasette/version.py | 2 +- docs/changelog.rst | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 0453346c..e5ad585f 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "0.62" +__version__ = "0.63a0" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index f9dcc980..bd93f4cb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,23 @@ Changelog ========= +.. _v0_63a0: + +0.63a0 (2022-09-26) +------------------- + +- The :ref:`plugin_hook_prepare_jinja2_environment` plugin hook now accepts an optional ``datasette`` argument. Hook implementations can also now return an ``async`` function which will be awaited automatically. (:issue:`1809`) +- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 `__) +- New tutorial: `Cleaning data with sqlite-utils and Datasette `__. +- Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`) +- ``truncate_cells_html`` setting now also affects long URLs in columns. (:issue:`1805`) +- ``Database(is_mutable=)`` now defaults to ``True``. (:issue:`1808`) +- Non-JavaScript textarea now increases height to fit the SQL query. (:issue:`1786`) +- More detailed command descriptions on the :ref:`CLI reference ` page. (:issue:`1787`) +- Datasette no longer enforces upper bounds on its depenedencies. (:issue:`1800`) +- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 `__) +- The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`) + .. _v0_62: 0.62 (2022-08-14) From 5f9f567acbc58c9fcd88af440e68034510fb5d2b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 26 Sep 2022 16:06:01 -0700 Subject: [PATCH 0037/1116] Show SQL query when reporting time limit error, closes #1819 --- datasette/database.py | 5 ++++- datasette/views/base.py | 21 +++++++++++++-------- tests/test_api.py | 12 +++++++++++- tests/test_html.py | 10 +++++++--- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 44467370..46094bd7 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -476,7 +476,10 @@ class WriteTask: class QueryInterrupted(Exception): - pass + def __init__(self, e, sql, params): + self.e = e + self.sql = sql + self.params = params class MultipleValues(Exception): diff --git a/datasette/views/base.py b/datasette/views/base.py index 221e1882..67aa3a42 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -1,10 +1,12 @@ import asyncio import csv import hashlib -import re import sys +import textwrap import time import urllib +from markupsafe import escape + import pint @@ -24,11 +26,9 @@ from datasette.utils import ( path_with_removed_args, path_with_format, sqlite3, - HASH_LENGTH, ) from datasette.utils.asgi import ( AsgiStream, - Forbidden, NotFound, Response, BadRequest, @@ -371,13 +371,18 @@ class DataView(BaseView): ) = response_or_template_contexts else: data, extra_template_data, templates = response_or_template_contexts - except QueryInterrupted: + except QueryInterrupted as ex: raise DatasetteError( - """ - SQL query took too long. The time limit is controlled by the + textwrap.dedent( + """ +

SQL query took too long. The time limit is controlled by the sql_time_limit_ms - configuration option. - """, + configuration option.

+ 
{}
+ """.format( + escape(ex.sql) + ) + ).strip(), title="SQL Interrupted", status=400, message_is_html=True, diff --git a/tests/test_api.py b/tests/test_api.py index 7a2bf91f..ad74d16e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -656,7 +656,17 @@ def test_custom_sql(app_client): def test_sql_time_limit(app_client_shorter_time_limit): response = app_client_shorter_time_limit.get("/fixtures.json?sql=select+sleep(0.5)") assert 400 == response.status - assert "SQL Interrupted" == response.json["title"] + assert response.json == { + "ok": False, + "error": ( + "

SQL query took too long. The time limit is controlled by the\n" + 'sql_time_limit_ms\n' + "configuration option.

\n" + "
select sleep(0.5)
" + ), + "status": 400, + "title": "SQL Interrupted", + } def test_custom_sql_time_limit(app_client): diff --git a/tests/test_html.py b/tests/test_html.py index bf915247..a99b0b6c 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -168,10 +168,14 @@ def test_disallowed_custom_sql_pragma(app_client): def test_sql_time_limit(app_client_shorter_time_limit): response = app_client_shorter_time_limit.get("/fixtures?sql=select+sleep(0.5)") assert 400 == response.status - expected_html_fragment = """ + expected_html_fragments = [ + """ sql_time_limit_ms - """.strip() - assert expected_html_fragment in response.text + """.strip(), + "
select sleep(0.5)
", + ] + for expected_html_fragment in expected_html_fragments: + assert expected_html_fragment in response.text def test_row_page_does_not_truncate(): From 7fb4ea4e39a15e1f7d3202949794d98af1cfa272 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 27 Sep 2022 21:06:40 -0700 Subject: [PATCH 0038/1116] Update note about render_cell signature, refs #1826 --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index f208e727..c9cab8ab 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -9,7 +9,7 @@ Each plugin can implement one or more hooks using the ``@hookimpl`` decorator ag When you implement a plugin hook you can accept any or all of the parameters that are documented as being passed to that hook. -For example, you can implement the ``render_cell`` plugin hook like this even though the full documented hook signature is ``render_cell(value, column, table, database, datasette)``: +For example, you can implement the ``render_cell`` plugin hook like this even though the full documented hook signature is ``render_cell(row, value, column, table, database, datasette)``: .. code-block:: python From 984b1df12cf19a6731889fc0665bb5f622e07b7c Mon Sep 17 00:00:00 2001 From: Adam Simpson Date: Wed, 28 Sep 2022 00:21:36 -0400 Subject: [PATCH 0039/1116] Add documentation for serving via OpenRC (#1825) * Add documentation for serving via OpenRC --- docs/deploying.rst | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/deploying.rst b/docs/deploying.rst index d4ad8836..c8552758 100644 --- a/docs/deploying.rst +++ b/docs/deploying.rst @@ -74,18 +74,30 @@ Once the service has started you can confirm that Datasette is running on port 8 curl 127.0.0.1:8000/-/versions.json # Should output JSON showing the installed version -Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx``. +Datasette will not be accessible from outside the server because it is listening on ``127.0.0.1``. You can expose it by instead listening on ``0.0.0.0``, but a better way is to set up a proxy such as ``nginx`` - see :ref:`deploying_proxy`. -Ubuntu offer `a tutorial on installing nginx `__. Once it is installed you can add configuration to proxy traffic through to Datasette that looks like this:: +.. _deploying_openrc: - server { - server_name mysubdomain.myhost.net; +Running Datasette using OpenRC +=============================== +OpenRC is the service manager on non-systemd Linux distributions like `Alpine Linux `__ and `Gentoo `__. - location / { - proxy_pass http://127.0.0.1:8000/; - proxy_set_header Host $host; - } - } +Create an init script at ``/etc/init.d/datasette`` with the following contents: + +.. code-block:: sh + + #!/sbin/openrc-run + + name="datasette" + command="datasette" + command_args="serve -h 0.0.0.0 /path/to/db.db" + command_background=true + pidfile="/run/${RC_SVCNAME}.pid" + +You then need to configure the service to run at boot and start it:: + + rc-update add datasette + rc-service datasette start .. _deploying_buildpacks: From 34defdc10aa293294ca01cfab70780755447e1d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 28 Sep 2022 17:39:36 -0700 Subject: [PATCH 0040/1116] Browse the plugins directory --- docs/writing_plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/writing_plugins.rst b/docs/writing_plugins.rst index 01ee8c90..a3fc88ec 100644 --- a/docs/writing_plugins.rst +++ b/docs/writing_plugins.rst @@ -234,7 +234,7 @@ To avoid accidentally conflicting with a database file that may be loaded into D - ``/-/upload-excel`` -Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `datasette-plugin topic `__ on GitHub. +Try to avoid registering URLs that clash with other plugins that your users might have installed. There is no central repository of reserved URL paths (yet) but you can review existing plugins by browsing the `plugins directory `. If your plugin includes functionality that relates to a specific database you could also register a URL route like this: From c92c4318e9892101f75fa158410c0a12c1d80b6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Sep 2022 10:55:40 -0700 Subject: [PATCH 0041/1116] Bump furo from 2022.9.15 to 2022.9.29 (#1827) Bumps [furo](https://github.com/pradyunsg/furo) from 2022.9.15 to 2022.9.29. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2022.09.15...2022.09.29) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index afcba1f0..fe258adb 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( setup_requires=["pytest-runner"], extras_require={ "docs": [ - "furo==2022.9.15", + "furo==2022.9.29", "sphinx-autobuild", "codespell", "blacken-docs", From 883e326dd6ef95f854f7750ef2d4b0e17082fa96 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 2 Oct 2022 14:26:16 -0700 Subject: [PATCH 0042/1116] Drop word-wrap: anywhere, refs #1828, #1805 --- datasette/static/app.css | 1 - 1 file changed, 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 08b724f6..712b9925 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -446,7 +446,6 @@ th { } table a:link { text-decoration: none; - word-wrap: anywhere; } .rows-and-columns td:before { display: block; From 4218c9cd742b79b1e3cb80878e42b7e39d16ded2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Oct 2022 11:45:36 -0700 Subject: [PATCH 0043/1116] reST markup fix --- docs/plugin_hooks.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index c9cab8ab..832a76b0 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -268,7 +268,7 @@ you have one: def extra_js_urls(): return ["/-/static-plugins/your-plugin/app.js"] -Note that `your-plugin` here should be the hyphenated plugin name - the name that is displayed in the list on the `/-/plugins` debug page. +Note that ``your-plugin`` here should be the hyphenated plugin name - the name that is displayed in the list on the ``/-/plugins`` debug page. If your code uses `JavaScript modules `__ you should include the ``"module": True`` key. See :ref:`customization_css_and_javascript` for more details. From b6ba117b7978b58b40e3c3c2b723b92c3010ed53 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Oct 2022 18:25:52 -0700 Subject: [PATCH 0044/1116] Clarify request or None for two hooks --- docs/plugin_hooks.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 832a76b0..b61f953a 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1281,7 +1281,7 @@ menu_links(datasette, actor, request) ``actor`` - dictionary or None The currently authenticated :ref:`actor `. -``request`` - :ref:`internals_request` +``request`` - :ref:`internals_request` or None The current HTTP request. This can be ``None`` if the request object is not available. This hook allows additional items to be included in the menu displayed by Datasette's top right menu icon. @@ -1330,7 +1330,7 @@ table_actions(datasette, actor, database, table, request) ``table`` - string The name of the table. -``request`` - :ref:`internals_request` +``request`` - :ref:`internals_request` or None The current HTTP request. This can be ``None`` if the request object is not available. This hook allows table actions to be displayed in a menu accessed via an action icon at the top of the table page. It should return a list of ``{"href": "...", "label": "..."}`` menu items. From bbf33a763537a1d913180b22bd3b5fe4a5e5b252 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 4 Oct 2022 21:32:11 -0700 Subject: [PATCH 0045/1116] Test for bool(results), closes #1832 --- tests/test_internals_database.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 9e81c1d6..4e33beed 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -30,6 +30,14 @@ async def test_results_first(db): assert isinstance(row, sqlite3.Row) +@pytest.mark.asyncio +@pytest.mark.parametrize("expected", (True, False)) +async def test_results_bool(db, expected): + where = "" if expected else "where pk = 0" + results = await db.execute("select * from facetable {}".format(where)) + assert bool(results) is expected + + @pytest.mark.parametrize( "query,expected", [ From eff112498ecc499323c26612d707908831446d25 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Thu, 6 Oct 2022 16:06:06 -0400 Subject: [PATCH 0046/1116] Useuse inspect data for hash and file size on startup Thanks, @fgregg Closes #1834 --- datasette/database.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 46094bd7..d75bd70c 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -48,9 +48,13 @@ class Database: self._read_connection = None self._write_connection = None if not self.is_mutable and not self.is_memory: - p = Path(path) - self.hash = inspect_hash(p) - self.cached_size = p.stat().st_size + if self.ds.inspect_data and self.ds.inspect_data.get(self.name): + self.hash = self.ds.inspect_data[self.name]["hash"] + self.cached_size = self.ds.inspect_data[self.name]["size"] + else: + p = Path(path) + self.hash = inspect_hash(p) + self.cached_size = p.stat().st_size @property def cached_table_counts(self): From b7fec7f9020b79c1fe60cc5a2def86b50eeb5af9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 7 Oct 2022 16:03:09 -0700 Subject: [PATCH 0047/1116] .sqlite/.sqlite3 extensions for config directory mode Closes #1646 --- datasette/app.py | 5 ++++- docs/settings.rst | 2 +- tests/test_config_dir.py | 11 +++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 03d1dacc..32a911c2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -217,7 +217,10 @@ class Datasette: self._secret = secret or secrets.token_hex(32) self.files = tuple(files or []) + tuple(immutables or []) if config_dir: - self.files += tuple([str(p) for p in config_dir.glob("*.db")]) + db_files = [] + for ext in ("db", "sqlite", "sqlite3"): + db_files.extend(config_dir.glob("*.{}".format(ext))) + self.files += tuple(str(f) for f in db_files) if ( config_dir and (config_dir / "inspect-data.json").exists() diff --git a/docs/settings.rst b/docs/settings.rst index 8437fb04..a6d50543 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -46,7 +46,7 @@ Datasette will detect the files in that directory and automatically configure it The files that can be included in this directory are as follows. All are optional. -* ``*.db`` - SQLite database files that will be served by Datasette +* ``*.db`` (or ``*.sqlite3`` or ``*.sqlite``) - SQLite database files that will be served by Datasette * ``metadata.json`` - :ref:`metadata` for those databases - ``metadata.yaml`` or ``metadata.yml`` can be used as well * ``inspect-data.json`` - the result of running ``datasette inspect *.db --inspect-file=inspect-data.json`` from the configuration directory - any database files listed here will be treated as immutable, so they should not be changed while Datasette is running * ``settings.json`` - settings that would normally be passed using ``--setting`` - here they should be stored as a JSON object of key/value pairs diff --git a/tests/test_config_dir.py b/tests/test_config_dir.py index f5ecf0d6..c2af3836 100644 --- a/tests/test_config_dir.py +++ b/tests/test_config_dir.py @@ -49,7 +49,7 @@ def config_dir(tmp_path_factory): (config_dir / "metadata.json").write_text(json.dumps(METADATA), "utf-8") (config_dir / "settings.json").write_text(json.dumps(SETTINGS), "utf-8") - for dbname in ("demo.db", "immutable.db"): + for dbname in ("demo.db", "immutable.db", "j.sqlite3", "k.sqlite"): db = sqlite3.connect(str(config_dir / dbname)) db.executescript( """ @@ -151,12 +151,11 @@ def test_databases(config_dir_client): response = config_dir_client.get("/-/databases.json") assert 200 == response.status databases = response.json - assert 2 == len(databases) + assert 4 == len(databases) databases.sort(key=lambda d: d["name"]) - assert "demo" == databases[0]["name"] - assert databases[0]["is_mutable"] - assert "immutable" == databases[1]["name"] - assert not databases[1]["is_mutable"] + for db, expected_name in zip(databases, ("demo", "immutable", "j", "k")): + assert expected_name == db["name"] + assert db["is_mutable"] == (expected_name != "immutable") @pytest.mark.parametrize("filename", ("metadata.yml", "metadata.yaml")) From 1a5e5f2aa951e5bd731067a49819efba68fbe8ef Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Oct 2022 14:42:52 -0700 Subject: [PATCH 0048/1116] Refactor breadcrumbs to respect permissions, refs #1831 --- datasette/app.py | 40 ++++++++++++++++++++++ datasette/templates/_crumbs.html | 15 ++++++++ datasette/templates/base.html | 4 +-- datasette/templates/database.html | 9 ----- datasette/templates/error.html | 7 ---- datasette/templates/logout.html | 7 ---- datasette/templates/permissions_debug.html | 7 ---- datasette/templates/query.html | 8 ++--- datasette/templates/row.html | 9 ++--- datasette/templates/show_json.html | 7 ---- datasette/templates/table.html | 8 ++--- tests/test_permissions.py | 1 + tests/test_plugins.py | 2 +- 13 files changed, 65 insertions(+), 59 deletions(-) create mode 100644 datasette/templates/_crumbs.html diff --git a/datasette/app.py b/datasette/app.py index 32a911c2..5fa4955c 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -631,6 +631,44 @@ class Datasette: else: return [] + async def _crumb_items(self, request, table=None, database=None): + crumbs = [] + # Top-level link + if await self.permission_allowed( + actor=request.actor, action="view-instance", default=True + ): + crumbs.append({"href": self.urls.instance(), "label": "home"}) + # Database link + if database: + if await self.permission_allowed( + actor=request.actor, + action="view-database", + resource=database, + default=True, + ): + crumbs.append( + { + "href": self.urls.database(database), + "label": database, + } + ) + # Table link + if table: + assert database, "table= requires database=" + if await self.permission_allowed( + actor=request.actor, + action="view-table", + resource=(database, table), + default=True, + ): + crumbs.append( + { + "href": self.urls.table(database, table), + "label": table, + } + ) + return crumbs + async def permission_allowed(self, actor, action, resource=None, default=False): """Check permissions using the permissions_allowed plugin hook""" result = None @@ -1009,6 +1047,8 @@ class Datasette: template_context = { **context, **{ + "request": request, + "crumb_items": self._crumb_items, "urls": self.urls, "actor": request.actor if request else None, "menu_links": menu_links, diff --git a/datasette/templates/_crumbs.html b/datasette/templates/_crumbs.html new file mode 100644 index 00000000..bd1ff0da --- /dev/null +++ b/datasette/templates/_crumbs.html @@ -0,0 +1,15 @@ +{% macro nav(request, database=None, table=None) -%} +{% if crumb_items is defined %} + {% set items=crumb_items(request=request, database=database, table=table) %} + {% if items %} +

+ {% for item in items %} + {{ item.label }} + {% if not loop.last %} + / + {% endif %} + {% endfor %} +

+ {% endif %} +{% endif %} +{%- endmacro %} diff --git a/datasette/templates/base.html b/datasette/templates/base.html index c3a71acb..87c939ac 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -1,4 +1,4 @@ - +{% import "_crumbs.html" as crumbs with context %} {% block title %}{% endblock %} @@ -17,7 +17,7 @@
-
+ + + + +{% include "_codemirror_foot.html" %} +{% include "_sql_parameter_scripts.html" %} +{% include "_execute_write_analysis_scripts.html" %} + + + +{% endblock %} diff --git a/datasette/templates/query_list.html b/datasette/templates/query_list.html index fa4859b1..a8c9a391 100644 --- a/datasette/templates/query_list.html +++ b/datasette/templates/query_list.html @@ -205,32 +205,32 @@

Queries

-
-

- - - {% if filters.is_write %}{% endif %} - {% if filters.is_private %}{% endif %} - {% if filters.source %}{% endif %} - {% if filters.owner_id %}{% endif %} - -

-
- - - {% if queries %} +
+

+ + + {% if filters.is_write %}{% endif %} + {% if filters.is_private %}{% endif %} + {% if filters.source %}{% endif %} + {% if filters.owner_id %}{% endif %} + +

+
+ + +
diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index f30a30bc..9efe3f81 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -436,6 +436,35 @@ async def _query_create_form_context( } +async def _query_edit_form_context( + datasette, + request, + db, + existing: StoredQuery, + *, + sql=None, + title=None, + description=None, + is_private=None, +): + sql = existing.sql if sql is None else sql + title = existing.title if title is None else title + description = existing.description if description is None else description + is_private = existing.is_private if is_private is None else is_private + analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor) + return { + "database": db.name, + "database_color": db.color, + "name": existing.name, + "sql": sql, + "title": title or "", + "description": description or "", + "is_private": is_private, + "query_url": datasette.urls.table(db.name, existing.name), + **analysis_data, + } + + async def _inserted_row_url(datasette, db, analysis, cursor): if cursor.rowcount != 1: return None diff --git a/datasette/views/stored_queries.py b/datasette/views/stored_queries.py index 8c4e849e..2753f876 100644 --- a/datasette/views/stored_queries.py +++ b/datasette/views/stored_queries.py @@ -18,6 +18,7 @@ from .query_helpers import ( _query_create_analysis_data, _query_create_form_context, _query_create_form_error_message, + _query_edit_form_context, _query_list_limit, ) @@ -464,13 +465,164 @@ class QueryUpdateView(BaseView): return Response.json({"ok": True}) -class QueryDeleteView(BaseView): - name = "query-delete" +class QueryEditView(BaseView): + name = "query-edit" + has_json_alternate = False - async def post(self, request): + async def _load(self, request): db = await self.ds.resolve_database(request) query_name = tilde_decode(request.url_vars["query"]) existing = await self.ds.get_query(db.name, query_name) + return db, query_name, existing + + async def _render_form( + self, + request, + db, + existing, + *, + sql=None, + title=None, + description=None, + is_private=None, + status=200, + ): + response = await self.render( + ["query_edit.html"], + request, + await _query_edit_form_context( + self.ds, + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + ), + ) + response.status = status + return response + + async def get(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + await self.ds.ensure_permission( + action="update-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ) + if existing.is_trusted: + return _error(["Trusted queries cannot be edited"], 403) + return await self._render_form(request, db, existing) + + async def post(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + if not await self.ds.allowed( + action="update-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ): + return _error(["Permission denied: need update-query"], 403) + if existing.is_trusted: + return _error(["Trusted queries cannot be edited"], 403) + + data, _ = await _json_or_form_payload(request) + if not isinstance(data, dict): + return _error(["Invalid form submission"], 400) + sql = data.get("sql") + sql = existing.sql if sql is None else sql.strip() + title = data.get("title") or "" + description = data.get("description") or "" + is_private = _as_bool(data.get("is_private")) + + update = { + "title": title, + "description": description, + "is_private": is_private, + } + if sql != existing.sql: + if not await self.ds.allowed( + action="execute-sql", + resource=DatabaseResource(db.name), + actor=request.actor, + ): + self.ds.add_message( + request, + "Permission denied: need execute-sql to change the SQL", + self.ds.ERROR, + ) + return await self._render_form( + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + status=403, + ) + update["sql"] = sql + + try: + update_kwargs = await _prepare_query_update( + self.ds, request, db, existing, update + ) + except QueryValidationError as ex: + self.ds.add_message(request, ex.message, self.ds.ERROR) + return await self._render_form( + request, + db, + existing, + sql=sql, + title=title, + description=description, + is_private=is_private, + status=ex.status, + ) + + await self.ds.update_query(db.name, query_name, **update_kwargs) + self.ds.add_message(request, "Query updated", self.ds.INFO) + return Response.redirect( + self.ds.urls.path(self.ds.urls.table(db.name, query_name)) + ) + + +class QueryDeleteView(BaseView): + name = "query-delete" + has_json_alternate = False + + async def _load(self, request): + db = await self.ds.resolve_database(request) + query_name = tilde_decode(request.url_vars["query"]) + existing = await self.ds.get_query(db.name, query_name) + return db, query_name, existing + + async def get(self, request): + db, query_name, existing = await self._load(request) + if existing is None: + return _error(["Query not found: {}".format(query_name)], 404) + await self.ds.ensure_permission( + action="delete-query", + resource=QueryResource(db.name, query_name), + actor=request.actor, + ) + return await self.render( + ["query_delete.html"], + request, + { + "database": db.name, + "database_color": db.color, + "query": stored_query_to_dict(existing), + "query_url": self.ds.urls.table(db.name, query_name), + }, + ) + + async def post(self, request): + db, query_name, existing = await self._load(request) if existing is None: return _error(["Query not found: {}".format(query_name)], 404) if not await self.ds.allowed( @@ -479,5 +631,14 @@ class QueryDeleteView(BaseView): actor=request.actor, ): return _error(["Permission denied: need delete-query"], 403) + + data, is_json = await _json_or_form_payload(request) await self.ds.remove_query(db.name, query_name) - return Response.json({"ok": True}) + if is_json: + return Response.json({"ok": True}) + self.ds.add_message( + request, + "Query “{}” deleted".format(existing.title or query_name), + self.ds.INFO, + ) + return Response.redirect(self.ds.urls.path(self.ds.urls.database(db.name))) diff --git a/docs/changelog.rst b/docs/changelog.rst index d5f8fa14..75e4f3e8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v1_0_a33: + +1.0a33 (unreleased) +------------------- + +- Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) + .. _v1_0_a32: 1.0a32 (2026-05-31) diff --git a/docs/plugins.rst b/docs/plugins.rst index d578e9e2..c2eb282a 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -271,6 +271,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship "register_token_handler" ] }, + { + "name": "datasette.default_query_actions", + "static": false, + "templates": false, + "version": null, + "hooks": [ + "query_actions" + ] + }, { "name": "datasette.events", "static": false, diff --git a/docs/sql_queries.rst b/docs/sql_queries.rst index c0ba67f0..371348fb 100644 --- a/docs/sql_queries.rst +++ b/docs/sql_queries.rst @@ -142,6 +142,15 @@ Datasette stores both configured queries and user-created queries in the ``queri Stored queries created by users default to private. Private stored queries can only be viewed, updated or deleted by the actor that created them. Broad ``view-query``, ``update-query`` or ``delete-query`` permission grants still do not allow other actors to access another actor's private stored queries. +Editing and deleting stored queries ++++++++++++++++++++++++++++++++++++ + +The page for a stored query includes a "Query actions" menu with **Edit this query** and **Delete this query** links for actors who have permission to use them. + +The owner of a stored query can always edit and delete it. For queries that are not private, any actor granted the ``update-query`` or ``delete-query`` permission can edit or delete the query, even if they did not create it. Private queries can only be edited or deleted by their owner, regardless of any broad permission grants. + +Editing a query lets you change its title, description, SQL and whether it is private. Changing the SQL also requires the ``execute-sql`` permission (and the relevant write permissions for writable queries). The same operations are available through the JSON API by sending a ``POST`` to ``///-/update`` or ``///-/delete``. Trusted stored queries cannot be edited or deleted through the web interface or the JSON API. + Stored queries created by users are untrusted. This means they execute using the permissions of the actor who runs them, as if that actor had pasted the SQL into the regular custom SQL interface or write SQL interface. Read-only stored queries require ``execute-sql``. Writable stored queries require ``execute-write-sql`` plus the relevant table-level write permissions. SQL functions are allowed and are not separately restricted by Datasette permissions. .. _trusted_stored_queries: diff --git a/tests/test_docs.py b/tests/test_docs.py index 9cf39f41..51caf595 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -77,6 +77,7 @@ def documented_views(): "QueryCreateAnalyzeView", "QueryDeleteView", "QueryDefinitionView", + "QueryEditView", "QueryListView", "QueryParametersView", "QueryStoreView", diff --git a/tests/test_queries.py b/tests/test_queries.py index 25e423d4..6e9bcbdb 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -3,6 +3,7 @@ import re from html import unescape import pytest +from bs4 import BeautifulSoup as Soup from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource @@ -712,6 +713,10 @@ async def test_query_list_search_filter_and_html(): "/data/-/queries?is_private=1", actor={"id": "root"}, ) + no_results_response = await ds.client.get( + "/data/-/queries?q=nope", + actor={"id": "root"}, + ) assert html_response.status_code == 200 assert "Demo query 02" in html_response.text @@ -799,6 +804,13 @@ async def test_query_list_search_filter_and_html(): 'Not private0' not in filtered_private_response.text ) + assert no_results_response.status_code == 200 + assert "No queries found." in no_results_response.text + assert 'class="query-list-filters core"' not in no_results_response.text + assert 'id="query-search"' not in no_results_response.text + assert 'class="query-list-facets"' not in no_results_response.text + assert "

Mode

" not in no_results_response.text + assert "

Visibility

" not in no_results_response.text @pytest.mark.asyncio @@ -1114,6 +1126,227 @@ async def test_query_update_api_rejects_trusted_queries_but_internal_update_allo assert query.title == "Internal" +async def _make_ds_with_user_query(name, *, is_private=False, owner_id="owner"): + ds = Datasette(memory=True, settings={"default_allow_sql": True}) + db = ds.add_memory_database(name, name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "saved", + "select * from dogs", + title="Saved query", + description="A saved query", + source="user", + owner_id=owner_id, + is_private=is_private, + ) + return ds + + +@pytest.mark.asyncio +async def test_query_edit_form_renders_and_updates_for_owner(): + ds = await _make_ds_with_user_query("query_edit_owner") + actor = {"id": "owner"} + + # GET renders the form pre-filled with existing values + get_response = await ds.client.get("/data/saved/-/edit", actor=actor) + assert get_response.status_code == 200 + assert 'value="Saved query"' in get_response.text + assert ">A saved query" in get_response.text + assert "select * from dogs" in get_response.text + # URL slug is shown but not editable + assert 'name="name"' not in get_response.text + + # POST updates the query and redirects back to the query page + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=actor, + data={ + "title": "Updated title", + "description": "Updated description", + "sql": "select id from dogs", + "is_private": "1", + }, + ) + assert post_response.status_code == 302 + assert post_response.headers["location"] == "/data/saved" + + query = await ds.get_query("data", "saved") + assert query.title == "Updated title" + assert query.description == "Updated description" + assert query.sql == "select id from dogs" + assert query.is_private is True + + +@pytest.mark.asyncio +async def test_query_edit_metadata_only_does_not_require_execute_sql(): + # An owner who can no longer execute SQL can still edit title/description + ds = await _make_ds_with_user_query("query_edit_metadata_only") + actor = {"id": "owner"} + + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=actor, + data={ + "title": "Renamed", + "description": "A saved query", + "sql": "select * from dogs", + }, + ) + assert post_response.status_code == 302 + query = await ds.get_query("data", "saved") + assert query.title == "Renamed" + + +@pytest.mark.asyncio +async def test_private_query_edit_delete_restricted_to_owner(): + ds = await _make_ds_with_user_query( + "query_edit_private", is_private=True, owner_id="owner" + ) + + # A different actor cannot view, edit or delete the private query + other = {"id": "intruder"} + assert (await ds.client.get("/data/saved/-/edit", actor=other)).status_code == 403 + assert (await ds.client.get("/data/saved/-/delete", actor=other)).status_code == 403 + delete_attempt = await ds.client.post( + "/data/saved/-/delete", + actor=other, + data={}, + ) + assert delete_attempt.status_code == 403 + assert await ds.get_query("data", "saved") is not None + + # The owner can edit and delete + owner = {"id": "owner"} + assert (await ds.client.get("/data/saved/-/edit", actor=owner)).status_code == 200 + + +@pytest.mark.asyncio +async def test_non_private_query_editable_by_permitted_non_owner(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "execute-sql": {"id": "editor"}, + "update-query": {"id": "editor"}, + "delete-query": {"id": "editor"}, + } + } + } + }, + ) + db = ds.add_memory_database("query_non_private_editor", name="data") + await db.execute_write("create table dogs (id integer primary key, name text)") + await ds.invoke_startup() + await ds.add_query( + "data", + "saved", + "select * from dogs", + title="Shared", + source="user", + owner_id="owner", + is_private=False, + ) + + editor = {"id": "editor"} + # Editor (not the owner) can edit because the query is not private + post_response = await ds.client.post( + "/data/saved/-/edit", + actor=editor, + data={ + "title": "Edited by editor", + "description": "", + "sql": "select * from dogs", + }, + ) + assert post_response.status_code == 302 + query = await ds.get_query("data", "saved") + assert query.title == "Edited by editor" + + # Editor can also delete it + delete_response = await ds.client.post( + "/data/saved/-/delete", + actor=editor, + data={}, + ) + assert delete_response.status_code == 302 + assert await ds.get_query("data", "saved") is None + + +@pytest.mark.asyncio +async def test_query_delete_confirmation_and_form_delete(): + ds = await _make_ds_with_user_query("query_delete_form") + actor = {"id": "owner"} + + get_response = await ds.client.get("/data/saved/-/delete", actor=actor) + assert get_response.status_code == 200 + assert "Are you sure" in get_response.text + assert "select * from dogs" in get_response.text + soup = Soup(get_response.text, "html.parser") + form = soup.select_one("form.query-delete-form") + assert form is not None + assert "core" in form["class"] + assert form.select_one('input[type="submit"][value="Delete query"]') is not None + + post_response = await ds.client.post( + "/data/saved/-/delete", + actor=actor, + data={}, + ) + assert post_response.status_code == 302 + assert post_response.headers["location"] == "/data" + assert await ds.get_query("data", "saved") is None + + +@pytest.mark.asyncio +async def test_query_action_menu_shows_edit_and_delete_for_owner(): + ds = await _make_ds_with_user_query("query_action_menu") + + owner_response = await ds.client.get("/data/saved", actor={"id": "owner"}) + assert owner_response.status_code == 200 + assert "/data/saved/-/edit" in owner_response.text + assert "/data/saved/-/delete" in owner_response.text + + # A different actor (the query is public) cannot edit/delete by default + other_response = await ds.client.get("/data/saved", actor={"id": "stranger"}) + assert other_response.status_code == 200 + assert "/data/saved/-/edit" not in other_response.text + assert "/data/saved/-/delete" not in other_response.text + + +@pytest.mark.asyncio +async def test_query_edit_rejected_for_trusted_query(): + ds = Datasette( + memory=True, + default_deny=True, + config={ + "databases": { + "data": { + "permissions": { + "execute-sql": {"id": "editor"}, + "update-query": {"id": "editor"}, + }, + "queries": {"trusted_report": {"sql": "select 1 as one"}}, + } + } + }, + ) + ds.add_memory_database("query_edit_trusted", name="data") + await ds.invoke_startup() + + response = await ds.client.get( + "/data/trusted_report/-/edit", actor={"id": "editor"} + ) + assert response.status_code == 403 + # Edit/delete links should not appear on a trusted/config query page + page = await ds.client.get("/data/trusted_report", actor={"id": "editor"}) + assert "/data/trusted_report/-/edit" not in page.text + + @pytest.mark.asyncio async def test_query_store_api_rejects_magic_parameters(): ds = Datasette(memory=True, default_deny=True) From 03f1ffdf8fbf0ed7da46be48f3fcc3f4698e1e21 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:45:01 -0700 Subject: [PATCH 1079/1116] Centralize JSON extra parsing --- datasette/extras.py | 6 ++++++ datasette/renderer.py | 3 ++- datasette/views/table.py | 7 ++----- tests/test_table_api.py | 11 +++++++++++ 4 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 datasette/extras.py diff --git a/datasette/extras.py b/datasette/extras.py new file mode 100644 index 00000000..01a9fb4b --- /dev/null +++ b/datasette/extras.py @@ -0,0 +1,6 @@ +def extra_names_from_request(request): + extra_bits = request.args.getlist("_extra") + extras = set() + for bit in extra_bits: + extras.update(part for part in bit.split(",") if part) + return extras diff --git a/datasette/renderer.py b/datasette/renderer.py index acf23e59..f40e3dbb 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -1,4 +1,5 @@ import json +from datasette.extras import extra_names_from_request from datasette.utils import ( value_as_boolean, remove_infinites, @@ -108,7 +109,7 @@ def json_renderer(request, args, data, error, truncated=None): # Don't include "columns" in output # https://github.com/simonw/datasette/issues/2136 - if isinstance(data, dict) and "columns" not in request.args.getlist("_extra"): + if isinstance(data, dict) and "columns" not in extra_names_from_request(request): data.pop("columns", None) # Handle _nl option for _shape=array diff --git a/datasette/views/table.py b/datasette/views/table.py index 4df1e1b4..9ba249f4 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -6,6 +6,7 @@ import urllib from asyncinject import Registry import markupsafe +from datasette.extras import extra_names_from_request from datasette.plugins import pm from datasette.database import QueryInterrupted from datasette.events import ( @@ -850,11 +851,7 @@ class TableDropView(BaseView): def _get_extras(request): - extra_bits = request.args.getlist("_extra") - extras = set() - for bit in extra_bits: - extras.update(bit.split(",")) - return extras + return extra_names_from_request(request) async def _columns_to_select(table_columns, pks, request): diff --git a/tests/test_table_api.py b/tests/test_table_api.py index ceeb646d..eeb3dc8b 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1376,6 +1376,17 @@ async def test_table_extras(ds_client, extra, expected_json): assert response.json() == expected_json +@pytest.mark.asyncio +async def test_table_extra_columns_can_be_comma_separated(ds_client): + response = await ds_client.get( + "/fixtures/primary_key_multiple_columns.json?_extra=columns,count" + ) + assert response.status_code == 200 + data = response.json() + assert data["columns"] == ["id", "content", "content2"] + assert data["count"] == 1 + + @pytest.mark.asyncio async def test_extra_render_cell(): """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook""" From 17bbe6855c34630c14b077e08247d453d371cdea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:52:10 -0700 Subject: [PATCH 1080/1116] Refactor table JSON extras into classes --- datasette/extras.py | 94 ++++ datasette/views/table.py | 584 ++----------------------- datasette/views/table_extras.py | 746 ++++++++++++++++++++++++++++++++ 3 files changed, 884 insertions(+), 540 deletions(-) create mode 100644 datasette/views/table_extras.py diff --git a/datasette/extras.py b/datasette/extras.py index 01a9fb4b..786ec4f4 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,6 +1,100 @@ +import re +from enum import Enum +from typing import ClassVar + +from asyncinject import Registry + + def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") extras = set() for bit in extra_bits: extras.update(part for part in bit.split(",") if part) return extras + + +class ExtraScope(Enum): + TABLE = "table" + + +class Provider: + name: ClassVar[str | None] = None + scopes: ClassVar[frozenset[ExtraScope]] = frozenset() + public: ClassVar[bool] = False + + @classmethod + def key(cls): + return cls.name or _camel_to_snake(cls.__name__) + + @classmethod + def available_for(cls, scope): + return scope in cls.scopes + + async def resolve(self, context): + raise NotImplementedError + + +class Extra(Provider): + description: ClassVar[str | None] = None + public: ClassVar[bool] = True + stable: ClassVar[bool] = True + expensive: ClassVar[bool] = False + docs_note: ClassVar[str | None] = None + + @classmethod + def documentation(cls): + return { + "name": cls.key(), + "description": cls.description, + "scopes": [ + scope.value for scope in sorted(cls.scopes, key=lambda s: s.value) + ], + "stable": cls.stable, + "expensive": cls.expensive, + "docs_note": cls.docs_note, + } + + +class ExtraRegistry: + def __init__(self, classes): + self.classes = list(classes) + self.classes_by_name = {cls.key(): cls for cls in self.classes} + + def classes_for_scope(self, scope, include_internal=True): + classes = [ + cls + for cls in self.classes + if cls.available_for(scope) and (include_internal or cls.public) + ] + return classes + + def public_classes_for_scope(self, scope): + return self.classes_for_scope(scope, include_internal=False) + + async def resolve(self, requested, context, scope): + registry = Registry() + + async def context_provider(): + return context + + registry.register(context_provider, name="context") + + for cls in self.classes_for_scope(scope): + registry.register(cls().resolve, name=cls.key()) + + public_names = {cls.key() for cls in self.public_classes_for_scope(scope)} + requested_public_names = [ + name + for name in requested + if name in public_names and name in registry._registry + ] + resolved = await registry.resolve_multi(requested_public_names) + return { + name: resolved[name] for name in requested_public_names if name in resolved + } + + +def _camel_to_snake(name): + name = re.sub(r"(Extra|Provider)$", "", name) + name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() diff --git a/datasette/views/table.py b/datasette/views/table.py index 9ba249f4..c2d520f8 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -3,12 +3,10 @@ import itertools import json import urllib -from asyncinject import Registry import markupsafe from datasette.extras import extra_names_from_request from datasette.plugins import pm -from datasette.database import QueryInterrupted from datasette.events import ( AlterTableEvent, DropTableEvent, @@ -47,6 +45,12 @@ from datasette.filters import Filters import sqlite_utils from .base import BaseView, DatasetteError, _error, stream_csv from .database import QueryView +from .table_extras import ( + TABLE_EXTRA_BUNDLES, + TableExtraContext, + resolve_table_extras, + table_extra_registry, +) LINK_WITH_LABEL = ( '{label} {id}' @@ -1465,560 +1469,60 @@ async def table_view_data( if extra_extras: extras.update(extra_extras) - async def extra_count_sql(): - return count_sql - - async def extra_count(): - "Total count of rows matching these filters" - # Calculate the total count for this query - count = None - if ( - not db.is_mutable - and datasette.inspect_data - and count_sql == f"select count(*) from {table_name} " - ): - # We can use a previously cached table row count - try: - count = datasette.inspect_data[database_name]["tables"][table_name][ - "count" - ] - except KeyError: - pass - - # Otherwise run a select count(*) ... - if count_sql and count is None and not nocount: - count_sql_limited = ( - f"select count(*) from (select * {from_sql} limit 10001)" - ) - try: - count_rows = list(await db.execute(count_sql_limited, from_sql_params)) - count = count_rows[0][0] - except QueryInterrupted: - pass - return count - - async def facet_instances(extra_count): - facet_instances = [] - facet_classes = list( - itertools.chain.from_iterable(pm.hook.register_facet_classes()) - ) - for facet_class in facet_classes: - facet_instances.append( - facet_class( - datasette, - request, - database_name, - sql=sql_no_order_no_limit, - params=params, - table=table_name, - table_config=table_metadata, - row_count=extra_count, - ) - ) - return facet_instances - - async def extra_facet_results(facet_instances): - "Results of facets calculated against this data" - facet_results = {} - facets_timed_out = [] - - if not nofacet: - # Run them in parallel - facet_awaitables = [facet.facet_results() for facet in facet_instances] - facet_awaitable_results = await run_sequential(*facet_awaitables) - for ( - instance_facet_results, - instance_facets_timed_out, - ) in facet_awaitable_results: - for facet_info in instance_facet_results: - base_key = facet_info["name"] - key = base_key - i = 1 - while key in facet_results: - i += 1 - key = f"{base_key}_{i}" - facet_results[key] = facet_info - facets_timed_out.extend(instance_facets_timed_out) - - return { - "results": facet_results, - "timed_out": facets_timed_out, - } - - async def extra_suggested_facets(facet_instances): - "Suggestions for facets that might return interesting results" - suggested_facets = [] - # Calculate suggested facets - if ( - datasette.setting("suggest_facets") - and datasette.setting("allow_facet") - and not _next - and not nofacet - and not nosuggest - ): - # Run them in parallel - facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] - for suggest_result in await run_sequential(*facet_suggest_awaitables): - suggested_facets.extend(suggest_result) - return suggested_facets - # Faceting if not datasette.setting("allow_facet") and any( arg.startswith("_facet") for arg in request.args ): raise BadRequest("_facet= is not allowed") - # human_description_en combines filters AND search, if provided - async def extra_human_description_en(): - "Human-readable description of the filters" - human_description_en = filters.human_description_en( - extra=extra_human_descriptions - ) - if sort or sort_desc: - human_description_en = " ".join( - [b for b in [human_description_en, sorted_by] if b] - ) - return human_description_en - - if sort or sort_desc: - sorted_by = "sorted by {}{}".format( - (sort or sort_desc), " descending" if sort_desc else "" - ) - - async def extra_next_url(): - "Full URL for the next page of results" - return next_url - - async def extra_columns(): - "Column names returned by this query" - return columns - - async def extra_all_columns(): - "All columns in the table, regardless of _col/_nocol filtering" - return list(table_columns) - - async def extra_primary_keys(): - "Primary keys for this table" - return pks - - async def extra_actions(): - async def actions(): - links = [] - kwargs = { - "datasette": datasette, - "database": database_name, - "actor": request.actor, - "request": request, - } - if is_view: - kwargs["view"] = table_name - method = pm.hook.view_actions - else: - kwargs["table"] = table_name - method = pm.hook.table_actions - for hook in method(**kwargs): - extra_links = await await_me_maybe(hook) - if extra_links: - links.extend(extra_links) - return links - - return actions - - async def extra_is_view(): - return is_view - - async def extra_debug(): - "Extra debug information" - return { - "resolved": repr(resolved), - "url_vars": request.url_vars, - "nofacet": nofacet, - "nosuggest": nosuggest, - } - - async def extra_request(): - "Full information about the request" - return { - "url": request.url, - "path": request.path, - "full_path": request.full_path, - "host": request.host, - "args": request.args._data, - } - - async def run_display_columns_and_rows(): - display_columns, display_rows = await display_columns_and_rows( - datasette, - database_name, - table_name, - results.description, - rows, - link_column=not is_view, - truncate_cells=datasette.setting("truncate_cells_html"), - sortable_columns=sortable_columns, - request=request, - ) - return { - "columns": display_columns, - "rows": display_rows, - } - - async def extra_display_columns(run_display_columns_and_rows): - return run_display_columns_and_rows["columns"] - - async def extra_display_rows(run_display_columns_and_rows): - return run_display_columns_and_rows["rows"] - - async def extra_render_cell(): - "Rendered HTML for each cell using the render_cell plugin hook" - pks_for_display = pks if pks else (["rowid"] if not is_view else []) - col_names = [col[0] for col in results.description] - ct_map = await datasette.get_column_types(database_name, table_name) - rendered_rows = [] - for row in rows: - rendered_row = {} - for value, column in zip(row, col_names): - ct = ct_map.get(column) - plugin_display_value = None - # Try column type render_cell first - if ct: - candidate = await ct.render_cell( - value=value, - column=column, - table=table_name, - database=database_name, - datasette=datasette, - request=request, - ) - if candidate is not None: - plugin_display_value = candidate - if plugin_display_value is None: - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=table_name, - pks=pks_for_display, - database=database_name, - datasette=datasette, - request=request, - column_type=ct, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - rendered_row[column] = str(plugin_display_value) - rendered_rows.append(rendered_row) - return rendered_rows - - async def extra_query(): - "Details of the underlying SQL query" - return { - "sql": sql, - "params": params, - } - - async def extra_column_types(): - "Column type assignments for this table" - ct_map = await datasette.get_column_types(database_name, table_name) - return { - col_name: { - "type": ct.name, - "config": ct.config, - } - for col_name, ct in ct_map.items() - } - - async def extra_set_column_type_ui(): - "Column type UI metadata for this table" - if is_view: - return None - - if not await datasette.allowed( - action="set-column-type", - resource=TableResource(database=database_name, table=table_name), - actor=request.actor, - ): - return None - - column_details = await datasette._get_resource_column_details( - database_name, table_name - ) - ct_map = await datasette.get_column_types(database_name, table_name) - columns = {} - for column_name, column_detail in column_details.items(): - current = ct_map.get(column_name) - columns[column_name] = { - "current": ( - {"type": current.name, "config": current.config} - if current is not None - else None - ), - "options": [ - { - "name": name, - "description": ct_cls.description, - } - for name, ct_cls in sorted(datasette._column_types.items()) - if datasette._column_type_is_applicable(ct_cls, column_detail) - ], - } - return { - "path": "{}/-/set-column-type".format( - datasette.urls.table(database_name, table_name) - ), - "columns": columns, - } - - async def extra_metadata(): - "Metadata about the table and database" - tablemetadata = await datasette.get_resource_metadata(database_name, table_name) - - rows = await datasette.get_internal_database().execute( - """ - SELECT - column_name, - value - FROM metadata_columns - WHERE database_name = ? - AND resource_name = ? - AND key = 'description' - """, - [database_name, table_name], - ) - tablemetadata["columns"] = dict(rows) - return tablemetadata - - async def extra_database(): - return database_name - - async def extra_table(): - return table_name - - async def extra_database_color(): - return db.color - - async def extra_form_hidden_args(): - form_hidden_args = [] - for key in request.args: - if ( - key.startswith("_") - and key not in ("_sort", "_sort_desc", "_search", "_next") - and "__" not in key - ): - for value in request.args.getlist(key): - form_hidden_args.append((key, value)) - return form_hidden_args - - async def extra_filters(): - return filters - - async def extra_custom_table_templates(): - return [ - f"_table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", - f"_table-table-{to_css_class(database_name)}-{to_css_class(table_name)}.html", - "_table.html", - ] - - async def extra_sorted_facet_results(extra_facet_results): - facet_configs = table_metadata.get("facets", []) - if facet_configs: - # Build ordered list of facet names from metadata config - metadata_facet_names = [] - for fc in facet_configs: - if isinstance(fc, str): - metadata_facet_names.append(fc) - elif isinstance(fc, dict): - metadata_facet_names.append(list(fc.values())[0]) - metadata_order = {name: i for i, name in enumerate(metadata_facet_names)} - metadata_facets = [] - request_facets = [] - for f in extra_facet_results["results"].values(): - if f["name"] in metadata_order: - metadata_facets.append(f) - else: - request_facets.append(f) - metadata_facets.sort(key=lambda f: metadata_order[f["name"]]) - request_facets.sort( - key=lambda f: (len(f["results"]), f["name"]), - reverse=True, - ) - return metadata_facets + request_facets - else: - return sorted( - extra_facet_results["results"].values(), - key=lambda f: (len(f["results"]), f["name"]), - reverse=True, - ) - - async def extra_table_definition(): - return await db.get_table_definition(table_name) - - async def extra_view_definition(): - return await db.get_view_definition(table_name) - - async def extra_renderers(extra_expandable_columns, extra_query): - renderers = {} - url_labels_extra = {} - if extra_expandable_columns: - url_labels_extra = {"_labels": "on"} - for key, (_, can_render) in datasette.renderers.items(): - it_can_render = call_with_supported_arguments( - can_render, - datasette=datasette, - columns=columns or [], - rows=rows or [], - sql=extra_query.get("sql", None), - query_name=None, - database=database_name, - table=table_name, - request=request, - view_name="table", - ) - it_can_render = await await_me_maybe(it_can_render) - if it_can_render: - renderers[key] = datasette.urls.path( - path_with_format( - request=request, - path=request.scope.get("route_path"), - format=key, - extra_qs={**url_labels_extra}, - ) - ) - return renderers - - async def extra_private(): - return private - - async def extra_expandable_columns(): - expandables = [] - db = datasette.databases[database_name] - for fk in await db.foreign_keys_for_table(table_name): - label_column = await db.label_column_for_table(fk["other_table"]) - expandables.append((fk, label_column)) - return expandables - - async def extra_extras(): - "Available ?_extra= blocks" - all_extras = [ - (key[len("extra_") :], fn.__doc__) - for key, fn in registry._registry.items() - if key.startswith("extra_") - ] - return [ - { - "name": name, - "description": doc, - "toggle_url": datasette.absolute_url( - request, - datasette.urls.path( - path_with_added_args(request, {"_extra": name}) - if name not in extras - else path_with_removed_args(request, {"_extra": name}) - ), - ), - "selected": name in extras, - } - for name, doc in all_extras - ] - - async def extra_facets_timed_out(extra_facet_results): - return extra_facet_results["timed_out"] - - bundles = { - "html": [ - "suggested_facets", - "facet_results", - "facets_timed_out", - "count", - "count_sql", - "human_description_en", - "next_url", - "metadata", - "query", - "columns", - "display_columns", - "display_rows", - "database", - "table", - "database_color", - "actions", - "filters", - "renderers", - "custom_table_templates", - "sorted_facet_results", - "table_definition", - "view_definition", - "is_view", - "private", - "primary_keys", - "all_columns", - "expandable_columns", - "form_hidden_args", - "set_column_type_ui", - ] - } - - for key, values in bundles.items(): + for key, values in TABLE_EXTRA_BUNDLES.items(): if f"_{key}" in extras: extras.update(values) extras.discard(f"_{key}") - registry = Registry( - extra_count, - extra_count_sql, - extra_facet_results, - extra_facets_timed_out, - extra_suggested_facets, - facet_instances, - extra_human_description_en, - extra_next_url, - extra_columns, - extra_all_columns, - extra_primary_keys, - run_display_columns_and_rows, - extra_display_columns, - extra_display_rows, - extra_render_cell, - extra_debug, - extra_request, - extra_query, - extra_column_types, - extra_set_column_type_ui, - extra_metadata, - extra_extras, - extra_database, - extra_table, - extra_database_color, - extra_actions, - extra_filters, - extra_renderers, - extra_custom_table_templates, - extra_sorted_facet_results, - extra_table_definition, - extra_view_definition, - extra_is_view, - extra_private, - extra_expandable_columns, - extra_form_hidden_args, + table_extra_context = TableExtraContext( + datasette=datasette, + request=request, + resolved=resolved, + db=db, + database_name=database_name, + table_name=table_name, + is_view=is_view, + private=private, + rows=rows, + columns=columns, + results_description=results.description, + table_columns=table_columns, + pks=pks, + count_sql=count_sql, + from_sql=from_sql, + from_sql_params=from_sql_params, + nocount=nocount, + nofacet=nofacet, + nosuggest=nosuggest, + next_arg=request.args.get("_next"), + next_value=next_value, + next_url=next_url, + sql=sql, + sql_no_order_no_limit=sql_no_order_no_limit, + params=params, + table_metadata=table_metadata, + filters=filters, + extra_human_descriptions=extra_human_descriptions, + sort=sort, + sort_desc=sort_desc, + sortable_columns=sortable_columns, + extras=extras, + extra_registry=table_extra_registry, + display_columns_and_rows=display_columns_and_rows, + run_sequential=run_sequential, ) - results = await registry.resolve_multi( - ["extra_{}".format(extra) for extra in extras] - ) data = { "ok": True, "next": next_value and str(next_value) or None, } - data.update( - { - key.replace("extra_", ""): value - for key, value in results.items() - if key.startswith("extra_") and key.replace("extra_", "") in extras - } - ) + data.update(await resolve_table_extras(extras, table_extra_context)) raw_sqlite_rows = rows[:page_size] # Apply transform_value for columns with assigned types ct_map = await datasette.get_column_types(database_name, table_name) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py new file mode 100644 index 00000000..2ec2adf0 --- /dev/null +++ b/datasette/views/table_extras.py @@ -0,0 +1,746 @@ +import itertools +from dataclasses import dataclass + +from datasette.database import QueryInterrupted +from datasette.extras import Extra, ExtraRegistry, ExtraScope, Provider +from datasette.plugins import pm +from datasette.resources import TableResource +from datasette.utils import ( + await_me_maybe, + call_with_supported_arguments, + path_with_added_args, + path_with_format, + path_with_removed_args, + to_css_class, +) + + +@dataclass(frozen=True) +class TableExtraContext: + datasette: object + request: object + resolved: object + db: object + database_name: str + table_name: str + is_view: bool + private: bool + rows: list + columns: list + results_description: list + table_columns: list + pks: list + count_sql: str + from_sql: str + from_sql_params: dict + nocount: object + nofacet: object + nosuggest: object + next_arg: object + next_value: object + next_url: str | None + sql: str + sql_no_order_no_limit: str + params: dict + table_metadata: dict + filters: object + extra_human_descriptions: list + sort: str | None + sort_desc: str | None + sortable_columns: set + extras: set + extra_registry: ExtraRegistry + display_columns_and_rows: object + run_sequential: object + + +class CountSqlExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.count_sql + + +class CountExtra(Extra): + description = "Total count of rows matching these filters" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context): + count = None + if ( + not context.db.is_mutable + and context.datasette.inspect_data + and context.count_sql == f"select count(*) from {context.table_name} " + ): + try: + count = context.datasette.inspect_data[context.database_name]["tables"][ + context.table_name + ]["count"] + except KeyError: + pass + + if context.count_sql and count is None and not context.nocount: + count_sql_limited = ( + f"select count(*) from (select * {context.from_sql} limit 10001)" + ) + try: + count_rows = list( + await context.db.execute(count_sql_limited, context.from_sql_params) + ) + count = count_rows[0][0] + except QueryInterrupted: + pass + return count + + +class FacetInstancesProvider(Provider): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, count): + facet_instances = [] + facet_classes = list( + itertools.chain.from_iterable(pm.hook.register_facet_classes()) + ) + for facet_class in facet_classes: + facet_instances.append( + facet_class( + context.datasette, + context.request, + context.database_name, + sql=context.sql_no_order_no_limit, + params=context.params, + table=context.table_name, + table_config=context.table_metadata, + row_count=count, + ) + ) + return facet_instances + + +class FacetResultsExtra(Extra): + description = "Results of facets calculated against this data" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context, facet_instances): + facet_results = {} + facets_timed_out = [] + + if not context.nofacet: + facet_awaitables = [facet.facet_results() for facet in facet_instances] + facet_awaitable_results = await context.run_sequential(*facet_awaitables) + for ( + instance_facet_results, + instance_facets_timed_out, + ) in facet_awaitable_results: + for facet_info in instance_facet_results: + base_key = facet_info["name"] + key = base_key + i = 1 + while key in facet_results: + i += 1 + key = f"{base_key}_{i}" + facet_results[key] = facet_info + facets_timed_out.extend(instance_facets_timed_out) + + return { + "results": facet_results, + "timed_out": facets_timed_out, + } + + +class FacetsTimedOutExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, facet_results): + return facet_results["timed_out"] + + +class SuggestedFacetsExtra(Extra): + description = "Suggestions for facets that might return interesting results" + scopes = frozenset({ExtraScope.TABLE}) + expensive = True + + async def resolve(self, context, facet_instances): + suggested_facets = [] + if ( + context.datasette.setting("suggest_facets") + and context.datasette.setting("allow_facet") + and not context.next_arg + and not context.nofacet + and not context.nosuggest + ): + facet_suggest_awaitables = [facet.suggest() for facet in facet_instances] + for suggest_result in await context.run_sequential( + *facet_suggest_awaitables + ): + suggested_facets.extend(suggest_result) + return suggested_facets + + +class HumanDescriptionEnExtra(Extra): + description = "Human-readable description of the filters" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + human_description_en = context.filters.human_description_en( + extra=context.extra_human_descriptions + ) + if context.sort or context.sort_desc: + sorted_by = "sorted by {}{}".format( + (context.sort or context.sort_desc), + " descending" if context.sort_desc else "", + ) + human_description_en = " ".join( + [b for b in [human_description_en, sorted_by] if b] + ) + return human_description_en + + +class NextUrlExtra(Extra): + description = "Full URL for the next page of results" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.next_url + + +class ColumnsExtra(Extra): + description = "Column names returned by this query" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.columns + + +class AllColumnsExtra(Extra): + description = "All columns in the table, regardless of _col/_nocol filtering" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return list(context.table_columns) + + +class PrimaryKeysExtra(Extra): + description = "Primary keys for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.pks + + +class ActionsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + async def actions(): + links = [] + kwargs = { + "datasette": context.datasette, + "database": context.database_name, + "actor": context.request.actor, + "request": context.request, + } + if context.is_view: + kwargs["view"] = context.table_name + method = pm.hook.view_actions + else: + kwargs["table"] = context.table_name + method = pm.hook.table_actions + for hook in method(**kwargs): + extra_links = await await_me_maybe(hook) + if extra_links: + links.extend(extra_links) + return links + + return actions + + +class IsViewExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.is_view + + +class DebugExtra(Extra): + description = "Extra debug information" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "resolved": repr(context.resolved), + "url_vars": context.request.url_vars, + "nofacet": context.nofacet, + "nosuggest": context.nosuggest, + } + + +class RequestExtra(Extra): + description = "Full information about the request" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "url": context.request.url, + "path": context.request.path, + "full_path": context.request.full_path, + "host": context.request.host, + "args": context.request.args._data, + } + + +class DisplayColumnsAndRowsProvider(Provider): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + display_columns, display_rows = await context.display_columns_and_rows( + context.datasette, + context.database_name, + context.table_name, + context.results_description, + context.rows, + link_column=not context.is_view, + truncate_cells=context.datasette.setting("truncate_cells_html"), + sortable_columns=context.sortable_columns, + request=context.request, + ) + return { + "columns": display_columns, + "rows": display_rows, + } + + +class DisplayColumnsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, display_columns_and_rows): + return display_columns_and_rows["columns"] + + +class DisplayRowsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, display_columns_and_rows): + return display_columns_and_rows["rows"] + + +class RenderCellExtra(Extra): + description = "Rendered HTML for each cell using the render_cell plugin hook" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + pks_for_display = ( + context.pks if context.pks else (["rowid"] if not context.is_view else []) + ) + col_names = [col[0] for col in context.results_description] + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + rendered_rows = [] + for row in context.rows: + rendered_row = {} + for value, column in zip(row, col_names): + ct = ct_map.get(column) + plugin_display_value = None + if ct: + candidate = await ct.render_cell( + value=value, + column=column, + table=context.table_name, + database=context.database_name, + datasette=context.datasette, + request=context.request, + ) + if candidate is not None: + plugin_display_value = candidate + if plugin_display_value is None: + for candidate in pm.hook.render_cell( + row=row, + value=value, + column=column, + table=context.table_name, + pks=pks_for_display, + database=context.database_name, + datasette=context.datasette, + request=context.request, + column_type=ct, + ): + candidate = await await_me_maybe(candidate) + if candidate is not None: + plugin_display_value = candidate + break + if plugin_display_value: + rendered_row[column] = str(plugin_display_value) + rendered_rows.append(rendered_row) + return rendered_rows + + +class QueryExtra(Extra): + description = "Details of the underlying SQL query" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return { + "sql": context.sql, + "params": context.params, + } + + +class ColumnTypesExtra(Extra): + description = "Column type assignments for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + return { + col_name: { + "type": ct.name, + "config": ct.config, + } + for col_name, ct in ct_map.items() + } + + +class SetColumnTypeUiExtra(Extra): + description = "Column type UI metadata for this table" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + if context.is_view: + return None + + if not await context.datasette.allowed( + action="set-column-type", + resource=TableResource( + database=context.database_name, table=context.table_name + ), + actor=context.request.actor, + ): + return None + + column_details = await context.datasette._get_resource_column_details( + context.database_name, context.table_name + ) + ct_map = await context.datasette.get_column_types( + context.database_name, context.table_name + ) + columns = {} + for column_name, column_detail in column_details.items(): + current = ct_map.get(column_name) + columns[column_name] = { + "current": ( + {"type": current.name, "config": current.config} + if current is not None + else None + ), + "options": [ + { + "name": name, + "description": ct_cls.description, + } + for name, ct_cls in sorted(context.datasette._column_types.items()) + if context.datasette._column_type_is_applicable( + ct_cls, column_detail + ) + ], + } + return { + "path": "{}/-/set-column-type".format( + context.datasette.urls.table(context.database_name, context.table_name) + ), + "columns": columns, + } + + +class MetadataExtra(Extra): + description = "Metadata about the table and database" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + tablemetadata = await context.datasette.get_resource_metadata( + context.database_name, context.table_name + ) + + rows = await context.datasette.get_internal_database().execute( + """ + SELECT + column_name, + value + FROM metadata_columns + WHERE database_name = ? + AND resource_name = ? + AND key = 'description' + """, + [context.database_name, context.table_name], + ) + tablemetadata["columns"] = dict(rows) + return tablemetadata + + +class DatabaseExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.database_name + + +class TableExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.table_name + + +class DatabaseColorExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.db.color + + +class FormHiddenArgsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + form_hidden_args = [] + for key in context.request.args: + if ( + key.startswith("_") + and key not in ("_sort", "_sort_desc", "_search", "_next") + and "__" not in key + ): + for value in context.request.args.getlist(key): + form_hidden_args.append((key, value)) + return form_hidden_args + + +class FiltersExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.filters + + +class CustomTableTemplatesExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return [ + f"_table-{to_css_class(context.database_name)}-{to_css_class(context.table_name)}.html", + f"_table-table-{to_css_class(context.database_name)}-{to_css_class(context.table_name)}.html", + "_table.html", + ] + + +class SortedFacetResultsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, facet_results): + facet_configs = context.table_metadata.get("facets", []) + if facet_configs: + metadata_facet_names = [] + for fc in facet_configs: + if isinstance(fc, str): + metadata_facet_names.append(fc) + elif isinstance(fc, dict): + metadata_facet_names.append(list(fc.values())[0]) + metadata_order = {name: i for i, name in enumerate(metadata_facet_names)} + metadata_facets = [] + request_facets = [] + for f in facet_results["results"].values(): + if f["name"] in metadata_order: + metadata_facets.append(f) + else: + request_facets.append(f) + metadata_facets.sort(key=lambda f: metadata_order[f["name"]]) + request_facets.sort( + key=lambda f: (len(f["results"]), f["name"]), + reverse=True, + ) + return metadata_facets + request_facets + else: + return sorted( + facet_results["results"].values(), + key=lambda f: (len(f["results"]), f["name"]), + reverse=True, + ) + + +class TableDefinitionExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return await context.db.get_table_definition(context.table_name) + + +class ViewDefinitionExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return await context.db.get_view_definition(context.table_name) + + +class RenderersExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context, expandable_columns, query): + renderers = {} + url_labels_extra = {} + if expandable_columns: + url_labels_extra = {"_labels": "on"} + for key, (_, can_render) in context.datasette.renderers.items(): + it_can_render = call_with_supported_arguments( + can_render, + datasette=context.datasette, + columns=context.columns or [], + rows=context.rows or [], + sql=query.get("sql", None), + query_name=None, + database=context.database_name, + table=context.table_name, + request=context.request, + view_name="table", + ) + it_can_render = await await_me_maybe(it_can_render) + if it_can_render: + renderers[key] = context.datasette.urls.path( + path_with_format( + request=context.request, + path=context.request.scope.get("route_path"), + format=key, + extra_qs={**url_labels_extra}, + ) + ) + return renderers + + +class PrivateExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + return context.private + + +class ExpandableColumnsExtra(Extra): + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + expandables = [] + db = context.datasette.databases[context.database_name] + for fk in await db.foreign_keys_for_table(context.table_name): + label_column = await db.label_column_for_table(fk["other_table"]) + expandables.append((fk, label_column)) + return expandables + + +class ExtrasExtra(Extra): + description = "Available ?_extra= blocks" + scopes = frozenset({ExtraScope.TABLE}) + + async def resolve(self, context): + all_extras = [ + (cls.key(), cls.description) + for cls in context.extra_registry.public_classes_for_scope(ExtraScope.TABLE) + ] + return [ + { + "name": name, + "description": description, + "toggle_url": context.datasette.absolute_url( + context.request, + context.datasette.urls.path( + path_with_added_args(context.request, {"_extra": name}) + if name not in context.extras + else path_with_removed_args(context.request, {"_extra": name}) + ), + ), + "selected": name in context.extras, + } + for name, description in all_extras + ] + + +TABLE_EXTRA_BUNDLES = { + "html": [ + "suggested_facets", + "facet_results", + "facets_timed_out", + "count", + "count_sql", + "human_description_en", + "next_url", + "metadata", + "query", + "columns", + "display_columns", + "display_rows", + "database", + "table", + "database_color", + "actions", + "filters", + "renderers", + "custom_table_templates", + "sorted_facet_results", + "table_definition", + "view_definition", + "is_view", + "private", + "primary_keys", + "all_columns", + "expandable_columns", + "form_hidden_args", + "set_column_type_ui", + ] +} + + +TABLE_EXTRA_CLASSES = [ + CountExtra, + CountSqlExtra, + FacetResultsExtra, + FacetsTimedOutExtra, + SuggestedFacetsExtra, + FacetInstancesProvider, + HumanDescriptionEnExtra, + NextUrlExtra, + ColumnsExtra, + AllColumnsExtra, + PrimaryKeysExtra, + DisplayColumnsAndRowsProvider, + DisplayColumnsExtra, + DisplayRowsExtra, + RenderCellExtra, + DebugExtra, + RequestExtra, + QueryExtra, + ColumnTypesExtra, + SetColumnTypeUiExtra, + MetadataExtra, + ExtrasExtra, + DatabaseExtra, + TableExtra, + DatabaseColorExtra, + ActionsExtra, + FiltersExtra, + RenderersExtra, + CustomTableTemplatesExtra, + SortedFacetResultsExtra, + TableDefinitionExtra, + ViewDefinitionExtra, + IsViewExtra, + PrivateExtra, + ExpandableColumnsExtra, + FormHiddenArgsExtra, +] + + +table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) + + +async def resolve_table_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) From 111eeaf3702cd5ee417532beb80c746d49d92a11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 20:56:00 -0700 Subject: [PATCH 1081/1116] Document table JSON extras from metadata --- datasette/views/table_extras.py | 18 +++++++ docs/json_api.rst | 95 +++++++++++++++++++++++++++++++++ docs/json_api_doc.py | 20 +++++++ 3 files changed, 133 insertions(+) create mode 100644 docs/json_api_doc.py diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 2ec2adf0..e71c15d6 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -55,6 +55,7 @@ class TableExtraContext: class CountSqlExtra(Extra): + description = "SQL query used to calculate the total count" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -151,6 +152,7 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): + description = "Facet calculations that timed out" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -231,6 +233,7 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): + description = "Table or view actions made available by plugin hooks" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -258,6 +261,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): + description = "Whether this resource is a view instead of a table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -313,6 +317,7 @@ class DisplayColumnsAndRowsProvider(Provider): class DisplayColumnsExtra(Extra): + description = "Column metadata used by the HTML table display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -320,6 +325,7 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): + description = "Row data formatted for the HTML table display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -482,6 +488,7 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): + description = "Database name" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -489,6 +496,7 @@ class DatabaseExtra(Extra): class TableExtra(Extra): + description = "Table name" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -496,6 +504,7 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): + description = "Color assigned to the database" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -503,6 +512,7 @@ class DatabaseColorExtra(Extra): class FormHiddenArgsExtra(Extra): + description = "Hidden form arguments used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -519,6 +529,7 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): + description = "Filters object used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -526,6 +537,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): + description = "Custom template names considered for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -537,6 +549,7 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): + description = "Facet results sorted for display" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -571,6 +584,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): + description = "SQL definition for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -578,6 +592,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): + description = "SQL definition for this view" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -585,6 +600,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): + description = "Alternative output renderers available for this table" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, expandable_columns, query): @@ -619,6 +635,7 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): + description = "Whether this table is private to the current actor" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -626,6 +643,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): + description = "Foreign key columns that can be expanded with labels" scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 65031bf4..af60a527 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -235,6 +235,101 @@ query string arguments: Only available if the :ref:`setting_trace_debug` setting is enabled. +.. _json_api_extra: + +Expanding table JSON responses +------------------------------ + +Table JSON responses can be expanded with one or more ``?_extra=`` parameters. +These can be repeated or comma-separated: + +:: + + ?_extra=columns&_extra=count,next_url + +The available table extras are listed below. + +.. [[[cog + from json_api_doc import table_extras + table_extras(cog) +.. ]]] + +.. list-table:: + :header-rows: 1 + + * - Extra + - Description + * - ``count`` + - Total count of rows matching these filters (May execute additional queries.) + * - ``count_sql`` + - SQL query used to calculate the total count + * - ``facet_results`` + - Results of facets calculated against this data (May execute additional queries.) + * - ``facets_timed_out`` + - Facet calculations that timed out + * - ``suggested_facets`` + - Suggestions for facets that might return interesting results (May execute additional queries.) + * - ``human_description_en`` + - Human-readable description of the filters + * - ``next_url`` + - Full URL for the next page of results + * - ``columns`` + - Column names returned by this query + * - ``all_columns`` + - All columns in the table, regardless of _col/_nocol filtering + * - ``primary_keys`` + - Primary keys for this table + * - ``display_columns`` + - Column metadata used by the HTML table display + * - ``display_rows`` + - Row data formatted for the HTML table display + * - ``render_cell`` + - Rendered HTML for each cell using the render_cell plugin hook + * - ``debug`` + - Extra debug information + * - ``request`` + - Full information about the request + * - ``query`` + - Details of the underlying SQL query + * - ``column_types`` + - Column type assignments for this table + * - ``set_column_type_ui`` + - Column type UI metadata for this table + * - ``metadata`` + - Metadata about the table and database + * - ``extras`` + - Available ?_extra= blocks + * - ``database`` + - Database name + * - ``table`` + - Table name + * - ``database_color`` + - Color assigned to the database + * - ``actions`` + - Table or view actions made available by plugin hooks + * - ``filters`` + - Filters object used by the HTML table interface + * - ``renderers`` + - Alternative output renderers available for this table + * - ``custom_table_templates`` + - Custom template names considered for this table + * - ``sorted_facet_results`` + - Facet results sorted for display + * - ``table_definition`` + - SQL definition for this table + * - ``view_definition`` + - SQL definition for this view + * - ``is_view`` + - Whether this resource is a view instead of a table + * - ``private`` + - Whether this table is private to the current actor + * - ``expandable_columns`` + - Foreign key columns that can be expanded with labels + * - ``form_hidden_args`` + - Hidden form arguments used by the HTML table interface + +.. [[[end]]] + .. _table_arguments: Table arguments diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py new file mode 100644 index 00000000..f07c3ba7 --- /dev/null +++ b/docs/json_api_doc.py @@ -0,0 +1,20 @@ +def table_extras(cog): + from datasette.extras import ExtraScope + from datasette.views.table_extras import table_extra_registry + + cog.out("\n.. list-table::\n") + cog.out(" :header-rows: 1\n\n") + cog.out(" * - Extra\n") + cog.out(" - Description\n") + for cls in table_extra_registry.public_classes_for_scope(ExtraScope.TABLE): + description = cls.description or "" + notes = [] + if cls.expensive: + notes.append("May execute additional queries.") + if cls.docs_note: + notes.append(cls.docs_note) + if notes: + description = "{} ({})".format(description, " ".join(notes)).strip() + cog.out(" * - ``{}``\n".format(cls.key())) + cog.out(" - {}\n".format(description)) + cog.out("\n") From 79c8aff31df16e514616a7778fad1386ac9b4b2c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:10:58 -0700 Subject: [PATCH 1082/1116] Add generated examples for table JSON extras --- datasette/extras.py | 11 + datasette/views/table_extras.py | 87 +++++- docs/json_api.rst | 470 +++++++++++++++++++++++++++----- docs/json_api_doc.py | 64 ++++- tests/test_docs.py | 10 + 5 files changed, 561 insertions(+), 81 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 786ec4f4..d867f26c 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,4 +1,5 @@ import re +from dataclasses import dataclass from enum import Enum from typing import ClassVar @@ -17,6 +18,14 @@ class ExtraScope(Enum): TABLE = "table" +@dataclass(frozen=True) +class ExtraExample: + path: str | None = None + key: str | None = None + value: object | None = None + note: str | None = None + + class Provider: name: ClassVar[str | None] = None scopes: ClassVar[frozenset[ExtraScope]] = frozenset() @@ -36,6 +45,7 @@ class Provider: class Extra(Provider): description: ClassVar[str | None] = None + example: ClassVar[ExtraExample | None] = None public: ClassVar[bool] = True stable: ClassVar[bool] = True expensive: ClassVar[bool] = False @@ -52,6 +62,7 @@ class Extra(Provider): "stable": cls.stable, "expensive": cls.expensive, "docs_note": cls.docs_note, + "example": cls.example, } diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index e71c15d6..0eefeaa9 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -2,7 +2,7 @@ import itertools from dataclasses import dataclass from datasette.database import QueryInterrupted -from datasette.extras import Extra, ExtraRegistry, ExtraScope, Provider +from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider from datasette.plugins import pm from datasette.resources import TableResource from datasette.utils import ( @@ -56,6 +56,7 @@ class TableExtraContext: class CountSqlExtra(Extra): description = "SQL query used to calculate the total count" + example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -64,6 +65,7 @@ class CountSqlExtra(Extra): class CountExtra(Extra): description = "Total count of rows matching these filters" + example = ExtraExample("/fixtures/facetable.json?_extra=count") scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -121,6 +123,22 @@ class FacetInstancesProvider(Provider): class FacetResultsExtra(Extra): description = "Results of facets calculated against this data" + example = ExtraExample( + value={ + "results": { + "state": { + "name": "state", + "type": "column", + "results": [ + {"value": "CA", "label": "CA", "count": 10}, + {"value": "MI", "label": "MI", "count": 4}, + ], + } + }, + "timed_out": [], + }, + note="Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.", + ) scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -153,6 +171,9 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): description = "Facet calculations that timed out" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -161,6 +182,15 @@ class FacetsTimedOutExtra(Extra): class SuggestedFacetsExtra(Extra): description = "Suggestions for facets that might return interesting results" + example = ExtraExample( + value=[ + { + "name": "state", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state", + } + ], + note="Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.", + ) scopes = frozenset({ExtraScope.TABLE}) expensive = True @@ -183,6 +213,9 @@ class SuggestedFacetsExtra(Extra): class HumanDescriptionEnExtra(Extra): description = "Human-readable description of the filters" + example = ExtraExample( + "/fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -202,6 +235,7 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" + example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -210,6 +244,7 @@ class NextUrlExtra(Extra): class ColumnsExtra(Extra): description = "Column names returned by this query" + example = ExtraExample("/fixtures/facetable.json?_extra=columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -218,6 +253,7 @@ class ColumnsExtra(Extra): class AllColumnsExtra(Extra): description = "All columns in the table, regardless of _col/_nocol filtering" + example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -226,6 +262,7 @@ class AllColumnsExtra(Extra): class PrimaryKeysExtra(Extra): description = "Primary keys for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -262,6 +299,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): description = "Whether this resource is a view instead of a table" + example = ExtraExample("/fixtures/simple_view.json?_extra=is_view") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -318,6 +356,28 @@ class DisplayColumnsAndRowsProvider(Provider): class DisplayColumnsExtra(Extra): description = "Column metadata used by the HTML table display" + example = ExtraExample( + value=[ + { + "name": "pk", + "sortable": True, + "is_pk": True, + "type": "INTEGER", + "notnull": 0, + }, + { + "name": "created", + "sortable": True, + "is_pk": False, + "type": "TEXT", + "notnull": 0, + "description": None, + "column_type": None, + "column_type_config": None, + }, + ], + note="Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.", + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, display_columns_and_rows): @@ -334,6 +394,13 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" + example = ExtraExample( + value=[ + {}, + {"content": "Custom rendered HTML"}, + ], + note="Only columns whose rendered value differs from the default are included.", + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -385,6 +452,7 @@ class RenderCellExtra(Extra): class QueryExtra(Extra): description = "Details of the underlying SQL query" + example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -396,6 +464,7 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" + example = ExtraExample(value={}) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -464,6 +533,7 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): description = "Metadata about the table and database" + example = ExtraExample("/fixtures/facetable.json?_extra=metadata") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -489,6 +559,7 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): description = "Database name" + example = ExtraExample("/fixtures/facetable.json?_extra=database") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -497,6 +568,7 @@ class DatabaseExtra(Extra): class TableExtra(Extra): description = "Table name" + example = ExtraExample("/fixtures/facetable.json?_extra=table") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -505,6 +577,7 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" + example = ExtraExample("/fixtures/facetable.json?_extra=database_color") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -513,6 +586,9 @@ class DatabaseColorExtra(Extra): class FormHiddenArgsExtra(Extra): description = "Hidden form arguments used by the HTML table interface" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -538,6 +614,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -550,6 +627,9 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): description = "Facet results sorted for display" + example = ExtraExample( + "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" + ) scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, facet_results): @@ -585,6 +665,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): description = "SQL definition for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=table_definition") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -593,6 +674,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): description = "SQL definition for this view" + example = ExtraExample("/fixtures/simple_view.json?_extra=view_definition") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -601,6 +683,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" + example = ExtraExample("/fixtures/facetable.json?_extra=renderers") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context, expandable_columns, query): @@ -636,6 +719,7 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): description = "Whether this table is private to the current actor" + example = ExtraExample("/fixtures/facetable.json?_extra=private") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -644,6 +728,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" + example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index af60a527..d12a388e 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -254,79 +254,405 @@ The available table extras are listed below. table_extras(cog) .. ]]] -.. list-table:: - :header-rows: 1 +``count`` + Total count of rows matching these filters (May execute additional queries.) - * - Extra - - Description - * - ``count`` - - Total count of rows matching these filters (May execute additional queries.) - * - ``count_sql`` - - SQL query used to calculate the total count - * - ``facet_results`` - - Results of facets calculated against this data (May execute additional queries.) - * - ``facets_timed_out`` - - Facet calculations that timed out - * - ``suggested_facets`` - - Suggestions for facets that might return interesting results (May execute additional queries.) - * - ``human_description_en`` - - Human-readable description of the filters - * - ``next_url`` - - Full URL for the next page of results - * - ``columns`` - - Column names returned by this query - * - ``all_columns`` - - All columns in the table, regardless of _col/_nocol filtering - * - ``primary_keys`` - - Primary keys for this table - * - ``display_columns`` - - Column metadata used by the HTML table display - * - ``display_rows`` - - Row data formatted for the HTML table display - * - ``render_cell`` - - Rendered HTML for each cell using the render_cell plugin hook - * - ``debug`` - - Extra debug information - * - ``request`` - - Full information about the request - * - ``query`` - - Details of the underlying SQL query - * - ``column_types`` - - Column type assignments for this table - * - ``set_column_type_ui`` - - Column type UI metadata for this table - * - ``metadata`` - - Metadata about the table and database - * - ``extras`` - - Available ?_extra= blocks - * - ``database`` - - Database name - * - ``table`` - - Table name - * - ``database_color`` - - Color assigned to the database - * - ``actions`` - - Table or view actions made available by plugin hooks - * - ``filters`` - - Filters object used by the HTML table interface - * - ``renderers`` - - Alternative output renderers available for this table - * - ``custom_table_templates`` - - Custom template names considered for this table - * - ``sorted_facet_results`` - - Facet results sorted for display - * - ``table_definition`` - - SQL definition for this table - * - ``view_definition`` - - SQL definition for this view - * - ``is_view`` - - Whether this resource is a view instead of a table - * - ``private`` - - Whether this table is private to the current actor - * - ``expandable_columns`` - - Foreign key columns that can be expanded with labels - * - ``form_hidden_args`` - - Hidden form arguments used by the HTML table interface + ``GET /fixtures/facetable.json?_extra=count`` + + .. code-block:: json + + 15 + +``count_sql`` + SQL query used to calculate the total count + + ``GET /fixtures/facetable.json?_size=0&_extra=count_sql`` + + .. code-block:: json + + "select count(*) from facetable " + +``facet_results`` + Results of facets calculated against this data (May execute additional queries.) + + Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results. + + .. code-block:: json + + { + "results": { + "state": { + "name": "state", + "type": "column", + "results": [ + { + "value": "CA", + "label": "CA", + "count": 10 + }, + { + "value": "MI", + "label": "MI", + "count": 4 + } + ] + } + }, + "timed_out": [] + } + +``facets_timed_out`` + Facet calculations that timed out + + ``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out`` + + .. code-block:: json + + [] + +``suggested_facets`` + Suggestions for facets that might return interesting results (May execute additional queries.) + + Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets. + + .. code-block:: json + + [ + { + "name": "state", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=suggested_facets&_facet=state" + } + ] + +``human_description_en`` + Human-readable description of the filters + + ``GET /fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en`` + + .. code-block:: json + + "where state = \"CA\" sorted by pk" + +``next_url`` + Full URL for the next page of results + + ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` + + .. code-block:: json + + "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" + +``columns`` + Column names returned by this query + + ``GET /fixtures/facetable.json?_extra=columns`` + + .. code-block:: json + + [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + "n" + ] + +``all_columns`` + All columns in the table, regardless of _col/_nocol filtering + + ``GET /fixtures/facetable.json?_col=pk&_extra=all_columns`` + + .. code-block:: json + + [ + "pk", + "created", + "planet_int", + "on_earth", + "state", + "_city_id", + "_neighborhood", + "tags", + "complex_array", + "distinct_some_null", + "n" + ] + +``primary_keys`` + Primary keys for this table + + ``GET /fixtures/facetable.json?_extra=primary_keys`` + + .. code-block:: json + + [ + "pk" + ] + +``display_columns`` + Column metadata used by the HTML table display + + Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns. + + .. code-block:: json + + [ + { + "name": "pk", + "sortable": true, + "is_pk": true, + "type": "INTEGER", + "notnull": 0 + }, + { + "name": "created", + "sortable": true, + "is_pk": false, + "type": "TEXT", + "notnull": 0, + "description": null, + "column_type": null, + "column_type_config": null + } + ] + +``display_rows`` + Row data formatted for the HTML table display + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + [ + {}, + { + "content": "Custom rendered HTML" + } + ] + +``debug`` + Extra debug information + +``request`` + Full information about the request + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/facetable.json?_size=1&_extra=query`` + + .. code-block:: json + + { + "sql": "select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 2", + "params": {} + } + +``column_types`` + Column type assignments for this table + + .. code-block:: json + + {} + +``set_column_type_ui`` + Column type UI metadata for this table + +``metadata`` + Metadata about the table and database + + ``GET /fixtures/facetable.json?_extra=metadata`` + + .. code-block:: json + + { + "columns": {} + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/facetable.json?_extra=database`` + + .. code-block:: json + + "fixtures" + +``table`` + Table name + + ``GET /fixtures/facetable.json?_extra=table`` + + .. code-block:: json + + "facetable" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/facetable.json?_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``actions`` + Table or view actions made available by plugin hooks + +``filters`` + Filters object used by the HTML table interface + +``renderers`` + Alternative output renderers available for this table + + ``GET /fixtures/facetable.json?_extra=renderers`` + + .. code-block:: json + + { + "json": "/fixtures/facetable.json?_extra=renderers&_format=json&_labels=on" + } + +``custom_table_templates`` + Custom template names considered for this table + + ``GET /fixtures/facetable.json?_extra=custom_table_templates`` + + .. code-block:: json + + [ + "_table-fixtures-facetable.html", + "_table-table-fixtures-facetable.html", + "_table.html" + ] + +``sorted_facet_results`` + Facet results sorted for display + + ``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results`` + + .. code-block:: json + + [ + { + "name": "state", + "type": "column", + "hideable": true, + "toggle_url": "/fixtures/facetable.json?_extra=sorted_facet_results", + "results": [ + { + "value": "CA", + "label": "CA", + "count": 10, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=CA", + "selected": false + }, + { + "value": "MI", + "label": "MI", + "count": 4, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MI", + "selected": false + }, + { + "value": "MC", + "label": "MC", + "count": 1, + "toggle_url": "http://localhost/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results&state=MC", + "selected": false + } + ], + "truncated": false + } + ] + +``table_definition`` + SQL definition for this table + + ``GET /fixtures/facetable.json?_extra=table_definition`` + + .. code-block:: json + + "CREATE TABLE facetable (\n pk integer primary key,\n created text,\n planet_int integer,\n on_earth integer,\n state text,\n _city_id integer,\n _neighborhood text,\n tags text,\n complex_array text,\n distinct_some_null,\n n text,\n FOREIGN KEY (\"_city_id\") REFERENCES [facet_cities](id)\n);" + +``view_definition`` + SQL definition for this view + + ``GET /fixtures/simple_view.json?_extra=view_definition`` + + .. code-block:: json + + "CREATE VIEW simple_view AS\n SELECT content, upper(content) AS upper_content FROM simple_primary_key;" + +``is_view`` + Whether this resource is a view instead of a table + + ``GET /fixtures/simple_view.json?_extra=is_view`` + + .. code-block:: json + + true + +``private`` + Whether this table is private to the current actor + + ``GET /fixtures/facetable.json?_extra=private`` + + .. code-block:: json + + false + +``expandable_columns`` + Foreign key columns that can be expanded with labels + + ``GET /fixtures/facetable.json?_extra=expandable_columns`` + + .. code-block:: json + + [ + [ + { + "column": "_city_id", + "other_table": "facet_cities", + "other_column": "id" + }, + "name" + ] + ] + +``form_hidden_args`` + Hidden form arguments used by the HTML table interface + + ``GET /fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args`` + + .. code-block:: json + + [ + [ + "_facet", + "state" + ], + [ + "_size", + "1" + ], + [ + "_extra", + "form_hidden_args" + ] + ] .. [[[end]]] diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index f07c3ba7..69ec6e5e 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -1,12 +1,20 @@ +import asyncio +import json +import pathlib +import tempfile +import textwrap + + def table_extras(cog): from datasette.extras import ExtraScope from datasette.views.table_extras import table_extra_registry - cog.out("\n.. list-table::\n") - cog.out(" :header-rows: 1\n\n") - cog.out(" * - Extra\n") - cog.out(" - Description\n") - for cls in table_extra_registry.public_classes_for_scope(ExtraScope.TABLE): + classes = table_extra_registry.public_classes_for_scope(ExtraScope.TABLE) + + live_examples = asyncio.run(_fetch_live_examples(classes)) + cog.out("\n") + for cls in classes: + example = cls.example description = cls.description or "" notes = [] if cls.expensive: @@ -15,6 +23,46 @@ def table_extras(cog): notes.append(cls.docs_note) if notes: description = "{} ({})".format(description, " ".join(notes)).strip() - cog.out(" * - ``{}``\n".format(cls.key())) - cog.out(" - {}\n".format(description)) - cog.out("\n") + + cog.out("``{}``\n".format(cls.key())) + cog.out(" {}\n\n".format(description)) + if example is None: + continue + + if example.path: + value = live_examples[(example.path, example.key or cls.key())] + cog.out(" ``GET {}``\n\n".format(example.path)) + else: + value = example.value + if example.note: + cog.out(" {}\n\n".format(example.note)) + cog.out(" .. code-block:: json\n\n") + cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) + cog.out("\n\n") + + +async def _fetch_live_examples(classes): + from datasette.app import Datasette + from datasette.fixtures import write_fixture_database + + examples = {} + with tempfile.TemporaryDirectory() as tmpdir: + db_path = pathlib.Path(tmpdir) / "fixtures.db" + write_fixture_database(db_path) + datasette = Datasette([str(db_path)], settings={"num_sql_threads": 1}) + try: + for cls in classes: + example = cls.example + if example is None or not example.path: + continue + key = example.key or cls.key() + response = await datasette.client.get(example.path) + assert response.status_code == 200, example.path + data = response.json() + assert key in data, "{} missing from {}".format(key, example.path) + examples[(example.path, key)] = data[key] + finally: + for db in datasette.databases.values(): + if not db.is_memory: + db.close() + return examples diff --git a/tests/test_docs.py b/tests/test_docs.py index 51caf595..784755e9 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -112,6 +112,16 @@ def test_table_filters_are_documented(documented_table_filters, subtests): assert f.key in documented_table_filters +def test_table_extra_examples_are_documented(): + from datasette.views.table_extras import CountExtra + + assert CountExtra.example.path == "/fixtures/facetable.json?_extra=count" + content = (docs_path / "json_api.rst").read_text() + section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + assert "GET /fixtures/facetable.json?_extra=count" in section + assert ".. code-block:: json" in section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 22f80b819625b9f6b5aa0661f58d97c89882a932 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:13:53 -0700 Subject: [PATCH 1083/1116] Clarify render_cell JSON extra example --- datasette/views/table_extras.py | 20 +++++++++++++++----- docs/json_api.rst | 26 +++++++++++++++++++------- tests/test_docs.py | 8 ++++++++ 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 0eefeaa9..b6e653c4 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -395,11 +395,21 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" example = ExtraExample( - value=[ - {}, - {"content": "Custom rendered HTML"}, - ], - note="Only columns whose rendered value differs from the default are included.", + value={ + "rows": [ + {"id": 1, "content": "hello"}, + {"id": 4, "content": "RENDER_CELL_DEMO"}, + ], + "render_cell": [ + {}, + {"content": "Custom rendered HTML"}, + ], + }, + note=( + "The ``render_cell`` array has one item per row, in the same order as " + "the ``rows`` array. Each object is keyed by column name. Only columns " + "whose rendered value differs from the default are included." + ), ) scopes = frozenset({ExtraScope.TABLE}) diff --git a/docs/json_api.rst b/docs/json_api.rst index d12a388e..24d59577 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -428,16 +428,28 @@ The available table extras are listed below. ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook - Only columns whose rendered value differs from the default are included. + The ``render_cell`` array has one item per row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. .. code-block:: json - [ - {}, - { - "content": "Custom rendered HTML" - } - ] + { + "rows": [ + { + "id": 1, + "content": "hello" + }, + { + "id": 4, + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + {}, + { + "content": "Custom rendered HTML" + } + ] + } ``debug`` Extra debug information diff --git a/tests/test_docs.py b/tests/test_docs.py index 784755e9..c4e0a849 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -122,6 +122,14 @@ def test_table_extra_examples_are_documented(): assert ".. code-block:: json" in section +def test_render_cell_extra_example_explains_row_and_column_mapping(): + content = (docs_path / "json_api.rst").read_text() + section = content.split("``render_cell``")[-1].split("``query``")[0] + assert "same order as the ``rows`` array" in section + assert '"rows": [' in section + assert '"render_cell": [' in section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 0fa872d43842d87af9e7b8c193f90addfcf164ba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 8 Jun 2026 21:20:06 -0700 Subject: [PATCH 1084/1116] Add debug and request JSON extra examples --- datasette/views/table_extras.py | 2 ++ docs/json_api.rst | 31 +++++++++++++++++++++++++++++++ tests/test_docs.py | 13 +++++++++++++ 3 files changed, 46 insertions(+) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index b6e653c4..e888ee9f 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -308,6 +308,7 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" + example = ExtraExample("/fixtures/facetable.json?_extra=debug") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): @@ -321,6 +322,7 @@ class DebugExtra(Extra): class RequestExtra(Extra): description = "Full information about the request" + example = ExtraExample("/fixtures/facetable.json?_extra=request") scopes = frozenset({ExtraScope.TABLE}) async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 24d59577..d418d16c 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -454,9 +454,40 @@ The available table extras are listed below. ``debug`` Extra debug information + ``GET /fixtures/facetable.json?_extra=debug`` + + .. code-block:: json + + { + "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", + "url_vars": { + "database": "fixtures", + "table": "facetable", + "format": "json" + }, + "nofacet": null, + "nosuggest": null + } + ``request`` Full information about the request + ``GET /fixtures/facetable.json?_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/facetable.json?_extra=request", + "path": "/fixtures/facetable.json", + "full_path": "/fixtures/facetable.json?_extra=request", + "host": "localhost", + "args": { + "_extra": [ + "request" + ] + } + } + ``query`` Details of the underlying SQL query diff --git a/tests/test_docs.py b/tests/test_docs.py index c4e0a849..3aa67730 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -130,6 +130,19 @@ def test_render_cell_extra_example_explains_row_and_column_mapping(): assert '"render_cell": [' in section +def test_debug_and_request_extra_examples_are_documented(): + content = (docs_path / "json_api.rst").read_text() + section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + + debug_section = section.split("``debug``")[-1].split("``request``")[0] + assert "GET /fixtures/facetable.json?_extra=debug" in debug_section + assert '"url_vars": {' in debug_section + + request_section = section.split("``request``")[-1].split("``query``")[0] + assert "GET /fixtures/facetable.json?_extra=request" in request_section + assert '"full_path":' in request_section + + @pytest.fixture(scope="session") def documented_labels(): labels = set() From 4d6daa175a67c4c6e895fe8b32ae051b1e9136a7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 9 Jun 2026 02:56:27 -0700 Subject: [PATCH 1085/1116] Add row and query JSON extras --- datasette/extras.py | 7 + datasette/views/database.py | 46 +++- datasette/views/row.py | 67 ++---- datasette/views/table_extras.py | 258 ++++++++++++++++++--- docs/json_api.rst | 386 +++++++++++++++++++++++++++++++- docs/json_api_doc.py | 141 ++++++++---- tests/test_api.py | 22 ++ tests/test_docs.py | 16 +- tests/test_table_api.py | 49 ++++ 9 files changed, 862 insertions(+), 130 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index d867f26c..f655e517 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -16,6 +16,8 @@ def extra_names_from_request(request): class ExtraScope(Enum): TABLE = "table" + ROW = "row" + QUERY = "query" @dataclass(frozen=True) @@ -46,11 +48,16 @@ class Provider: class Extra(Provider): description: ClassVar[str | None] = None example: ClassVar[ExtraExample | None] = None + examples: ClassVar[dict[ExtraScope, ExtraExample | list[ExtraExample]]] = {} public: ClassVar[bool] = True stable: ClassVar[bool] = True expensive: ClassVar[bool] = False docs_note: ClassVar[str | None] = None + @classmethod + def example_for_scope(cls, scope): + return cls.examples.get(scope, cls.example) + @classmethod def documentation(cls): return { diff --git a/datasette/views/database.py b/datasette/views/database.py index a1647ca9..96a58758 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -11,6 +11,7 @@ import sqlite_utils import textwrap from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent +from datasette.extras import extra_names_from_request from datasette.database import QueryInterrupted from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import stored_query_to_dict @@ -38,6 +39,11 @@ from datasette.plugins import pm from .base import BaseView, DatasetteError, View, _error, stream_csv from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns +from .table_extras import ( + QueryExtraContext, + resolve_query_extras, + table_extra_registry, +) from . import Context @@ -692,6 +698,34 @@ class QueryView(View): except DatasetteError: raise + extras = extra_names_from_request(request) + metadata = None + data = {"ok": True, "rows": rows, "columns": columns} + if extras: + metadata = await datasette.get_database_metadata(database) + if stored_query: + metadata = stored_query_to_dict(stored_query) + metadata.pop("source", None) + query_extra_context = QueryExtraContext( + datasette=datasette, + request=request, + db=db, + database_name=database, + private=private, + rows=rows, + columns=columns, + sql=sql, + params=params_for_query, + query_name=stored_query.name if stored_query else None, + stored_query=stored_query, + stored_query_write=stored_query_write, + error=query_error, + metadata=metadata, + extras=extras, + extra_registry=table_extra_registry, + ) + data.update(await resolve_query_extras(extras, query_extra_context)) + # Handle formats from plugins if format_ == "csv": if not sql: @@ -721,7 +755,7 @@ class QueryView(View): error=query_error, # These will be deprecated in Datasette 1.0: args=request.args, - data={"ok": True, "rows": rows, "columns": columns}, + data=data, ) if asyncio.iscoroutine(result): result = await result @@ -770,11 +804,11 @@ class QueryView(View): ) } ) - metadata = await datasette.get_database_metadata(database) - if stored_query: - metadata = stored_query_to_dict(stored_query) - metadata.pop("source", None) - + if metadata is None: + metadata = await datasette.get_database_metadata(database) + if stored_query: + metadata = stored_query_to_dict(stored_query) + metadata.pop("source", None) renderers = {} for key, (_, can_render) in datasette.renderers.items(): it_can_render = call_with_supported_arguments( diff --git a/datasette/views/row.py b/datasette/views/row.py index 4eacfe49..3fe213d7 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -15,6 +15,7 @@ import json import markupsafe import sqlite_utils from .table import display_columns_and_rows, _get_extras +from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry class RowView(DataView): @@ -172,52 +173,26 @@ class RowView(DataView): extras.add("foreign_key_tables") # Process extras - if "foreign_key_tables" in extras: - data["foreign_key_tables"] = await self.foreign_key_tables( - database, table, pk_values - ) - - if "render_cell" in extras: - # Call render_cell plugin hook for each cell - ct_map = await self.ds.get_column_types(database, table) - rendered_rows = [] - for row in rows: - rendered_row = {} - for value, column in zip(row, columns): - ct = ct_map.get(column) - plugin_display_value = None - # Try column type render_cell first - if ct: - candidate = await ct.render_cell( - value=value, - column=column, - table=table, - database=database, - datasette=self.ds, - request=request, - ) - if candidate is not None: - plugin_display_value = candidate - if plugin_display_value is None: - for candidate in pm.hook.render_cell( - row=row, - value=value, - column=column, - table=table, - pks=resolved.pks, - database=database, - datasette=self.ds, - request=request, - column_type=ct, - ): - candidate = await await_me_maybe(candidate) - if candidate is not None: - plugin_display_value = candidate - break - if plugin_display_value: - rendered_row[column] = str(plugin_display_value) - rendered_rows.append(rendered_row) - data["render_cell"] = rendered_rows + row_extra_context = RowExtraContext( + datasette=self.ds, + request=request, + resolved=resolved, + db=db, + database_name=database, + table_name=table, + private=private, + rows=rows, + columns=columns, + results_description=results.description, + pks=pks, + pk_values=pk_values, + sql=resolved.sql, + params=resolved.params, + extras=extras, + extra_registry=table_extra_registry, + foreign_key_tables=self.foreign_key_tables, + ) + data.update(await resolve_row_extras(extras, row_extra_context)) return ( data, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index e888ee9f..ec104be3 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -8,6 +8,7 @@ from datasette.resources import TableResource from datasette.utils import ( await_me_maybe, call_with_supported_arguments, + named_parameters as derive_named_parameters, path_with_added_args, path_with_format, path_with_removed_args, @@ -52,6 +53,50 @@ class TableExtraContext: extra_registry: ExtraRegistry display_columns_and_rows: object run_sequential: object + scope: ExtraScope = ExtraScope.TABLE + + +@dataclass(frozen=True) +class RowExtraContext: + datasette: object + request: object + resolved: object + db: object + database_name: str + table_name: str + private: bool + rows: list + columns: list + results_description: list + pks: list + pk_values: list + sql: str + params: dict + extras: set + extra_registry: ExtraRegistry + foreign_key_tables: object + scope: ExtraScope = ExtraScope.ROW + + +@dataclass(frozen=True) +class QueryExtraContext: + datasette: object + request: object + db: object + database_name: str + private: bool + rows: list + columns: list + sql: str | None + params: dict + query_name: str | None + stored_query: object + stored_query_write: bool + error: str | None + metadata: dict + extras: set + extra_registry: ExtraRegistry + scope: ExtraScope = ExtraScope.QUERY class CountSqlExtra(Extra): @@ -245,7 +290,15 @@ class NextUrlExtra(Extra): class ColumnsExtra(Extra): description = "Column names returned by this query" example = ExtraExample("/fixtures/facetable.json?_extra=columns") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=columns" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=columns" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.columns @@ -263,7 +316,12 @@ class AllColumnsExtra(Extra): class PrimaryKeysExtra(Extra): description = "Primary keys for this table" example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=primary_keys" + ) + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): return context.pks @@ -309,21 +367,49 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" example = ExtraExample("/fixtures/facetable.json?_extra=debug") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=debug" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=debug" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - return { - "resolved": repr(context.resolved), + debug = { "url_vars": context.request.url_vars, - "nofacet": context.nofacet, - "nosuggest": context.nosuggest, } + if context.scope == ExtraScope.TABLE: + debug["resolved"] = repr(context.resolved) + elif context.scope == ExtraScope.ROW: + debug["resolved"] = { + "table": context.table_name, + "sql": context.sql, + "params": context.params, + "pks": context.pks, + "pk_values": context.pk_values, + } + if hasattr(context, "nofacet"): + debug["nofacet"] = context.nofacet + if hasattr(context, "nosuggest"): + debug["nosuggest"] = context.nosuggest + return debug class RequestExtra(Extra): description = "Full information about the request" example = ExtraExample("/fixtures/facetable.json?_extra=request") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=request" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=request" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return { @@ -413,15 +499,48 @@ class RenderCellExtra(Extra): "whose rendered value differs from the default are included." ), ) - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + value={ + "rows": [{"id": 4, "content": "RENDER_CELL_DEMO"}], + "render_cell": [{"content": "Custom rendered HTML"}], + }, + note=( + "The ``render_cell`` array has one item for the requested row. " + "The object is keyed by column name. Only columns whose rendered " + "value differs from the default are included." + ), + ), + ExtraScope.QUERY: ExtraExample( + value={ + "rows": [{"content": "RENDER_CELL_DEMO"}], + "render_cell": [{"content": "Custom rendered HTML"}], + }, + note=( + "The ``render_cell`` array has one item per query result row, in " + "the same order as the ``rows`` array. Each object is keyed by " + "column name. Only columns whose rendered value differs from the " + "default are included." + ), + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + table_name = getattr(context, "table_name", None) + is_view = getattr(context, "is_view", False) + pks = getattr(context, "pks", []) pks_for_display = ( - context.pks if context.pks else (["rowid"] if not context.is_view else []) + pks if pks else (["rowid"] if table_name and not is_view else []) ) - col_names = [col[0] for col in context.results_description] - ct_map = await context.datasette.get_column_types( - context.database_name, context.table_name + if hasattr(context, "results_description"): + col_names = [col[0] for col in context.results_description] + else: + col_names = context.columns + ct_map = ( + await context.datasette.get_column_types(context.database_name, table_name) + if table_name + else {} ) rendered_rows = [] for row in context.rows: @@ -433,7 +552,7 @@ class RenderCellExtra(Extra): candidate = await ct.render_cell( value=value, column=column, - table=context.table_name, + table=table_name, database=context.database_name, datasette=context.datasette, request=context.request, @@ -445,7 +564,7 @@ class RenderCellExtra(Extra): row=row, value=value, column=column, - table=context.table_name, + table=table_name, pks=pks_for_display, database=context.database_name, datasette=context.datasette, @@ -465,19 +584,36 @@ class RenderCellExtra(Extra): class QueryExtra(Extra): description = "Details of the underlying SQL query" example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=query" + ), + ExtraScope.QUERY: [ + ExtraExample("/fixtures/-/query.json?sql=select+1+as+one&_extra=query"), + ExtraExample("/fixtures/neighborhood_search.json?text=town&_extra=query"), + ], + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + params = context.params + if context.scope == ExtraScope.QUERY and context.sql: + parameter_names = set(derive_named_parameters(context.sql)) + params = { + key: value + for key, value in dict(context.params).items() + if key in parameter_names + } return { "sql": context.sql, - "params": context.params, + "params": params, } class ColumnTypesExtra(Extra): description = "Column type assignments for this table" example = ExtraExample(value={}) - scopes = frozenset({ExtraScope.TABLE}) + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): ct_map = await context.datasette.get_column_types( @@ -544,11 +680,22 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): - description = "Metadata about the table and database" + description = "Metadata about the table, database or stored query" example = ExtraExample("/fixtures/facetable.json?_extra=metadata") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=metadata" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/neighborhood_search.json?text=town&_extra=metadata" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): + if context.scope == ExtraScope.QUERY: + return context.metadata + tablemetadata = await context.datasette.get_resource_metadata( context.database_name, context.table_name ) @@ -572,7 +719,15 @@ class MetadataExtra(Extra): class DatabaseExtra(Extra): description = "Database name" example = ExtraExample("/fixtures/facetable.json?_extra=database") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=database" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=database" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.database_name @@ -581,7 +736,10 @@ class DatabaseExtra(Extra): class TableExtra(Extra): description = "Table name" example = ExtraExample("/fixtures/facetable.json?_extra=table") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample("/fixtures/simple_primary_key/1.json?_extra=table") + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) async def resolve(self, context): return context.table_name @@ -590,7 +748,15 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" example = ExtraExample("/fixtures/facetable.json?_extra=database_color") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=database_color" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=database_color" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.db.color @@ -703,6 +869,8 @@ class RenderersExtra(Extra): url_labels_extra = {} if expandable_columns: url_labels_extra = {"_labels": "on"} + table_name = getattr(context, "table_name", None) + view_name = "table" if context.scope == ExtraScope.TABLE else "database" for key, (_, can_render) in context.datasette.renderers.items(): it_can_render = call_with_supported_arguments( can_render, @@ -710,11 +878,11 @@ class RenderersExtra(Extra): columns=context.columns or [], rows=context.rows or [], sql=query.get("sql", None), - query_name=None, + query_name=getattr(context, "query_name", None), database=context.database_name, - table=context.table_name, + table=table_name, request=context.request, - view_name="table", + view_name=view_name, ) it_can_render = await await_me_maybe(it_can_render) if it_can_render: @@ -730,9 +898,17 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): - description = "Whether this table is private to the current actor" + description = "Whether this resource is private to the current actor" example = ExtraExample("/fixtures/facetable.json?_extra=private") - scopes = frozenset({ExtraScope.TABLE}) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=private" + ), + ExtraScope.QUERY: ExtraExample( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" + ), + } + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): return context.private @@ -752,14 +928,27 @@ class ExpandableColumnsExtra(Extra): return expandables +class ForeignKeyTablesExtra(Extra): + description = "Tables that link to this row using foreign keys" + example = ExtraExample( + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" + ) + scopes = frozenset({ExtraScope.ROW}) + + async def resolve(self, context): + return await context.foreign_key_tables( + context.database_name, context.table_name, context.pk_values + ) + + class ExtrasExtra(Extra): description = "Available ?_extra= blocks" - scopes = frozenset({ExtraScope.TABLE}) + scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): all_extras = [ (cls.key(), cls.description) - for cls in context.extra_registry.public_classes_for_scope(ExtraScope.TABLE) + for cls in context.extra_registry.public_classes_for_scope(context.scope) ] return [ { @@ -850,6 +1039,7 @@ TABLE_EXTRA_CLASSES = [ IsViewExtra, PrivateExtra, ExpandableColumnsExtra, + ForeignKeyTablesExtra, FormHiddenArgsExtra, ] @@ -859,3 +1049,11 @@ table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) async def resolve_table_extras(extras, context): return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) + + +async def resolve_row_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.ROW) + + +async def resolve_query_extras(extras, context): + return await table_extra_registry.resolve(extras, context, ExtraScope.QUERY) diff --git a/docs/json_api.rst b/docs/json_api.rst index d418d16c..379d26a0 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -237,23 +237,26 @@ query string arguments: .. _json_api_extra: -Expanding table JSON responses ------------------------------- +Expanding JSON responses +------------------------ -Table JSON responses can be expanded with one or more ``?_extra=`` parameters. +Table, row and query JSON responses can be expanded with one or more ``?_extra=`` parameters. These can be repeated or comma-separated: :: ?_extra=columns&_extra=count,next_url -The available table extras are listed below. - .. [[[cog from json_api_doc import table_extras table_extras(cog) .. ]]] +Table JSON responses +~~~~~~~~~~~~~~~~~~~~ + +The available table extras are listed below. + ``count`` Total count of rows matching these filters (May execute additional queries.) @@ -459,12 +462,12 @@ The available table extras are listed below. .. code-block:: json { - "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", "url_vars": { "database": "fixtures", "table": "facetable", "format": "json" }, + "resolved": "ResolvedTable(db=, table='facetable', is_view=False)", "nofacet": null, "nosuggest": null } @@ -511,7 +514,7 @@ The available table extras are listed below. Column type UI metadata for this table ``metadata`` - Metadata about the table and database + Metadata about the table, database or stored query ``GET /fixtures/facetable.json?_extra=metadata`` @@ -649,7 +652,7 @@ The available table extras are listed below. true ``private`` - Whether this table is private to the current actor + Whether this resource is private to the current actor ``GET /fixtures/facetable.json?_extra=private`` @@ -697,6 +700,373 @@ The available table extras are listed below. ] ] +Row JSON responses +~~~~~~~~~~~~~~~~~~ + +The following extras are available for row JSON responses. + +``columns`` + Column names returned by this query + + ``GET /fixtures/simple_primary_key/1.json?_extra=columns`` + + .. code-block:: json + + [ + "id", + "content" + ] + +``primary_keys`` + Primary keys for this table + + ``GET /fixtures/simple_primary_key/1.json?_extra=primary_keys`` + + .. code-block:: json + + [ + "id" + ] + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + The ``render_cell`` array has one item for the requested row. The object is keyed by column name. Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + { + "rows": [ + { + "id": 4, + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + { + "content": "Custom rendered HTML" + } + ] + } + +``debug`` + Extra debug information + + ``GET /fixtures/simple_primary_key/1.json?_extra=debug`` + + .. code-block:: json + + { + "url_vars": { + "database": "fixtures", + "table": "simple_primary_key", + "pks": "1", + "format": "json" + }, + "resolved": { + "table": "simple_primary_key", + "sql": "select * from simple_primary_key where \"id\"=:p0", + "params": { + "p0": "1" + }, + "pks": [ + "id" + ], + "pk_values": [ + "1" + ] + } + } + +``request`` + Full information about the request + + ``GET /fixtures/simple_primary_key/1.json?_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/simple_primary_key/1.json?_extra=request", + "path": "/fixtures/simple_primary_key/1.json", + "full_path": "/fixtures/simple_primary_key/1.json?_extra=request", + "host": "localhost", + "args": { + "_extra": [ + "request" + ] + } + } + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/simple_primary_key/1.json?_extra=query`` + + .. code-block:: json + + { + "sql": "select * from simple_primary_key where \"id\"=:p0", + "params": { + "p0": "1" + } + } + +``column_types`` + Column type assignments for this table + + .. code-block:: json + + {} + +``metadata`` + Metadata about the table, database or stored query + + ``GET /fixtures/simple_primary_key/1.json?_extra=metadata`` + + .. code-block:: json + + { + "columns": {} + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/simple_primary_key/1.json?_extra=database`` + + .. code-block:: json + + "fixtures" + +``table`` + Table name + + ``GET /fixtures/simple_primary_key/1.json?_extra=table`` + + .. code-block:: json + + "simple_primary_key" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/simple_primary_key/1.json?_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``private`` + Whether this resource is private to the current actor + + ``GET /fixtures/simple_primary_key/1.json?_extra=private`` + + .. code-block:: json + + false + +``foreign_key_tables`` + Tables that link to this row using foreign keys + + ``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables`` + + .. code-block:: json + + [ + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f1", + "count": 1, + "link": "/fixtures/complex_foreign_keys?f1=1" + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f2", + "count": 0, + "link": "/fixtures/complex_foreign_keys?f2=1" + }, + { + "other_table": "complex_foreign_keys", + "column": "id", + "other_column": "f3", + "count": 1, + "link": "/fixtures/complex_foreign_keys?f3=1" + }, + { + "other_table": "foreign_key_references", + "column": "id", + "other_column": "foreign_key_with_blank_label", + "count": 0, + "link": "/fixtures/foreign_key_references?foreign_key_with_blank_label=1" + }, + { + "other_table": "foreign_key_references", + "column": "id", + "other_column": "foreign_key_with_label", + "count": 1, + "link": "/fixtures/foreign_key_references?foreign_key_with_label=1" + } + ] + +Query JSON responses +~~~~~~~~~~~~~~~~~~~~ + +The following extras are available for arbitrary SQL query responses and stored, named query responses. + +``columns`` + Column names returned by this query + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=columns`` + + .. code-block:: json + + [ + "one" + ] + +``render_cell`` + Rendered HTML for each cell using the render_cell plugin hook + + The ``render_cell`` array has one item per query result row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. + + .. code-block:: json + + { + "rows": [ + { + "content": "RENDER_CELL_DEMO" + } + ], + "render_cell": [ + { + "content": "Custom rendered HTML" + } + ] + } + +``debug`` + Extra debug information + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=debug`` + + .. code-block:: json + + { + "url_vars": { + "database": "fixtures", + "format": "json" + } + } + +``request`` + Full information about the request + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=request`` + + .. code-block:: json + + { + "url": "http://localhost/fixtures/-/query.json?sql=select+1+as+one&_extra=request", + "path": "/fixtures/-/query.json", + "full_path": "/fixtures/-/query.json?sql=select+1+as+one&_extra=request", + "host": "localhost", + "args": { + "sql": [ + "select 1 as one" + ], + "_extra": [ + "request" + ] + } + } + +``query`` + Details of the underlying SQL query + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query`` + + .. code-block:: json + + { + "sql": "select 1 as one", + "params": {} + } + + ``GET /fixtures/neighborhood_search.json?text=town&_extra=query`` + + .. code-block:: json + + { + "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities\n on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n", + "params": { + "text": "town" + } + } + +``metadata`` + Metadata about the table, database or stored query + + ``GET /fixtures/neighborhood_search.json?text=town&_extra=metadata`` + + .. code-block:: json + + { + "database": "fixtures", + "name": "neighborhood_search", + "sql": "\nselect _neighborhood, facet_cities.name, state\nfrom facetable\n join facet_cities\n on facetable._city_id = facet_cities.id\nwhere _neighborhood like '%' || :text || '%'\norder by _neighborhood;\n", + "title": "Search neighborhoods", + "description": null, + "description_html": null, + "hide_sql": false, + "fragment": null, + "params": [], + "parameters": [], + "is_write": false, + "is_private": false, + "is_trusted": true, + "owner_id": null, + "on_success_message": null, + "on_success_message_sql": null, + "on_success_redirect": null, + "on_error_message": null, + "on_error_redirect": null + } + +``extras`` + Available ?_extra= blocks + +``database`` + Database name + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database`` + + .. code-block:: json + + "fixtures" + +``database_color`` + Color assigned to the database + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database_color`` + + .. code-block:: json + + "9403e5" + +``private`` + Whether this resource is private to the current actor + + ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=private`` + + .. code-block:: json + + false + .. [[[end]]] .. _table_arguments: diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index 69ec6e5e..44ef4a42 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -9,39 +9,80 @@ def table_extras(cog): from datasette.extras import ExtraScope from datasette.views.table_extras import table_extra_registry - classes = table_extra_registry.public_classes_for_scope(ExtraScope.TABLE) + scopes = [ + ( + ExtraScope.TABLE, + "Table JSON responses", + "The available table extras are listed below.", + ), + ( + ExtraScope.ROW, + "Row JSON responses", + "The following extras are available for row JSON responses.", + ), + ( + ExtraScope.QUERY, + "Query JSON responses", + ( + "The following extras are available for arbitrary SQL query " + "responses and stored, named query responses." + ), + ), + ] + classes_by_scope = [ + (scope, heading, intro, table_extra_registry.public_classes_for_scope(scope)) + for scope, heading, intro in scopes + ] - live_examples = asyncio.run(_fetch_live_examples(classes)) + live_examples = asyncio.run( + _fetch_live_examples( + [ + (scope, cls) + for scope, _, _, classes in classes_by_scope + for cls in classes + ] + ) + ) cog.out("\n") - for cls in classes: - example = cls.example - description = cls.description or "" - notes = [] - if cls.expensive: - notes.append("May execute additional queries.") - if cls.docs_note: - notes.append(cls.docs_note) - if notes: - description = "{} ({})".format(description, " ".join(notes)).strip() + for scope, heading, intro, classes in classes_by_scope: + cog.out("{}\n{}\n\n".format(heading, "~" * len(heading))) + cog.out("{}\n\n".format(intro)) + for cls in classes: + examples = _examples_for_scope(cls, scope) + description = cls.description or "" + notes = [] + if cls.expensive: + notes.append("May execute additional queries.") + if cls.docs_note: + notes.append(cls.docs_note) + if notes: + description = "{} ({})".format(description, " ".join(notes)).strip() - cog.out("``{}``\n".format(cls.key())) - cog.out(" {}\n\n".format(description)) - if example is None: - continue - - if example.path: - value = live_examples[(example.path, example.key or cls.key())] - cog.out(" ``GET {}``\n\n".format(example.path)) - else: - value = example.value - if example.note: - cog.out(" {}\n\n".format(example.note)) - cog.out(" .. code-block:: json\n\n") - cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) - cog.out("\n\n") + cog.out("``{}``\n".format(cls.key())) + cog.out(" {}\n\n".format(description)) + for example in examples: + if example.path: + value = live_examples[(example.path, example.key or cls.key())] + cog.out(" ``GET {}``\n\n".format(example.path)) + else: + value = example.value + if example.note: + cog.out(" {}\n\n".format(example.note)) + cog.out(" .. code-block:: json\n\n") + cog.out(textwrap.indent(json.dumps(value, indent=2), " ")) + cog.out("\n\n") -async def _fetch_live_examples(classes): +def _examples_for_scope(cls, scope): + examples = cls.example_for_scope(scope) + if examples is None: + return [] + if isinstance(examples, list): + return examples + return [examples] + + +async def _fetch_live_examples(scoped_classes): from datasette.app import Datasette from datasette.fixtures import write_fixture_database @@ -49,18 +90,40 @@ async def _fetch_live_examples(classes): with tempfile.TemporaryDirectory() as tmpdir: db_path = pathlib.Path(tmpdir) / "fixtures.db" write_fixture_database(db_path) - datasette = Datasette([str(db_path)], settings={"num_sql_threads": 1}) + datasette = Datasette( + [str(db_path)], + settings={"num_sql_threads": 1}, + config={ + "databases": { + "fixtures": { + "queries": { + "neighborhood_search": { + "sql": textwrap.dedent(""" + select _neighborhood, facet_cities.name, state + from facetable + join facet_cities + on facetable._city_id = facet_cities.id + where _neighborhood like '%' || :text || '%' + order by _neighborhood; + """), + "title": "Search neighborhoods", + } + } + } + } + }, + ) try: - for cls in classes: - example = cls.example - if example is None or not example.path: - continue - key = example.key or cls.key() - response = await datasette.client.get(example.path) - assert response.status_code == 200, example.path - data = response.json() - assert key in data, "{} missing from {}".format(key, example.path) - examples[(example.path, key)] = data[key] + for scope, cls in scoped_classes: + for example in _examples_for_scope(cls, scope): + if not example.path: + continue + key = example.key or cls.key() + response = await datasette.client.get(example.path) + assert response.status_code == 200, example.path + data = response.json() + assert key in data, "{} missing from {}".format(key, example.path) + examples[(example.path, key)] = data[key] finally: for db in datasette.databases.values(): if not db.is_memory: diff --git a/tests/test_api.py b/tests/test_api.py index f6187529..e1385b6f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -426,6 +426,28 @@ async def test_row_foreign_key_tables(ds_client): ] +@pytest.mark.asyncio +async def test_row_extras(ds_client): + response = await ds_client.get( + "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables" + ) + assert response.status_code == 200 + data = response.json() + assert data["database"] == "fixtures" + assert data["table"] == "simple_primary_key" + assert data["primary_keys"] == ["id"] + assert data["query"]["sql"] == 'select * from simple_primary_key where "id"=:p0' + assert data["query"]["params"] == {"p0": "1"} + assert data["request"]["path"] == "/fixtures/simple_primary_key/1.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "table": "simple_primary_key", + "pks": "1", + "format": "json", + } + assert len(data["foreign_key_tables"]) == 5 + + @pytest.mark.asyncio async def test_row_extra_render_cell(): """Test that _extra=render_cell returns rendered HTML from render_cell plugin hook on row pages""" diff --git a/tests/test_docs.py b/tests/test_docs.py index 3aa67730..13b3a549 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -132,7 +132,7 @@ def test_render_cell_extra_example_explains_row_and_column_mapping(): def test_debug_and_request_extra_examples_are_documented(): content = (docs_path / "json_api.rst").read_text() - section = content.split(".. _json_api_extra:")[-1].split(".. _table_arguments:")[0] + section = content.split("Table JSON responses")[-1].split("Row JSON responses")[0] debug_section = section.split("``debug``")[-1].split("``request``")[0] assert "GET /fixtures/facetable.json?_extra=debug" in debug_section @@ -143,6 +143,20 @@ def test_debug_and_request_extra_examples_are_documented(): assert '"full_path":' in request_section +def test_row_and_query_extra_sections_are_documented(): + content = (docs_path / "json_api.rst").read_text() + assert "Row JSON responses" in content + assert ( + "``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables``" + in content + ) + assert "Query JSON responses" in content + assert "``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=query``" in content + assert ( + "``GET /fixtures/neighborhood_search.json?text=town&_extra=query``" in content + ) + + @pytest.fixture(scope="session") def documented_labels(): labels = set() diff --git a/tests/test_table_api.py b/tests/test_table_api.py index eeb3dc8b..388e3979 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -68,6 +68,55 @@ async def test_table_shape_arrayfirst(ds_client): ] +@pytest.mark.asyncio +async def test_query_extras_for_arbitrary_sql(ds_client): + response = await ds_client.get( + "/fixtures/-/query.json?" + + urllib.parse.urlencode( + { + "sql": "select 1 as one", + "_extra": "columns,database,query,request,debug", + } + ) + ) + assert response.status_code == 200 + data = response.json() + assert data["rows"] == [{"one": 1}] + assert data["columns"] == ["one"] + assert data["database"] == "fixtures" + assert data["query"]["sql"] == "select 1 as one" + assert data["request"]["path"] == "/fixtures/-/query.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "format": "json", + } + + +@pytest.mark.asyncio +async def test_query_extras_for_stored_query(ds_client): + response = await ds_client.get( + "/fixtures/neighborhood_search.json?" + + urllib.parse.urlencode( + { + "text": "town", + "_extra": "columns,database,query,request,debug", + } + ) + ) + assert response.status_code == 200 + data = response.json() + assert data["columns"] == ["_neighborhood", "name", "state"] + assert data["database"] == "fixtures" + assert data["query"]["sql"].strip().startswith("select _neighborhood") + assert data["query"]["params"]["text"] == "town" + assert data["request"]["path"] == "/fixtures/neighborhood_search.json" + assert data["debug"]["url_vars"] == { + "database": "fixtures", + "table": "neighborhood_search", + "format": "json", + } + + @pytest.mark.asyncio async def test_table_shape_objects(ds_client): response = await ds_client.get("/fixtures/simple_primary_key.json?_shape=objects") From d8605ef4c2c054610d2f4fbf1c00d182afa617e4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 19:58:00 -0700 Subject: [PATCH 1086/1116] Fix execute_isolated_fn() against immutable databases execute_isolated_fn() always opened its temporary connection with write=True, which is not allowed for immutable databases - so APIs that rely on it, like SQL analysis when storing a query, failed. An immutable database can never receive writes, so there is no write queue to block: in that case the function now opens a read-only connection and runs it on the executor, bypassing the write thread entirely. Mutable databases keep the existing write-thread behavior. Also fixed a latent bug in the write thread where a connect() failure for an isolated task would crash the thread instead of delivering the exception back to the caller. Closes #2768 Co-Authored-By: Claude Fable 5 --- datasette/database.py | 46 +++++++++++++++++++------------- tests/test_internals_database.py | 33 +++++++++++++++++++++++ tests/test_queries.py | 34 ++++++++++++++++++++++- 3 files changed, 94 insertions(+), 19 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 0a32442c..6cd5d11e 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -298,13 +298,14 @@ class Database: async def execute_isolated_fn(self, fn): self._check_not_closed() - # Open a new connection just for the duration of this function + # Open a new connection just for the duration of this function, # blocking the write queue to avoid any writes occurring during it - if self.ds.executor is None: - # non-threaded mode - isolated_connection = self.connect(write=True) + write = self.is_mutable + + def _run(): + isolated_connection = self.connect(write=write) try: - result = fn(isolated_connection) + return fn(isolated_connection) finally: isolated_connection.close() try: @@ -312,10 +313,18 @@ class Database: except ValueError: # Was probably a memory connection pass - return result - else: - # Threaded mode - send to write thread - return await self._send_to_write_thread(fn, isolated_connection=True) + + if self.ds.executor is None: + # non-threaded mode + return _run() + if not write: + # Immutable database - no writes can ever occur, so there is no + # write queue to block; run against a fresh read-only connection + return await asyncio.get_running_loop().run_in_executor( + self.ds.executor, _run + ) + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) async def analyze_sql(self, sql, params=None) -> SQLAnalysis: self._check_not_closed() @@ -449,20 +458,21 @@ class Database: if conn_exception is not None: exception = conn_exception elif task.isolated_connection: - isolated_connection = self.connect(write=True) try: - result = task.fn(isolated_connection) + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass except Exception as e: sys.stderr.write("{}\n".format(e)) sys.stderr.flush() exception = e - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass else: try: if task.transaction: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index bb209649..bad4e8ca 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -863,6 +863,39 @@ async def test_execute_isolated(db, disable_threads): assert not await db.execute_isolated_fn(table_exists_checker("created_by_isolated")) +@pytest.mark.asyncio +async def test_execute_isolated_connect_failure_does_not_kill_write_thread(): + # A connect() failure for an isolated task should be returned to the + # caller as an exception, not crash the write thread + class ConnectError(Exception): + pass + + ds = Datasette(memory=True) + db = ds.add_memory_database("test_isolated_connect_failure") + # Start the write thread with a healthy dedicated write connection + await db.execute_write("create table dogs (id integer primary key)") + + original_connect = db.connect + + def broken_connect(write=False): + raise ConnectError("Could not connect") + + db.connect = broken_connect + try: + with pytest.raises(ConnectError): + await asyncio.wait_for(db.execute_isolated_fn(lambda conn: None), timeout=2) + finally: + db.connect = original_connect + + # Write thread should still be alive and processing tasks + assert db._write_thread.is_alive() + await db.execute_write("insert into dogs (id) values (1)") + count = await db.execute_isolated_fn( + lambda conn: conn.execute("select count(*) from dogs").fetchone()[0] + ) + assert count == 1 + + @pytest.mark.asyncio async def test_analyze_sql(): ds = Datasette(memory=True) diff --git a/tests/test_queries.py b/tests/test_queries.py index 6e9bcbdb..0354f73a 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -9,7 +9,7 @@ from datasette.app import Datasette from datasette.resources import DatabaseResource, QueryResource from datasette.stored_queries import StoredQuery, StoredQueryPage from datasette.utils.asgi import Forbidden -from datasette.utils.sqlite import supports_returning +from datasette.utils.sqlite import sqlite3, supports_returning requires_sqlite_returning = pytest.mark.skipif( not supports_returning(), reason="SQLite does not support RETURNING" @@ -593,6 +593,38 @@ async def test_query_store_api_creates_read_only_query(): assert data["query"]["owner_id"] == "root" +@pytest.mark.asyncio +async def test_query_store_api_creates_query_for_immutable_database(tmp_path): + db_path = tmp_path / "immutable.db" + conn = sqlite3.connect(str(db_path)) + conn.execute("create table dogs (id integer primary key, name text)") + conn.commit() + conn.close() + + ds = Datasette([], immutables=[str(db_path)], default_deny=True) + ds.root_enabled = True + await ds.invoke_startup() + + response = await ds.client.post( + "/immutable/-/queries/store", + actor={"id": "root"}, + json={ + "query": { + "name": "by_name", + "sql": "select * from dogs where name = :name", + } + }, + ) + + ds.close() + assert response.status_code == 201 + data = response.json() + assert data["ok"] is True + assert data["query"]["name"] == "by_name" + assert data["query"]["parameters"] == ["name"] + assert data["query"]["is_write"] is False + + @pytest.mark.asyncio async def test_query_list_and_definition_api(): ds = Datasette(memory=True) From 3c1012dcc2995d184ea24fe70e8ccd6580592aff Mon Sep 17 00:00:00 2001 From: Viraat Das Date: Wed, 10 Jun 2026 20:15:03 -0700 Subject: [PATCH 1087/1116] Fix write query failing when a named parameter is called :sql (#2765) Closes #2761 --- .../templates/_sql_parameter_scripts.html | 34 ++++++++++++----- datasette/templates/_sql_parameters.html | 5 ++- datasette/views/execute_write.py | 2 + datasette/views/query_helpers.py | 14 ++++--- tests/test_api_write.py | 38 +++++++++++++++++++ tests/test_html.py | 4 +- tests/test_stored_queries.py | 2 +- 7 files changed, 79 insertions(+), 20 deletions(-) diff --git a/datasette/templates/_sql_parameter_scripts.html b/datasette/templates/_sql_parameter_scripts.html index 159a141c..9b83889e 100644 --- a/datasette/templates/_sql_parameter_scripts.html +++ b/datasette/templates/_sql_parameter_scripts.html @@ -27,16 +27,20 @@ window.datasetteSqlParameters = (() => { manager.section .querySelectorAll("[data-parameter-control]") .forEach((control) => { - manager.parameterState.set(control.name, controlState(control)); + manager.parameterState.set( + control.dataset.parameterName, + controlState(control) + ); }); } - function createControl(parameter, id, state) { + function createControl(parameter, id, state, namePrefix) { const control = document.createElement(state.expanded ? "textarea" : "input"); control.id = id; - control.name = parameter; + control.name = `${namePrefix || ""}${parameter}`; control.value = state.value; control.setAttribute("data-parameter-control", ""); + control.dataset.parameterName = parameter; if (state.expanded) { control.rows = 5; } else { @@ -53,10 +57,16 @@ window.datasetteSqlParameters = (() => { value, selectionStart ) { - const replacement = createControl(control.name, control.id, { - value: value === undefined ? control.value : value, - expanded: expand, - }); + const parameter = control.dataset.parameterName; + const replacement = createControl( + parameter, + control.id, + { + value: value === undefined ? control.value : value, + expanded: expand, + }, + manager.namePrefix + ); button.textContent = expand ? "Collapse" : "Expand"; button.setAttribute("aria-expanded", expand ? "true" : "false"); control.replaceWith(replacement); @@ -64,7 +74,7 @@ window.datasetteSqlParameters = (() => { if (selectionStart !== undefined && replacement.setSelectionRange) { replacement.setSelectionRange(selectionStart, selectionStart); } - manager.parameterState.set(replacement.name, controlState(replacement)); + manager.parameterState.set(parameter, controlState(replacement)); } function renderParameters(manager, parameters) { @@ -99,7 +109,7 @@ window.datasetteSqlParameters = (() => { label.htmlFor = id; label.textContent = parameter; - const control = createControl(parameter, id, state); + const control = createControl(parameter, id, state, manager.namePrefix); row.append(label, control); if (manager.allowExpand) { @@ -124,7 +134,10 @@ window.datasetteSqlParameters = (() => { if (!control.matches || !control.matches("[data-parameter-control]")) { return; } - manager.parameterState.set(control.name, controlState(control)); + manager.parameterState.set( + control.dataset.parameterName, + controlState(control) + ); }); if (!manager.allowExpand) { @@ -230,6 +243,7 @@ window.datasetteSqlParameters = (() => { ? section.dataset.allowExpand === "1" : false : options.allowExpand, + namePrefix: section ? section.dataset.parameterNamePrefix || "" : "", parameterState: new Map(), }; if (section) { diff --git a/datasette/templates/_sql_parameters.html b/datasette/templates/_sql_parameters.html index 58801d40..b5c1bde8 100644 --- a/datasette/templates/_sql_parameters.html +++ b/datasette/templates/_sql_parameters.html @@ -1,9 +1,10 @@ -
+{% set sql_parameter_name_prefix = sql_parameter_name_prefix|default("") %} +
{% if parameter_names %}

Parameters

{% for parameter in parameter_names %} {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %} -

{% if sql_parameters_allow_expand|default(false) %} {% endif %}

+

{% if sql_parameters_allow_expand|default(false) %} {% endif %}

{% endfor %} {% endif %}
diff --git a/datasette/views/execute_write.py b/datasette/views/execute_write.py index c5d55b80..2817f56e 100644 --- a/datasette/views/execute_write.py +++ b/datasette/views/execute_write.py @@ -9,6 +9,7 @@ from .base import BaseView, _error from .database import display_rows as display_query_rows from .query_helpers import ( QueryValidationError, + SQL_PARAMETER_FORM_PREFIX, _analysis_is_write, _analysis_rows, _analysis_rows_with_permissions, @@ -295,6 +296,7 @@ class ExecuteWriteView(BaseView): "execute_write_columns": execute_write_columns, "execute_write_display_rows": execute_write_display_rows, "execute_write_truncated": execute_write_truncated, + "sql_parameter_name_prefix": SQL_PARAMETER_FORM_PREFIX, "execute_disabled": bool(execute_disabled_reason), "execute_disabled_reason": execute_disabled_reason, "table_columns": table_columns, diff --git a/datasette/views/query_helpers.py b/datasette/views/query_helpers.py index 9efe3f81..026a999f 100644 --- a/datasette/views/query_helpers.py +++ b/datasette/views/query_helpers.py @@ -49,6 +49,8 @@ _query_write_fields = { "on_error_redirect", } +SQL_PARAMETER_FORM_PREFIX = "_sql_param_" + class QueryValidationError(Exception): def __init__(self, message, status=400, *, flash=False): @@ -289,11 +291,13 @@ def _coerce_execute_write_payload(data, is_json): ) params = data.get("params") or {} else: - params = { - key: value - for key, value in data.items() - if key not in {"sql", "csrftoken", "_json"} - } + params = {} + for key, value in data.items(): + if key in {"sql", "csrftoken", "_json"}: + continue + if key.startswith(SQL_PARAMETER_FORM_PREFIX): + key = key[len(SQL_PARAMETER_FORM_PREFIX) :] + params[key] = value if not isinstance(params, dict): raise QueryValidationError("params must be a dictionary") return data.get("sql"), params diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 64f91701..b7ceb6b2 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -794,6 +794,44 @@ async def test_update_row_alter(ds_write): assert response.json() == {"ok": True} +@pytest.mark.asyncio +async def test_execute_write_form_parameter_called_sql(): + ds = Datasette(memory=True, default_deny=True) + ds.root_enabled = True + db = ds.add_memory_database("execute_write_parameter_sql", name="data") + await db.execute_write("create table docs (id integer primary key, title text)") + await db.execute_write("insert into docs (id, title) values (1, 'Initial')") + await ds.invoke_startup() + + form_response = await ds.client.get( + "/data/-/execute-write", + actor={"id": "root"}, + params={"sql": "update docs set title = :sql where id = :id"}, + ) + assert form_response.status_code == 200 + assert 'data-parameter-name-prefix="_sql_param_"' in form_response.text + assert '' in form_response.text + assert 'name="_sql_param_sql"' in form_response.text + assert 'data-parameter-name="sql"' in form_response.text + assert 'name="_sql_param_id"' in form_response.text + + response = await ds.client.post( + "/data/-/execute-write", + actor={"id": "root"}, + data={ + "sql": "update docs set title = :sql where id = :id", + "_sql_param_sql": "Updated", + "_sql_param_id": "1", + }, + ) + + assert response.status_code == 200 + assert "Query executed, 1 row affected" in response.text + assert (await db.execute("select title from docs where id = 1")).first()[ + 0 + ] == "Updated" + + @pytest.mark.asyncio @pytest.mark.parametrize( "input,expected_errors", diff --git a/tests/test_html.py b/tests/test_html.py index bb7f612e..20ab22bc 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -342,7 +342,7 @@ async def test_query_parameter_form_fields(ds_client): response = await ds_client.get("/fixtures/-/query?sql=select+:name") assert response.status_code == 200 assert ( - ' ' + ' ' in response.text ) assert 'data-parameters-url="/fixtures/-/query/parameters"' in response.text @@ -351,7 +351,7 @@ async def test_query_parameter_form_fields(ds_client): response2 = await ds_client.get("/fixtures/-/query?sql=select+:name&name=hello") assert response2.status_code == 200 assert ( - ' ' + ' ' in response2.text ) diff --git a/tests/test_stored_queries.py b/tests/test_stored_queries.py index 2c648d5f..46420749 100644 --- a/tests/test_stored_queries.py +++ b/tests/test_stored_queries.py @@ -201,7 +201,7 @@ def test_error_in_on_success_message_sql(stored_write_client): def test_custom_params(stored_write_client): response = stored_write_client.get("/data/update_name?extra=foo") assert ( - '' + '' in response.text ) From f4b450603559b6a6412ed67e9eb170255dd1ab6b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:49:23 -0700 Subject: [PATCH 1088/1116] Remove legacy ?_extras= row parameter The pre-1.0 ?_extras= (plural) parameter was kept for backwards compatibility with the old row JSON API. ?_extra= is the documented mechanism now that row pages share the extras registry. Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 5 ----- tests/test_api.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index 3fe213d7..ce15a822 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -165,13 +165,8 @@ class RowView(DataView): "primary_key_values": pk_values, } - # Handle _extra parameter (new style) extras = _get_extras(request) - # Also support legacy _extras parameter for backward compatibility - if "foreign_key_tables" in (request.args.get("_extras") or "").split(","): - extras.add("foreign_key_tables") - # Process extras row_extra_context = RowExtraContext( datasette=self.ds, diff --git a/tests/test_api.py b/tests/test_api.py index e1385b6f..f57d0206 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -383,7 +383,7 @@ async def test_row_strange_table_name(ds_client): @pytest.mark.asyncio async def test_row_foreign_key_tables(ds_client): response = await ds_client.get( - "/fixtures/simple_primary_key/1.json?_extras=foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" ) assert response.status_code == 200 # Foreign keys are sorted by (other_table, column, other_column) From d825d8c4f38d980356abc50c739a440585253062 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:53:41 -0700 Subject: [PATCH 1089/1116] Remove _get_extras() shim in favor of extra_names_from_request() Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 5 +++-- datasette/views/table.py | 6 +----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index ce15a822..c6721ca0 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -14,7 +14,8 @@ from datasette.plugins import pm import json import markupsafe import sqlite_utils -from .table import display_columns_and_rows, _get_extras +from datasette.extras import extra_names_from_request +from .table import display_columns_and_rows from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry @@ -165,7 +166,7 @@ class RowView(DataView): "primary_key_values": pk_values, } - extras = _get_extras(request) + extras = extra_names_from_request(request) # Process extras row_extra_context = RowExtraContext( diff --git a/datasette/views/table.py b/datasette/views/table.py index c2d520f8..1b298c50 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -854,10 +854,6 @@ class TableDropView(BaseView): return Response.json({"ok": True}, status=200) -def _get_extras(request): - return extra_names_from_request(request) - - async def _columns_to_select(table_columns, pks, request): columns = list(table_columns) if "_col" in request.args: @@ -1461,7 +1457,7 @@ async def table_view_data( rows = rows[:page_size] # Resolve extras - extras = _get_extras(request) + extras = extra_names_from_request(request) if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")): extras.add("facet_results") if request.args.get("_shape") == "object": From df8a61450b478e66c458b3f05c286daeb2c2a6b0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 21:57:02 -0700 Subject: [PATCH 1090/1116] Remove hasattr/getattr probing from multi-scope extras TableExtraContext, RowExtraContext and QueryExtraContext now share normalized table_name, is_view, pks and query_name fields (defaulting to None/False where inapplicable) so DebugExtra, RenderCellExtra and RenderersExtra can read them directly. RenderCellExtra uses context.columns in every scope - the table and row views both derive columns from results.description so output is unchanged. Co-Authored-By: Claude Fable 5 --- datasette/views/row.py | 1 - datasette/views/table_extras.py | 30 +++++++++++++----------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/datasette/views/row.py b/datasette/views/row.py index c6721ca0..e15dfce9 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -179,7 +179,6 @@ class RowView(DataView): private=private, rows=rows, columns=columns, - results_description=results.description, pks=pks, pk_values=pk_values, sql=resolved.sql, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index ec104be3..63c87a6f 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -53,6 +53,7 @@ class TableExtraContext: extra_registry: ExtraRegistry display_columns_and_rows: object run_sequential: object + query_name: str | None = None scope: ExtraScope = ExtraScope.TABLE @@ -67,7 +68,6 @@ class RowExtraContext: private: bool rows: list columns: list - results_description: list pks: list pk_values: list sql: str @@ -75,6 +75,7 @@ class RowExtraContext: extras: set extra_registry: ExtraRegistry foreign_key_tables: object + is_view: bool = False scope: ExtraScope = ExtraScope.ROW @@ -96,6 +97,9 @@ class QueryExtraContext: metadata: dict extras: set extra_registry: ExtraRegistry + table_name: str | None = None + is_view: bool = False + pks: list | None = None scope: ExtraScope = ExtraScope.QUERY @@ -383,6 +387,8 @@ class DebugExtra(Extra): } if context.scope == ExtraScope.TABLE: debug["resolved"] = repr(context.resolved) + debug["nofacet"] = context.nofacet + debug["nosuggest"] = context.nosuggest elif context.scope == ExtraScope.ROW: debug["resolved"] = { "table": context.table_name, @@ -391,10 +397,6 @@ class DebugExtra(Extra): "pks": context.pks, "pk_values": context.pk_values, } - if hasattr(context, "nofacet"): - debug["nofacet"] = context.nofacet - if hasattr(context, "nosuggest"): - debug["nosuggest"] = context.nosuggest return debug @@ -527,16 +529,10 @@ class RenderCellExtra(Extra): scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - table_name = getattr(context, "table_name", None) - is_view = getattr(context, "is_view", False) - pks = getattr(context, "pks", []) - pks_for_display = ( - pks if pks else (["rowid"] if table_name and not is_view else []) + table_name = context.table_name + pks_for_display = context.pks or ( + ["rowid"] if table_name and not context.is_view else [] ) - if hasattr(context, "results_description"): - col_names = [col[0] for col in context.results_description] - else: - col_names = context.columns ct_map = ( await context.datasette.get_column_types(context.database_name, table_name) if table_name @@ -545,7 +541,7 @@ class RenderCellExtra(Extra): rendered_rows = [] for row in context.rows: rendered_row = {} - for value, column in zip(row, col_names): + for value, column in zip(row, context.columns): ct = ct_map.get(column) plugin_display_value = None if ct: @@ -869,7 +865,7 @@ class RenderersExtra(Extra): url_labels_extra = {} if expandable_columns: url_labels_extra = {"_labels": "on"} - table_name = getattr(context, "table_name", None) + table_name = context.table_name view_name = "table" if context.scope == ExtraScope.TABLE else "database" for key, (_, can_render) in context.datasette.renderers.items(): it_can_render = call_with_supported_arguments( @@ -878,7 +874,7 @@ class RenderersExtra(Extra): columns=context.columns or [], rows=context.rows or [], sql=query.get("sql", None), - query_name=getattr(context, "query_name", None), + query_name=context.query_name, database=context.database_name, table=table_name, request=context.request, From ab62ec96d187fa05f7d672d48a0b3f962fb8c228 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:45:13 -0700 Subject: [PATCH 1091/1116] Fix _extra=private for arbitrary SQL query pages QueryView hardcoded private=False unless the request was for a stored query, so /db/-/query.json?_extra=private reported false even when execute-sql was restricted to the authenticated actor. Use check_visibility() like the table and row views do. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 6 ++++-- tests/test_table_api.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index 96a58758..e6efddea 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -612,11 +612,13 @@ class QueryView(View): ) else: - await datasette.ensure_permission( + visible, private = await datasette.check_visibility( + request.actor, action="execute-sql", resource=DatabaseResource(database=database), - actor=request.actor, ) + if not visible: + raise Forbidden("execute-sql") # Flattened because of ?sql=&name1=value1&name2=value2 feature params = {key: request.args.get(key) for key in request.args} diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 388e3979..4ab2f596 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -117,6 +117,29 @@ async def test_query_extras_for_stored_query(ds_client): } +def test_query_extra_private_for_arbitrary_sql(): + with make_app_client(config={"allow_sql": {"id": "root"}}) as client: + cookies = {"ds_actor": client.actor_cookie({"id": "root"})} + response = client.get( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private", + cookies=cookies, + ) + assert response.status == 200 + assert response.json["private"] is True + # Anonymous users cannot execute SQL at all here + anon = client.get("/fixtures/-/query.json?sql=select+1+as+one") + assert anon.status == 403 + + +def test_query_extra_private_false_when_sql_is_public(): + with make_app_client() as client: + response = client.get( + "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" + ) + assert response.status == 200 + assert response.json["private"] is False + + @pytest.mark.asyncio async def test_table_shape_objects(ds_client): response = await ds_client.get("/fixtures/simple_primary_key.json?_shape=objects") From 8f888515b618bc0eb18e23c861b938b6bbbbf5d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:47:26 -0700 Subject: [PATCH 1092/1116] Fix _extra=query to report the params that were actually bound QueryExtra re-derived named parameters from the SQL with a regex, which missed parameters declared in a stored query's params list, reported magic _-prefixed parameters with raw querystring values that were never bound, and echoed the entire querystring when no SQL was present. QueryView now passes its named_parameter_values dict - the parameters it actually bound - through QueryExtraContext. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 2 +- datasette/views/table_extras.py | 11 +--------- tests/test_table_api.py | 38 +++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index e6efddea..a719fa4f 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -717,7 +717,7 @@ class QueryView(View): rows=rows, columns=columns, sql=sql, - params=params_for_query, + params=named_parameter_values, query_name=stored_query.name if stored_query else None, stored_query=stored_query, stored_query_write=stored_query_write, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 63c87a6f..21a908a0 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -8,7 +8,6 @@ from datasette.resources import TableResource from datasette.utils import ( await_me_maybe, call_with_supported_arguments, - named_parameters as derive_named_parameters, path_with_added_args, path_with_format, path_with_removed_args, @@ -592,17 +591,9 @@ class QueryExtra(Extra): scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) async def resolve(self, context): - params = context.params - if context.scope == ExtraScope.QUERY and context.sql: - parameter_names = set(derive_named_parameters(context.sql)) - params = { - key: value - for key, value in dict(context.params).items() - if key in parameter_names - } return { "sql": context.sql, - "params": params, + "params": context.params, } diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 4ab2f596..cfa3b512 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -131,6 +131,44 @@ def test_query_extra_private_for_arbitrary_sql(): assert anon.status == 403 +def test_query_extra_query_reports_bound_params(): + config = { + "databases": { + "fixtures": { + "queries": { + "declared_params": { + "sql": "select 1 as one", + "params": ["foo"], + }, + "magic_host": { + "sql": "select :_header_host as h", + }, + } + } + } + } + with make_app_client(config=config) as client: + # Declared parameters are reported even when the regex cannot find them + response = client.get("/fixtures/declared_params.json?foo=bar&_extra=query") + assert response.status == 200 + assert response.json["query"]["params"] == {"foo": "bar"} + # Magic parameters are bound internally and should not be reported, + # especially not as a value taken from the querystring + response = client.get( + "/fixtures/magic_host.json?_extra=query&_header_host=spoofed" + ) + assert response.status == 200 + assert response.json["rows"] == [{"h": "localhost"}] + assert response.json["query"]["params"] == {} + + +def test_query_extra_query_does_not_echo_querystring_without_sql(): + with make_app_client() as client: + response = client.get("/fixtures/-/query.json?_extra=query&foo=bar") + assert response.status == 200 + assert response.json["query"]["params"] == {} + + def test_query_extra_private_false_when_sql_is_public(): with make_app_client() as client: response = client.get( From b635dc53f42e06908c7510d743e85100a6488f22 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:50:44 -0700 Subject: [PATCH 1093/1116] Make filters, actions and display_rows extras internal These three extras return values that exist for the HTML templates - a Filters instance, an async function and markupsafe/sqlite3.Row data - so requesting them on a .json page returned a 500 serialization error, while the generated documentation and ?_extra=extras both advertised them as API surface. They are now public=False: ignored like any unknown name on JSON requests, omitted from the docs and the extras list, and still resolved for the HTML view via the new include_internal flag on ExtraRegistry.resolve(). Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 17 ++++++++--------- datasette/views/table.py | 9 ++++++++- datasette/views/table_extras.py | 12 ++++++++++-- docs/json_api.rst | 9 --------- tests/test_table_api.py | 18 ++++++++++++++++++ 5 files changed, 44 insertions(+), 21 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index f655e517..d5847937 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -89,7 +89,7 @@ class ExtraRegistry: def public_classes_for_scope(self, scope): return self.classes_for_scope(scope, include_internal=False) - async def resolve(self, requested, context, scope): + async def resolve(self, requested, context, scope, include_internal=False): registry = Registry() async def context_provider(): @@ -100,15 +100,14 @@ class ExtraRegistry: for cls in self.classes_for_scope(scope): registry.register(cls().resolve, name=cls.key()) - public_names = {cls.key() for cls in self.public_classes_for_scope(scope)} - requested_public_names = [ - name - for name in requested - if name in public_names and name in registry._registry - ] - resolved = await registry.resolve_multi(requested_public_names) + allowed_names = { + cls.key() + for cls in self.classes_for_scope(scope, include_internal=include_internal) + } + requested_names = [name for name in requested if name in allowed_names] + resolved = await registry.resolve_multi(requested_names) return { - name: resolved[name] for name in requested_public_names if name in resolved + name: resolved[name] for name in requested_names if name in resolved } diff --git a/datasette/views/table.py b/datasette/views/table.py index 1b298c50..3cf8e6c6 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1518,7 +1518,14 @@ async def table_view_data( "ok": True, "next": next_value and str(next_value) or None, } - data.update(await resolve_table_extras(extras, table_extra_context)) + data.update( + await resolve_table_extras( + extras, + table_extra_context, + # The HTML view needs extras that are not JSON serializable + include_internal=bool(extra_extras), + ) + ) raw_sqlite_rows = rows[:page_size] # Apply transform_value for columns with assigned types ct_map = await datasette.get_column_types(database_name, table_name) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 21a908a0..c98ae22c 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -333,6 +333,8 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): description = "Table or view actions made available by plugin hooks" scopes = frozenset({ExtraScope.TABLE}) + # Returns an async function for the HTML templates - not JSON serializable + public = False async def resolve(self, context): async def actions(): @@ -476,6 +478,8 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): description = "Row data formatted for the HTML table display" scopes = frozenset({ExtraScope.TABLE}) + # Contains markupsafe/sqlite3.Row values - not JSON serializable + public = False async def resolve(self, context, display_columns_and_rows): return display_columns_and_rows["rows"] @@ -772,6 +776,8 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): description = "Filters object used by the HTML table interface" scopes = frozenset({ExtraScope.TABLE}) + # Returns a Filters instance for the HTML templates - not JSON serializable + public = False async def resolve(self, context): return context.filters @@ -1034,8 +1040,10 @@ TABLE_EXTRA_CLASSES = [ table_extra_registry = ExtraRegistry(TABLE_EXTRA_CLASSES) -async def resolve_table_extras(extras, context): - return await table_extra_registry.resolve(extras, context, ExtraScope.TABLE) +async def resolve_table_extras(extras, context, include_internal=False): + return await table_extra_registry.resolve( + extras, context, ExtraScope.TABLE, include_internal=include_internal + ) async def resolve_row_extras(extras, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 379d26a0..6b595577 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -425,9 +425,6 @@ The available table extras are listed below. } ] -``display_rows`` - Row data formatted for the HTML table display - ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook @@ -554,12 +551,6 @@ The available table extras are listed below. "9403e5" -``actions`` - Table or view actions made available by plugin hooks - -``filters`` - Filters object used by the HTML table interface - ``renderers`` Alternative output renderers available for this table diff --git a/tests/test_table_api.py b/tests/test_table_api.py index cfa3b512..0cb67164 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -117,6 +117,24 @@ async def test_query_extras_for_stored_query(ds_client): } +@pytest.mark.parametrize("extra", ["filters", "actions", "display_rows"]) +@pytest.mark.asyncio +async def test_html_only_extras_are_not_available_via_json(ds_client, extra): + # These extras exist for the HTML view; their values are not JSON + # serializable so they are internal, not part of the JSON API + response = await ds_client.get(f"/fixtures/facetable.json?_extra={extra}") + assert response.status_code == 200 + assert extra not in response.json() + + +@pytest.mark.asyncio +async def test_html_only_extras_are_not_advertised(ds_client): + response = await ds_client.get("/fixtures/facetable.json?_extra=extras") + assert response.status_code == 200 + names = {e["name"] for e in response.json()["extras"]} + assert {"filters", "actions", "display_rows"}.isdisjoint(names) + + def test_query_extra_private_for_arbitrary_sql(): with make_app_client(config={"allow_sql": {"id": "root"}}) as client: cookies = {"ds_actor": client.actor_cookie({"id": "root"})} From bbf0424c4519441715f73ee6468e0c53cc959861 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:51:25 -0700 Subject: [PATCH 1094/1116] Changelog for row/query extras and related fixes Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 75e4f3e8..19089dd1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,13 @@ Changelog ------------------- - Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) +- Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. See :ref:`json_api_extra` for the full list. +- New generated reference documentation for every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra`. +- ``?_extra=`` values can be separated by commas as well as repeated, e.g. ``?_extra=count,next_url``. Previously a comma-separated value that included ``columns`` failed to include the ``columns`` key in the response. +- The ``?_extra=private`` extra on arbitrary SQL query pages now correctly reflects whether the SQL execution permission is private to the current actor - it previously always returned ``false``. +- The ``?_extra=query`` extra on query pages now reports the named parameters that were actually bound when the query executed, including parameters declared in a stored query's ``params`` list. Magic ``_``-prefixed parameters are no longer echoed back with unbound values taken from the querystring. +- Extras that exist to serve the HTML interface (``filters``, ``actions``, ``display_rows``) are no longer advertised or reachable through the JSON API, where requesting them previously returned a 500 serialization error. +- The pre-1.0 ``?_extras=`` (plural) parameter on row pages has been removed - use ``?_extra=foreign_key_tables`` instead. .. _v1_0_a32: From 6babd23cec9c41edd3d0ba2fab1c319905446b21 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:53:00 -0700 Subject: [PATCH 1095/1116] QueryView: only resolve extras for renderer formats, single metadata path Extras were resolved before the format dispatch, so a .csv request carrying ?_extra= parameters paid for extras (including per-cell render_cell plugin calls) whose results were then discarded, and the HTML path duplicated the stored-query metadata derivation. Extras now resolve inside the renderer-dispatch branch only, and both consumers share a query_metadata() helper that no longer fetches database metadata just to throw it away for stored queries. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 55 ++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index a719fa4f..ad3fb843 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -700,33 +700,12 @@ class QueryView(View): except DatasetteError: raise - extras = extra_names_from_request(request) - metadata = None - data = {"ok": True, "rows": rows, "columns": columns} - if extras: - metadata = await datasette.get_database_metadata(database) + async def query_metadata(): if stored_query: metadata = stored_query_to_dict(stored_query) metadata.pop("source", None) - query_extra_context = QueryExtraContext( - datasette=datasette, - request=request, - db=db, - database_name=database, - private=private, - rows=rows, - columns=columns, - sql=sql, - params=named_parameter_values, - query_name=stored_query.name if stored_query else None, - stored_query=stored_query, - stored_query_write=stored_query_write, - error=query_error, - metadata=metadata, - extras=extras, - extra_registry=table_extra_registry, - ) - data.update(await resolve_query_extras(extras, query_extra_context)) + return metadata + return await datasette.get_database_metadata(database) # Handle formats from plugins if format_ == "csv": @@ -740,6 +719,28 @@ class QueryView(View): return await stream_csv(datasette, fetch_data_for_csv, request, db.name) elif format_ in datasette.renderers.keys(): + data = {"ok": True, "rows": rows, "columns": columns} + extras = extra_names_from_request(request) + if extras: + query_extra_context = QueryExtraContext( + datasette=datasette, + request=request, + db=db, + database_name=database, + private=private, + rows=rows, + columns=columns, + sql=sql, + params=named_parameter_values, + query_name=stored_query.name if stored_query else None, + stored_query=stored_query, + stored_query_write=stored_query_write, + error=query_error, + metadata=await query_metadata(), + extras=extras, + extra_registry=table_extra_registry, + ) + data.update(await resolve_query_extras(extras, query_extra_context)) # Dispatch request to the correct output format renderer # (CSV is not handled here due to streaming) result = call_with_supported_arguments( @@ -806,11 +807,7 @@ class QueryView(View): ) } ) - if metadata is None: - metadata = await datasette.get_database_metadata(database) - if stored_query: - metadata = stored_query_to_dict(stored_query) - metadata.pop("source", None) + metadata = await query_metadata() renderers = {} for key, (_, can_render) in datasette.renderers.items(): it_can_render = call_with_supported_arguments( From a1b6a6976d0ddafba6b927ce9fb83e62cb9091c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:55:28 -0700 Subject: [PATCH 1096/1116] Remove dead weight from the extras machinery - TableExtraContext.next_value, RowExtraContext.resolved and QueryExtraContext.stored_query/stored_query_write/error had no readers - drop the fields and the arguments that populated them - Extra.documentation() and the stable classvar were unused parallel descriptions of what the docs generator reads directly - ExtraRegistry.resolve no longer carries an always-true membership guard (resolve_multi returns every requested registered name) Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 19 +------------------ datasette/views/database.py | 3 --- datasette/views/row.py | 1 - datasette/views/table.py | 1 - datasette/views/table_extras.py | 5 ----- 5 files changed, 1 insertion(+), 28 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index d5847937..4aa93057 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -50,7 +50,6 @@ class Extra(Provider): example: ClassVar[ExtraExample | None] = None examples: ClassVar[dict[ExtraScope, ExtraExample | list[ExtraExample]]] = {} public: ClassVar[bool] = True - stable: ClassVar[bool] = True expensive: ClassVar[bool] = False docs_note: ClassVar[str | None] = None @@ -58,20 +57,6 @@ class Extra(Provider): def example_for_scope(cls, scope): return cls.examples.get(scope, cls.example) - @classmethod - def documentation(cls): - return { - "name": cls.key(), - "description": cls.description, - "scopes": [ - scope.value for scope in sorted(cls.scopes, key=lambda s: s.value) - ], - "stable": cls.stable, - "expensive": cls.expensive, - "docs_note": cls.docs_note, - "example": cls.example, - } - class ExtraRegistry: def __init__(self, classes): @@ -106,9 +91,7 @@ class ExtraRegistry: } requested_names = [name for name in requested if name in allowed_names] resolved = await registry.resolve_multi(requested_names) - return { - name: resolved[name] for name in requested_names if name in resolved - } + return {name: resolved[name] for name in requested_names} def _camel_to_snake(name): diff --git a/datasette/views/database.py b/datasette/views/database.py index ad3fb843..46e26496 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -733,9 +733,6 @@ class QueryView(View): sql=sql, params=named_parameter_values, query_name=stored_query.name if stored_query else None, - stored_query=stored_query, - stored_query_write=stored_query_write, - error=query_error, metadata=await query_metadata(), extras=extras, extra_registry=table_extra_registry, diff --git a/datasette/views/row.py b/datasette/views/row.py index e15dfce9..c300758b 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -172,7 +172,6 @@ class RowView(DataView): row_extra_context = RowExtraContext( datasette=self.ds, request=request, - resolved=resolved, db=db, database_name=database, table_name=table, diff --git a/datasette/views/table.py b/datasette/views/table.py index 3cf8e6c6..65388c9c 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1497,7 +1497,6 @@ async def table_view_data( nofacet=nofacet, nosuggest=nosuggest, next_arg=request.args.get("_next"), - next_value=next_value, next_url=next_url, sql=sql, sql_no_order_no_limit=sql_no_order_no_limit, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index c98ae22c..493135f3 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -37,7 +37,6 @@ class TableExtraContext: nofacet: object nosuggest: object next_arg: object - next_value: object next_url: str | None sql: str sql_no_order_no_limit: str @@ -60,7 +59,6 @@ class TableExtraContext: class RowExtraContext: datasette: object request: object - resolved: object db: object database_name: str table_name: str @@ -90,9 +88,6 @@ class QueryExtraContext: sql: str | None params: dict query_name: str | None - stored_query: object - stored_query_write: bool - error: str | None metadata: dict extras: set extra_registry: ExtraRegistry From cfafa5b37f5350303600e912955c43e210a113b7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 22:56:40 -0700 Subject: [PATCH 1097/1116] Use plain set literals for Extra scopes frozenset({...}) was immutability ceremony for class attributes that nothing mutates. scopes = {ExtraScope.TABLE} reads cleaner. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 2 +- datasette/views/table_extras.py | 74 ++++++++++++++++----------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 4aa93057..fee92939 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -30,7 +30,7 @@ class ExtraExample: class Provider: name: ClassVar[str | None] = None - scopes: ClassVar[frozenset[ExtraScope]] = frozenset() + scopes: ClassVar[set[ExtraScope]] = set() public: ClassVar[bool] = False @classmethod diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 493135f3..ce1d7bdf 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -100,7 +100,7 @@ class QueryExtraContext: class CountSqlExtra(Extra): description = "SQL query used to calculate the total count" example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.count_sql @@ -109,7 +109,7 @@ class CountSqlExtra(Extra): class CountExtra(Extra): description = "Total count of rows matching these filters" example = ExtraExample("/fixtures/facetable.json?_extra=count") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context): @@ -141,7 +141,7 @@ class CountExtra(Extra): class FacetInstancesProvider(Provider): - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, count): facet_instances = [] @@ -182,7 +182,7 @@ class FacetResultsExtra(Extra): }, note="Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context, facet_instances): @@ -217,7 +217,7 @@ class FacetsTimedOutExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, facet_results): return facet_results["timed_out"] @@ -234,7 +234,7 @@ class SuggestedFacetsExtra(Extra): ], note="Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} expensive = True async def resolve(self, context, facet_instances): @@ -259,7 +259,7 @@ class HumanDescriptionEnExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?state=CA&_sort=pk&_extra=human_description_en" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): human_description_en = context.filters.human_description_en( @@ -279,7 +279,7 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.next_url @@ -296,7 +296,7 @@ class ColumnsExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=columns" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.columns @@ -305,7 +305,7 @@ class ColumnsExtra(Extra): class AllColumnsExtra(Extra): description = "All columns in the table, regardless of _col/_nocol filtering" example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return list(context.table_columns) @@ -319,7 +319,7 @@ class PrimaryKeysExtra(Extra): "/fixtures/simple_primary_key/1.json?_extra=primary_keys" ) } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): return context.pks @@ -327,7 +327,7 @@ class PrimaryKeysExtra(Extra): class ActionsExtra(Extra): description = "Table or view actions made available by plugin hooks" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Returns an async function for the HTML templates - not JSON serializable public = False @@ -358,7 +358,7 @@ class ActionsExtra(Extra): class IsViewExtra(Extra): description = "Whether this resource is a view instead of a table" example = ExtraExample("/fixtures/simple_view.json?_extra=is_view") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return context.is_view @@ -375,7 +375,7 @@ class DebugExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=debug" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): debug = { @@ -407,7 +407,7 @@ class RequestExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=request" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return { @@ -420,7 +420,7 @@ class RequestExtra(Extra): class DisplayColumnsAndRowsProvider(Provider): - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): display_columns, display_rows = await context.display_columns_and_rows( @@ -464,7 +464,7 @@ class DisplayColumnsExtra(Extra): ], note="Shape abbreviated from /fixtures/facetable.json?_size=1&_extra=display_columns.", ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, display_columns_and_rows): return display_columns_and_rows["columns"] @@ -472,7 +472,7 @@ class DisplayColumnsExtra(Extra): class DisplayRowsExtra(Extra): description = "Row data formatted for the HTML table display" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Contains markupsafe/sqlite3.Row values - not JSON serializable public = False @@ -524,7 +524,7 @@ class RenderCellExtra(Extra): ), ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): table_name = context.table_name @@ -587,7 +587,7 @@ class QueryExtra(Extra): ExtraExample("/fixtures/neighborhood_search.json?text=town&_extra=query"), ], } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return { @@ -599,7 +599,7 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" example = ExtraExample(value={}) - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): ct_map = await context.datasette.get_column_types( @@ -616,7 +616,7 @@ class ColumnTypesExtra(Extra): class SetColumnTypeUiExtra(Extra): description = "Column type UI metadata for this table" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): if context.is_view: @@ -676,7 +676,7 @@ class MetadataExtra(Extra): "/fixtures/neighborhood_search.json?text=town&_extra=metadata" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): if context.scope == ExtraScope.QUERY: @@ -713,7 +713,7 @@ class DatabaseExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=database" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.database_name @@ -725,7 +725,7 @@ class TableExtra(Extra): examples = { ExtraScope.ROW: ExtraExample("/fixtures/simple_primary_key/1.json?_extra=table") } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): return context.table_name @@ -742,7 +742,7 @@ class DatabaseColorExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=database_color" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.db.color @@ -753,7 +753,7 @@ class FormHiddenArgsExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): form_hidden_args = [] @@ -770,7 +770,7 @@ class FormHiddenArgsExtra(Extra): class FiltersExtra(Extra): description = "Filters object used by the HTML table interface" - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} # Returns a Filters instance for the HTML templates - not JSON serializable public = False @@ -781,7 +781,7 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return [ @@ -796,7 +796,7 @@ class SortedFacetResultsExtra(Extra): example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" ) - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, facet_results): facet_configs = context.table_metadata.get("facets", []) @@ -832,7 +832,7 @@ class SortedFacetResultsExtra(Extra): class TableDefinitionExtra(Extra): description = "SQL definition for this table" example = ExtraExample("/fixtures/facetable.json?_extra=table_definition") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return await context.db.get_table_definition(context.table_name) @@ -841,7 +841,7 @@ class TableDefinitionExtra(Extra): class ViewDefinitionExtra(Extra): description = "SQL definition for this view" example = ExtraExample("/fixtures/simple_view.json?_extra=view_definition") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): return await context.db.get_view_definition(context.table_name) @@ -850,7 +850,7 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" example = ExtraExample("/fixtures/facetable.json?_extra=renderers") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context, expandable_columns, query): renderers = {} @@ -896,7 +896,7 @@ class PrivateExtra(Extra): "/fixtures/-/query.json?sql=select+1+as+one&_extra=private" ), } - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): return context.private @@ -905,7 +905,7 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") - scopes = frozenset({ExtraScope.TABLE}) + scopes = {ExtraScope.TABLE} async def resolve(self, context): expandables = [] @@ -921,7 +921,7 @@ class ForeignKeyTablesExtra(Extra): example = ExtraExample( "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" ) - scopes = frozenset({ExtraScope.ROW}) + scopes = {ExtraScope.ROW} async def resolve(self, context): return await context.foreign_key_tables( @@ -931,7 +931,7 @@ class ForeignKeyTablesExtra(Extra): class ExtrasExtra(Extra): description = "Available ?_extra= blocks" - scopes = frozenset({ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}) + scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): all_extras = [ From 4edea3ad2637f4bf275f3a322e4c7747b964d907 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:04:12 -0700 Subject: [PATCH 1098/1116] Build extras registries once per scope instead of per request ExtraRegistry.resolve() previously constructed a fresh asyncinject Registry on every table, row and query request - instantiating all ~37 Extra classes and re-running inspect.signature reflection over each resolve method every time. The Extra classes are stateless, so the asyncinject Registry for each scope is now built lazily once and shared, along with the allowed-name sets. The per-request context reaches the shared registry through a contextvars.ContextVar provider rather than resolve_multi(results=...) seeding: asyncinject's parallel executor never schedules anything when the only initially-ready node is an unregistered pre-seeded value, so seeding would have stalled every resolution. asyncio tasks copy the caller's context, which keeps concurrent resolves isolated - covered by a new test. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 63 ++++++++++++++++++++++++++++++++---------- tests/test_extras.py | 65 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 15 deletions(-) create mode 100644 tests/test_extras.py diff --git a/datasette/extras.py b/datasette/extras.py index fee92939..2c3450b2 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,3 +1,4 @@ +import contextvars import re from dataclasses import dataclass from enum import Enum @@ -5,6 +6,11 @@ from typing import ClassVar from asyncinject import Registry +# Per-request context for Extra.resolve(), so the asyncinject registries can +# be shared across requests. asyncio tasks copy the caller's context, so +# concurrent resolve() calls each see their own value. +_resolve_context = contextvars.ContextVar("datasette_extras_context") + def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") @@ -62,6 +68,13 @@ class ExtraRegistry: def __init__(self, classes): self.classes = list(classes) self.classes_by_name = {cls.key(): cls for cls in self.classes} + # Lazily-built shared state, keyed by scope. Safe to share across + # requests because Extra instances are stateless and asyncinject's + # Registry keeps per-call state local to each resolve_multi() call. + # If extras classes ever become registerable at runtime (e.g. via a + # plugin hook) these caches will need invalidating. + self._scope_registries = {} + self._allowed_names = {} def classes_for_scope(self, scope, include_internal=True): classes = [ @@ -74,23 +87,43 @@ class ExtraRegistry: def public_classes_for_scope(self, scope): return self.classes_for_scope(scope, include_internal=False) + def _registry_for_scope(self, scope): + registry = self._scope_registries.get(scope) + if registry is None: + registry = Registry() + + async def context_provider(): + return _resolve_context.get() + + registry.register(context_provider, name="context") + for cls in self.classes_for_scope(scope): + registry.register(cls().resolve, name=cls.key()) + self._scope_registries[scope] = registry + return registry + + def _allowed_names_for_scope(self, scope, include_internal): + key = (scope, include_internal) + names = self._allowed_names.get(key) + if names is None: + names = { + cls.key() + for cls in self.classes_for_scope( + scope, include_internal=include_internal + ) + } + self._allowed_names[key] = names + return names + async def resolve(self, requested, context, scope, include_internal=False): - registry = Registry() - - async def context_provider(): - return context - - registry.register(context_provider, name="context") - - for cls in self.classes_for_scope(scope): - registry.register(cls().resolve, name=cls.key()) - - allowed_names = { - cls.key() - for cls in self.classes_for_scope(scope, include_internal=include_internal) - } + allowed_names = self._allowed_names_for_scope(scope, include_internal) requested_names = [name for name in requested if name in allowed_names] - resolved = await registry.resolve_multi(requested_names) + token = _resolve_context.set(context) + try: + resolved = await self._registry_for_scope(scope).resolve_multi( + requested_names + ) + finally: + _resolve_context.reset(token) return {name: resolved[name] for name in requested_names} diff --git a/tests/test_extras.py b/tests/test_extras.py new file mode 100644 index 00000000..ad8a9f00 --- /dev/null +++ b/tests/test_extras.py @@ -0,0 +1,65 @@ +import asyncio + +import pytest + +from datasette.extras import Extra, ExtraRegistry, ExtraScope + + +class SlowValueExtra(Extra): + description = "Returns context['value'], optionally slowly" + scopes = {ExtraScope.TABLE} + + async def resolve(self, context): + if context["slow"]: + await asyncio.sleep(0.05) + return context["value"] + + +class DependentExtra(Extra): + description = "Depends on slow_value" + scopes = {ExtraScope.TABLE} + + async def resolve(self, context, slow_value): + return slow_value + 1 + + +def test_registry_is_built_once_per_scope(): + registry = ExtraRegistry([SlowValueExtra, DependentExtra]) + first = registry._registry_for_scope(ExtraScope.TABLE) + second = registry._registry_for_scope(ExtraScope.TABLE) + assert first is second + + +@pytest.mark.asyncio +async def test_concurrent_resolves_do_not_share_state(): + # The asyncinject registry is shared across requests - resolved values + # must not leak between concurrent resolve() calls with different contexts + registry = ExtraRegistry([SlowValueExtra, DependentExtra]) + slow, fast = await asyncio.gather( + registry.resolve( + {"slow_value", "dependent"}, + {"value": 100, "slow": True}, + ExtraScope.TABLE, + ), + registry.resolve( + {"slow_value", "dependent"}, + {"value": 200, "slow": False}, + ExtraScope.TABLE, + ), + ) + assert slow == {"slow_value": 100, "dependent": 101} + assert fast == {"slow_value": 200, "dependent": 201} + + +@pytest.mark.asyncio +async def test_table_row_and_query_scopes_use_separate_registries(): + from datasette.views.table_extras import table_extra_registry + + registries = { + scope: table_extra_registry._registry_for_scope(scope) for scope in ExtraScope + } + assert len(set(map(id, registries.values()))) == 3 + # Scope-specific extras only registered where they belong + assert "count" in registries[ExtraScope.TABLE]._registry + assert "count" not in registries[ExtraScope.QUERY]._registry + assert "foreign_key_tables" in registries[ExtraScope.ROW]._registry From 96226621325c5aa19e6a700efcd1b441006958fa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:15:18 -0700 Subject: [PATCH 1099/1116] Fix SQL injection via bracket escape bypass in escape_sqlite() (#2677) escape_sqlite() wrapped identifiers in [brackets] without escaping any ] characters inside the string. Since SQLite does not support escaping ] within bracket quoting, an identifier containing ] could break out and inject arbitrary SQL. Fall back to double-quote quoting (doubling any embedded ") when the identifier contains ]. Co-Authored-By: Claude Fable 5 --- datasette/utils/__init__.py | 4 ++++ tests/test_utils.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 2dff9667..55e539b9 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -410,6 +410,10 @@ def escape_css_string(s): def escape_sqlite(s): if _boring_keyword_re.match(s) and (s.lower() not in reserved_words): return s + elif "]" in s: + # SQLite does not support escaping ] inside [bracket] quoting, so fall + # back to double-quote quoting (doubling any embedded ") - #2677 + return '"{}"'.format(s.replace('"', '""')) else: return f"[{s}]" diff --git a/tests/test_utils.py b/tests/test_utils.py index 64607244..74f1963f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -216,6 +216,38 @@ def test_detect_fts(open_quote, close_quote): conn.close() +@pytest.mark.parametrize( + "identifier,expected", + ( + ("plain", "plain"), + ("select", "[select]"), + ("has space", "[has space]"), + ("has'quote", "[has'quote]"), + # Identifiers containing ] must fall back to double-quote quoting + # (SQLite does not support escaping ] inside [brackets]) - #2677 + ("has]bracket", '"has]bracket"'), + ('has"dquote]', '"has""dquote]"'), + ), +) +def test_escape_sqlite(identifier, expected): + assert utils.escape_sqlite(identifier) == expected + + +def test_escape_sqlite_prevents_injection(): + # https://github.com/simonw/datasette/issues/2677 + conn = utils.sqlite3.connect(":memory:") + conn.execute("CREATE TABLE users (id INTEGER, password TEXT)") + conn.execute("INSERT INTO users VALUES (1, 'super_secret_password')") + malicious = "users] UNION SELECT password FROM users--" + conn.execute('CREATE TABLE "{}" (id INTEGER)'.format(malicious)) + sql = "select count(*) from {}".format(utils.escape_sqlite(malicious)) + results = conn.execute(sql).fetchall() + conn.close() + # The injected UNION must not execute - only the empty malicious table + # is queried, so we get a single count row and no leaked password + assert results == [(0,)] + + @pytest.mark.parametrize("table", ("regular", "has'single quote")) def test_detect_fts_different_table_names(table): sql = """ From 1c514d69f6cc09c820c119e7bbf4dc75235e90cc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:17:16 -0700 Subject: [PATCH 1100/1116] Prevent open redirect via backslash in path (#2680) asgi_send_redirect() only collapsed leading forward slashes, so a path like /\example.com/ produced a Location of /\example.com. Browsers normalise backslashes to forward slashes, turning that into the protocol-relative //example.com and redirecting off-site. Collapse any run of leading slashes and backslashes to a single slash. Co-Authored-By: Claude Fable 5 --- datasette/utils/asgi.py | 8 +++++--- tests/test_custom_pages.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 35f243b6..55eba1bb 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -330,9 +330,11 @@ async def asgi_send_html(send, html, status=200, headers=None): async def asgi_send_redirect(send, location, status=302): - # Prevent open redirect vulnerability: strip multiple leading slashes - # //example.com would be interpreted as a protocol-relative URL (e.g., https://example.com/) - location = re.sub(r"^/+", "/", location) + # Prevent open redirect vulnerability: collapse leading slashes and + # backslashes down to a single slash. //example.com is a protocol-relative + # URL, and browsers normalise backslashes to slashes so /\example.com would + # be treated as //example.com - https://github.com/simonw/datasette/issues/2680 + location = re.sub(r"^[/\\]+", "/", location) await asgi_send( send, "", diff --git a/tests/test_custom_pages.py b/tests/test_custom_pages.py index 39a4c06b..86cdcc6b 100644 --- a/tests/test_custom_pages.py +++ b/tests/test_custom_pages.py @@ -104,3 +104,24 @@ def test_custom_route_pattern_with_slash_slash_302(custom_pages_client): response = custom_pages_client.get("//example.com/") assert response.status == 302 assert response.headers["location"] == "/example.com" + + +@pytest.mark.parametrize( + "path", + ( + "/\\example.com/", + "/\\\\example.com/", + "/\\/example.com/", + ), +) +def test_redirect_does_not_allow_backslash_open_redirect(custom_pages_client, path): + # https://github.com/simonw/datasette/issues/2680 + # Browsers normalise backslashes to forward slashes, so a Location of + # /\example.com would be treated as the protocol-relative //example.com + response = custom_pages_client.get(path) + assert response.status == 302 + location = response.headers["location"] + assert location == "/example.com" + # Must not start with anything a browser reads as protocol-relative + assert not location.startswith("//") + assert not location.startswith("/\\") From c31bb55011567d13f39d8096da4aef5b5a8a720a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:24:21 -0700 Subject: [PATCH 1101/1116] Add regression test for --default-deny index 500 (#2644) datasette --default-deny --root with no config file previously 500'd on the instance and database index pages: rendering them computes is_private (include_is_private=True), which references the anon_rules CTE, but that CTE was only defined when anonymous permission rules existed. This was fixed by the empty-anon_rules fallback added in 4b5fac9c; this commit adds a regression test that fails without that fallback (SQLite "no such table: anon_rules" -> 500). Co-Authored-By: Claude Fable 5 --- tests/test_default_deny.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_default_deny.py b/tests/test_default_deny.py index 81e95b84..f1e43064 100644 --- a/tests/test_default_deny.py +++ b/tests/test_default_deny.py @@ -127,3 +127,23 @@ async def test_default_deny_basic_permissions(): # Authenticated user without explicit permission should also be denied assert await ds.allowed(action="view-instance", actor={"id": "user"}) is False + + +@pytest.mark.asyncio +async def test_default_deny_root_no_config_index_does_not_500(): + # https://github.com/simonw/datasette/issues/2644 + # --default-deny --root with no config file must not 500 on the index + # pages. Rendering those pages computes is_private (include_is_private), + # which references the anon_rules CTE - that CTE must still be defined + # even when there are no anonymous permission rules at all. + ds = Datasette(default_deny=True) + ds.root_enabled = True + await ds.invoke_startup() + db = ds.add_memory_database("test_db_2644") + await db.execute_write("create table test_table (id integer primary key)") + await ds._refresh_schemas() + + cookie = ds.sign({"a": {"id": "root"}}, "actor") + for path in ("/", "/test_db_2644", "/test_db_2644/test_table"): + response = await ds.client.get(path, cookies={"ds_actor": cookie}) + assert response.status_code == 200, f"{path} returned {response.status_code}" From d5141a5778ac5ce6a6f4cfda990b2ab556b7f9f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:27:13 -0700 Subject: [PATCH 1102/1116] Fix /-/check 500 for query actions (#2756) _check_permission_for_actor() constructed child resources with resource_class(database=parent, table=child), but QueryResource takes a "query" argument, not "table", so /-/check?action=delete-query (and view-query / update-query) raised TypeError. Construct the resource positionally so it works for any child resource class. Co-Authored-By: Claude Fable 5 --- datasette/views/special.py | 8 +++++--- tests/test_permissions.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/datasette/views/special.py b/datasette/views/special.py index 75c54c3c..aa063ad6 100644 --- a/datasette/views/special.py +++ b/datasette/views/special.py @@ -497,11 +497,13 @@ async def _check_permission_for_actor(ds, action, parent, child, actor): if action_obj.resource_class is None: resource_obj = None elif action_obj.takes_parent and action_obj.takes_child: - # Child-level resource (e.g., TableResource, QueryResource) - resource_obj = action_obj.resource_class(database=parent, table=child) + # Child-level resource (e.g., TableResource, QueryResource). The child + # argument is named differently per resource class (table, query, ...), + # so pass positionally - https://github.com/simonw/datasette/issues/2756 + resource_obj = action_obj.resource_class(parent, child) elif action_obj.takes_parent: # Parent-level resource (e.g., DatabaseResource) - resource_obj = action_obj.resource_class(database=parent) + resource_obj = action_obj.resource_class(parent) else: # This shouldn't happen given validation in Action.__post_init__ return {"error": f"Invalid action configuration: {action}"}, 500 diff --git a/tests/test_permissions.py b/tests/test_permissions.py index e5e75432..8323fe92 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1733,6 +1733,29 @@ async def test_permission_check_view_requires_debug_permission(): assert data["allowed"] is True +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ("view-query", "update-query", "delete-query")) +async def test_permission_check_view_query_actions(action): + # https://github.com/simonw/datasette/issues/2756 + # QueryResource takes a "query" argument, not "table", so /-/check must + # not assume every child resource class accepts table= + ds = Datasette() + ds.root_enabled = True + root_token = await ds.create_token("root", handler="signed") + response = await ds.client.get( + f"/-/check.json?action={action}&parent=mydb&child=myquery", + headers={"Authorization": f"Bearer {root_token}"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["action"] == action + assert data["resource"] == { + "parent": "mydb", + "child": "myquery", + "path": "/mydb/myquery", + } + + @pytest.mark.asyncio async def test_root_allow_block_with_table_restricted_actor(): """ From 154ea483eaba7a636289a6972baeced7163acd60 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:30:12 -0700 Subject: [PATCH 1103/1116] Pass columns and rows to can_render for canned queries (#2711) The HTML branch of QueryView built an empty data dict before looping over register_output_renderer can_render callbacks, so renderers that depend on the result columns or rows (e.g. datasette-atom, datasette-ics) never appeared as export options for canned queries. Populate data with the executed query's rows, columns, SQL and query name. Co-Authored-By: Claude Fable 5 --- datasette/views/database.py | 10 +++++++++- tests/test_plugins.py | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/datasette/views/database.py b/datasette/views/database.py index a1647ca9..66887f9b 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -762,7 +762,15 @@ class QueryView(View): ) ), ) - data = {} + data = { + "ok": query_error is None, + "rows": rows, + "columns": columns, + "query": {"sql": sql, "params": params}, + "query_name": stored_query.name if stored_query else None, + "database": database, + "table": None, + } headers.update( { "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format( diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 32276437..cf753c9e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -626,6 +626,31 @@ async def test_hook_register_output_renderer_can_render(ds_client): }.items() <= ds_client.ds._can_render_saw.items() +@pytest.mark.asyncio +async def test_hook_register_output_renderer_can_render_canned_query(ds_client): + # https://github.com/simonw/datasette/issues/2711 + # can_render for a canned query must be passed the query's columns, rows + # and SQL - previously it received an empty data dict, so renderers that + # depend on the columns (datasette-atom, datasette-ics) never showed up. + response = await ds_client.get("/fixtures/pragma_cache_size") + assert response.status_code == 200 + saw = ds_client.ds._can_render_saw + assert saw["columns"] == ["cache_size"] + assert len(saw["rows"]) == 1 + assert saw["sql"] == "PRAGMA cache_size;" + assert saw["query_name"] == "pragma_cache_size" + # The renderer's export link should therefore be offered + links = ( + Soup(response.text, "html.parser") + .find("p", {"class": "export-links"}) + .find_all("a") + ) + actual = [link["href"] for link in links] + assert any( + href.startswith("/fixtures/pragma_cache_size.testall") for href in actual + ) + + @pytest.mark.asyncio async def test_hook_prepare_jinja2_environment(ds_client): ds_client.ds._HELLO = "HI" From 92848c06b8cf5b8bb6b93088bb547810f9fce8b6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 10 Jun 2026 23:43:32 -0700 Subject: [PATCH 1104/1116] Stop facet counts from wrapping (#2754) ul.tight-bullets li uses word-break: break-all so long facet labels can wrap, but that also let the count number break across lines. Wrap each count in a span.facet-count with white-space: nowrap so the label can still wrap while the count stays on one line. Co-Authored-By: Claude Fable 5 --- datasette/static/app.css | 5 +++++ datasette/templates/_facet_results.html | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 815f6db8..6d675d9f 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -706,6 +706,11 @@ button.core[type=button] { color: #666; padding-right: 0.25em; } +/* The label may wrap (word-break: break-all on the li) but the count should + stay on one line - https://github.com/simonw/datasette/issues/2754 */ +.facet-count { + white-space: nowrap; +} .facet-info li, .facet-info ul { margin: 0; diff --git a/datasette/templates/_facet_results.html b/datasette/templates/_facet_results.html index 034e9678..570bb37e 100644 --- a/datasette/templates/_facet_results.html +++ b/datasette/templates/_facet_results.html @@ -12,9 +12,9 @@
    {% for facet_value in facet_info.results %} {% if not facet_value.selected %} -
  • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
    • +
  • {{ (facet_value.label | string()) or "-" }} {{ "{:,}".format(facet_value.count) }}
    • {% else %} -
  • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
    • +
  • {{ facet_value.label or "-" }} · {{ "{:,}".format(facet_value.count) }} ✖
    • {% endif %} {% endfor %} {% if facet_info.truncated %} From 9adb5416743a7312758e986c254baa1758228ad0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 06:42:08 -0700 Subject: [PATCH 1105/1116] Use asyncinject 0.7 results= seeding for per-request extras context asyncinject 0.7 fixed the parallel executor stalling when every initially-ready node is a seeded value, and made seeded values take precedence over registered functions. That lets the shared per-scope registries receive the per-request context directly via resolve_multi(results={'context': ...}) instead of the contextvars.ContextVar workaround. Co-Authored-By: Claude Fable 5 --- datasette/extras.py | 21 +++------------------ pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/datasette/extras.py b/datasette/extras.py index 2c3450b2..5cab52a4 100644 --- a/datasette/extras.py +++ b/datasette/extras.py @@ -1,4 +1,3 @@ -import contextvars import re from dataclasses import dataclass from enum import Enum @@ -6,11 +5,6 @@ from typing import ClassVar from asyncinject import Registry -# Per-request context for Extra.resolve(), so the asyncinject registries can -# be shared across requests. asyncio tasks copy the caller's context, so -# concurrent resolve() calls each see their own value. -_resolve_context = contextvars.ContextVar("datasette_extras_context") - def extra_names_from_request(request): extra_bits = request.args.getlist("_extra") @@ -91,11 +85,6 @@ class ExtraRegistry: registry = self._scope_registries.get(scope) if registry is None: registry = Registry() - - async def context_provider(): - return _resolve_context.get() - - registry.register(context_provider, name="context") for cls in self.classes_for_scope(scope): registry.register(cls().resolve, name=cls.key()) self._scope_registries[scope] = registry @@ -117,13 +106,9 @@ class ExtraRegistry: async def resolve(self, requested, context, scope, include_internal=False): allowed_names = self._allowed_names_for_scope(scope, include_internal) requested_names = [name for name in requested if name in allowed_names] - token = _resolve_context.set(context) - try: - resolved = await self._registry_for_scope(scope).resolve_multi( - requested_names - ) - finally: - _resolve_context.reset(token) + resolved = await self._registry_for_scope(scope).resolve_multi( + requested_names, results={"context": context} + ) return {name: resolved[name] for name in requested_names} diff --git a/pyproject.toml b/pyproject.toml index 38085476..0d136d60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "mergedeep>=1.1.1", "itsdangerous>=1.1", "sqlite-utils>=3.30", - "asyncinject>=0.6.1", + "asyncinject>=0.7", "setuptools", "pip", ] From 648a34ce8196ecf02504c0daed594bd1cd540210 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:13:07 -0700 Subject: [PATCH 1106/1116] Fix for test I broke in 92848c06 refs #2754 --- tests/test_table_html.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index 2e671d55..63e233fa 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -349,7 +349,11 @@ async def test_facet_display(ds_client): { "name": a.text, "qs": a["href"].split("?")[-1], - "count": int(str(a.parent).split("")[1].split("<")[0]), + "count": int( + a.parent.find( + "span", {"class": "facet-count"} + ).text.replace(",", "") + ), } for a in div.find("ul").find_all("a") ], @@ -695,7 +699,7 @@ async def test_table_html_foreign_key_facets(ds_client): assert response.status_code == 200 assert ( '
  • - 1
    • ' + ' data-facet-value="3">- 1' ) in response.text From 26f3b20e58bd2ad582d5fae326acf22b42627eb1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 07:29:27 -0700 Subject: [PATCH 1107/1116] Fix to our pytest plugin to better support pytest-cov Refs https://github.com/simonw/datasette/pulls#issuecomment-4681621052 --- datasette/_pytest_plugin.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 5fb6b473..103c616d 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -19,23 +19,38 @@ import weakref import pytest -from datasette.app import Datasette - _active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( "datasette_active_instances", default=None ) -_original_init = Datasette.__init__ +_original_init = None -def _tracking_init(self, *args, **kwargs): - _original_init(self, *args, **kwargs) - instances = _active_instances.get() - if instances is not None: - instances.append(weakref.ref(self)) +def _install_tracking(): + # datasette.app is imported lazily here rather than at module level: + # as a pytest11 entry point this module is imported during pytest + # startup, before pytest-cov starts measuring, so a module-level + # import would drag in all of datasette and make every import-time + # line in the package invisible to coverage + global _original_init + if _original_init is not None: + return + from datasette.app import Datasette + + _original_init = Datasette.__init__ + + def _tracking_init(self, *args, **kwargs): + _original_init(self, *args, **kwargs) + instances = _active_instances.get() + if instances is not None: + instances.append(weakref.ref(self)) + + Datasette.__init__ = _tracking_init -Datasette.__init__ = _tracking_init +def pytest_configure(config): + if _enabled(config): + _install_tracking() def pytest_addoption(parser): From 993169ae496aa0fa30271b6cb4dfc50202f6e7c1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 08:24:37 -0700 Subject: [PATCH 1108/1116] Release 1.0a33 Refs #2735, #2677, #2680, #2711, #2756, #2761, #2768, #2754 --- datasette/version.py | 2 +- docs/changelog.rst | 40 ++++++++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/datasette/version.py b/datasette/version.py index 1e8c61d5..9536d459 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a32" +__version__ = "1.0a33" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19089dd1..48bef0bf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,15 +9,43 @@ Changelog 1.0a33 (unreleased) ------------------- -- Stored queries can now be edited and deleted from the web interface. The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) -- Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. See :ref:`json_api_extra` for the full list. -- New generated reference documentation for every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra`. -- ``?_extra=`` values can be separated by commas as well as repeated, e.g. ``?_extra=count,next_url``. Previously a comma-separated value that included ``columns`` failed to include the ``columns`` key in the response. -- The ``?_extra=private`` extra on arbitrary SQL query pages now correctly reflects whether the SQL execution permission is private to the current actor - it previously always returned ``false``. -- The ``?_extra=query`` extra on query pages now reports the named parameters that were actually bound when the query executed, including parameters declared in a stored query's ``params`` list. Magic ``_``-prefixed parameters are no longer echoed back with unbound values taken from the querystring. +Stored queries can now be edited and deleted through the web interface, and the JSON API ``?_extra=`` mechanism has been extended to cover row and query pages in addition to tables. This release also fixes two security issues: an identifier-quoting bug involving table and column names that contain ``]``, and an open redirect. + +Editing and deleting stored queries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The stored query page gained a "Query actions" menu with **Edit this query** and **Delete this query** links for actors with the necessary permissions. The owner of a query can always edit or delete it; for queries that are not private, any actor with the :ref:`update-query ` or :ref:`delete-query ` permission can do so too. Private queries remain editable and deletable only by their owner. See :ref:`stored_queries` for details. (:issue:`2735`) + +``?_extra=`` support for row and query pages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Row and query JSON pages now support the same ``?_extra=`` mechanism as table pages. Row pages can request extras such as ``foreign_key_tables``, ``query``, ``metadata`` and ``database_color``; arbitrary SQL and stored query pages can request extras such as ``columns``, ``query``, ``metadata`` and ``private``. The implementation was refactored into a registry of extra classes shared by all three page types. + +New generated reference documentation describes every ``?_extra=`` parameter available on table, row and query JSON pages, with example output captured from a live Datasette instance at documentation build time. See :ref:`json_api_extra` for the full list. + +You can explore the new extras using this `Datasette extras API explorer tool `__. + +Other improvements and fixes to the extras mechanism: + - Extras that exist to serve the HTML interface (``filters``, ``actions``, ``display_rows``) are no longer advertised or reachable through the JSON API, where requesting them previously returned a 500 serialization error. - The pre-1.0 ``?_extras=`` (plural) parameter on row pages has been removed - use ``?_extra=foreign_key_tables`` instead. +Security fixes +~~~~~~~~~~~~~~ + +- Fixed an identifier-quoting bug in ``datasette.utils.escape_sqlite()``. Datasette uses this helper when constructing SQL around table and column names; identifiers containing ``]`` could break out of SQLite bracket quoting and alter the generated SQL, for example by adding a ``UNION SELECT``. Identifiers containing ``]`` are now quoted using double quotes instead. (:issue:`2677`) +- Fixed an open redirect vulnerability. Requesting a path such as ``/\example.com/`` produced a redirect with a ``Location: /\example.com`` header - browsers normalize backslashes to forward slashes, turning that into the protocol-relative URL ``//example.com`` and redirecting the user off-site. Any run of leading slashes and backslashes in a redirect path is now collapsed to a single slash. (:issue:`2680`) + +Bug fixes +~~~~~~~~~ + +- ``can_render()`` callbacks registered by the :ref:`register_output_renderer() ` plugin hook now receive the result ``rows`` and ``columns`` for stored queries. Previously renderers that inspect the available columns - such as `datasette-atom `__ and `datasette-ics `__ - never appeared as export options on stored query pages. (:issue:`2711`) +- Fixed a 500 error from the :ref:`/-/check ` permission debugging endpoint when checking query actions such as ``view-query``, ``update-query`` and ``delete-query``. (:issue:`2756`) +- Write queries that use a named parameter called ``:sql`` no longer fail with an error. (:issue:`2761`) +- :ref:`db.execute_isolated_fn() ` now works against immutable databases, using a read-only connection that bypasses the write thread. It previously always attempted to open a writable connection, which would fail - breaking features built on top of it, such as the SQL analysis step used when storing a query. An exception raised while opening the connection for an isolated function no longer crashes the write thread. (:issue:`2768`) +- Facet counts are now displayed on the same line as the facet value instead of wrapping onto a second line. (:issue:`2754`) +- Datasette's pytest plugin no longer imports the rest of Datasette at pytest startup time. This means plugin test suites using ``pytest-cov`` now correctly record coverage of code that runs when ``datasette`` modules are first imported. + .. _v1_0_a32: 1.0a32 (2026-05-31) From 1d4212122e5597f2e13625193fb7d45b25928447 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 10:36:16 -0700 Subject: [PATCH 1109/1116] Add release date for 1.0a33 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 48bef0bf..c0bd7e6b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changelog .. _v1_0_a33: -1.0a33 (unreleased) +1.0a33 (2026-06-11) ------------------- Stored queries can now be edited and deleted through the web interface, and the JSON API ``?_extra=`` mechanism has been extended to cover row and query pages in addition to tables. This release also fixes two security issues: an identifier-quoting bug involving table and column names that contain ``]``, and an open redirect. From fa86ac7b11c44ef80146db6eed25d88c954ee37a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 11 Jun 2026 19:41:24 -0700 Subject: [PATCH 1110/1116] Clearer examples and descriptions for JSON API extras (#2773) Review of the generated ?_extra= documentation found several extras with no example output or with examples that needed explanation: - extras: now shows an abbreviated example of the toggle list and has a clearer description (which also improves the live API output) - set_column_type_ui: example of the shape seen with set-column-type permission, plus a note that it is null otherwise - column_types: live example generated from a table with an assigned column type instead of an empty {} - metadata: live table example now demonstrates a table description and column descriptions; row and query examples gained explanatory notes - expandable_columns, foreign_key_tables, facets_timed_out, next_url, renderers: notes explaining the shape of their output Also added docs_note cross-references to the relevant documentation: facets, pagination, render_cell and register_output_renderer plugin hooks, column type configuration and API, metadata, custom templates, permissions and foreign key label expansion. foreign_key_tables is now flagged as potentially executing additional queries. https://claude.ai/code/session_01EfjBe6E817m9XNFW7EX3Vm Co-authored-by: Claude --- datasette/views/table_extras.py | 182 +++++++++++++++++++++++++-- docs/json_api.rst | 215 ++++++++++++++++++++++++++------ docs/json_api_doc.py | 19 ++- 3 files changed, 367 insertions(+), 49 deletions(-) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index ce1d7bdf..948f3daa 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -184,6 +184,7 @@ class FacetResultsExtra(Extra): ) scopes = {ExtraScope.TABLE} expensive = True + docs_note = "See :ref:`facets` for details of how facets work." async def resolve(self, context, facet_instances): facet_results = {} @@ -215,7 +216,12 @@ class FacetResultsExtra(Extra): class FacetsTimedOutExtra(Extra): description = "Facet calculations that timed out" example = ExtraExample( - "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out" + "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out", + note=( + "A list of the names of any facets that exceeded the " + ":ref:`setting_facet_time_limit_ms` time limit - an empty list " + "if every facet calculation completed." + ), ) scopes = {ExtraScope.TABLE} @@ -236,6 +242,9 @@ class SuggestedFacetsExtra(Extra): ) scopes = {ExtraScope.TABLE} expensive = True + docs_note = ( + "Suggestions are controlled by the :ref:`setting_suggest_facets` setting." + ) async def resolve(self, context, facet_instances): suggested_facets = [] @@ -278,7 +287,13 @@ class HumanDescriptionEnExtra(Extra): class NextUrlExtra(Extra): description = "Full URL for the next page of results" - example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url") + example = ExtraExample( + "/fixtures/facetable.json?_size=1&_extra=next_url", + note=( + "``null`` if there are no more pages of results. " + "See :ref:`json_api_pagination`." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -366,6 +381,10 @@ class IsViewExtra(Extra): class DebugExtra(Extra): description = "Extra debug information" + docs_note = ( + "The contents of this block are not a stable part of the Datasette " + "API and may change without warning." + ) example = ExtraExample("/fixtures/facetable.json?_extra=debug") examples = { ExtraScope.ROW: ExtraExample( @@ -482,6 +501,10 @@ class DisplayRowsExtra(Extra): class RenderCellExtra(Extra): description = "Rendered HTML for each cell using the render_cell plugin hook" + docs_note = ( + "See the :ref:`render_cell() plugin hook ` " + "documentation." + ) example = ExtraExample( value={ "rows": [ @@ -598,7 +621,28 @@ class QueryExtra(Extra): class ColumnTypesExtra(Extra): description = "Column type assignments for this table" - example = ExtraExample(value={}) + docs_note = ( + "An empty object if no column types have been assigned. Column types " + "can be assigned in :ref:`configuration " + "` or using the :ref:`set column " + "type API `." + ) + example = ExtraExample( + "/fixtures/facetable.json?_size=0&_extra=column_types", + note=( + "This example is from an instance where the ``tags`` column has " + "been assigned the ``json`` column type." + ), + ) + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/facetable/1.json?_extra=column_types", + note=( + "This example is from an instance where the ``tags`` column " + "has been assigned the ``json`` column type." + ), + ) + } scopes = {ExtraScope.TABLE, ExtraScope.ROW} async def resolve(self, context): @@ -615,7 +659,40 @@ class ColumnTypesExtra(Extra): class SetColumnTypeUiExtra(Extra): - description = "Column type UI metadata for this table" + description = "Information needed to build an interface for assigning column types" + docs_note = ( + "``null`` unless the current actor is allowed to use the :ref:`set " + "column type API ` for this table." + ) + example = ExtraExample( + value={ + "path": "/fixtures/facetable/-/set-column-type", + "columns": { + "created": { + "current": None, + "options": [ + {"name": "email", "description": "Email address"}, + {"name": "json", "description": "JSON data"}, + {"name": "url", "description": "URL"}, + ], + }, + "tags": { + "current": {"type": "json", "config": None}, + "options": [ + {"name": "email", "description": "Email address"}, + {"name": "json", "description": "JSON data"}, + {"name": "url", "description": "URL"}, + ], + }, + }, + }, + note=( + "Shape abbreviated to two columns, as seen by an actor with " + "``set-column-type`` permission. ``current`` is the column type " + "currently assigned to each column and ``options`` lists the " + "types that could be assigned to it." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -667,13 +744,33 @@ class SetColumnTypeUiExtra(Extra): class MetadataExtra(Extra): description = "Metadata about the table, database or stored query" - example = ExtraExample("/fixtures/facetable.json?_extra=metadata") + docs_note = "See :ref:`metadata` for how to attach metadata to tables." + example = ExtraExample( + "/fixtures/facetable.json?_extra=metadata", + note=( + "This example is from an instance where the ``facetable`` table " + "has a metadata ``description`` and a :ref:`column description " + "` for its ``state`` column. The " + "``columns`` object is empty for tables with no column " + "descriptions." + ), + ) examples = { ExtraScope.ROW: ExtraExample( - "/fixtures/simple_primary_key/1.json?_extra=metadata" + "/fixtures/simple_primary_key/1.json?_extra=metadata", + note=( + "This table has no metadata, so only an empty ``columns`` " + "object is returned." + ), ), ExtraScope.QUERY: ExtraExample( - "/fixtures/neighborhood_search.json?text=town&_extra=metadata" + "/fixtures/neighborhood_search.json?text=town&_extra=metadata", + note=( + "For stored queries this returns the full configuration of " + "the query, including the :ref:`stored query options " + "`. For ``?sql=`` queries it returns an " + "empty object." + ), ), } scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} @@ -733,6 +830,10 @@ class TableExtra(Extra): class DatabaseColorExtra(Extra): description = "Color assigned to the database" + docs_note = ( + "A six character hex color, without the leading ``#``, derived from " + "a hash of the database name and used in the Datasette interface." + ) example = ExtraExample("/fixtures/facetable.json?_extra=database_color") examples = { ExtraScope.ROW: ExtraExample( @@ -780,6 +881,11 @@ class FiltersExtra(Extra): class CustomTableTemplatesExtra(Extra): description = "Custom template names considered for this table" + docs_note = ( + "The first template in this list that exists will be used to render " + "the table on the HTML version of this page. See " + ":ref:`customization_custom_templates`." + ) example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates") scopes = {ExtraScope.TABLE} @@ -793,6 +899,12 @@ class CustomTableTemplatesExtra(Extra): class SortedFacetResultsExtra(Extra): description = "Facet results sorted for display" + docs_note = ( + "The same data as ``facet_results``, as a list in the order used by " + "the HTML interface: facets from :ref:`facet configuration " + "` first, then other facets ordered by their number " + "of results." + ) example = ExtraExample( "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results" ) @@ -849,7 +961,15 @@ class ViewDefinitionExtra(Extra): class RenderersExtra(Extra): description = "Alternative output renderers available for this table" - example = ExtraExample("/fixtures/facetable.json?_extra=renderers") + example = ExtraExample( + "/fixtures/facetable.json?_extra=renderers", + note=( + "Each key is the name of an output format, each value the URL " + "for this data in that format. Plugins can add additional " + "formats using the :ref:`register_output_renderer() plugin hook " + "`." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context, expandable_columns, query): @@ -887,6 +1007,10 @@ class RenderersExtra(Extra): class PrivateExtra(Extra): description = "Whether this resource is private to the current actor" + docs_note = ( + "``true`` if the current actor can see this resource but an " + "anonymous user could not. See :ref:`authentication_permissions`." + ) example = ExtraExample("/fixtures/facetable.json?_extra=private") examples = { ExtraScope.ROW: ExtraExample( @@ -904,7 +1028,15 @@ class PrivateExtra(Extra): class ExpandableColumnsExtra(Extra): description = "Foreign key columns that can be expanded with labels" - example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns") + docs_note = "See :ref:`expand_foreign_keys` for how to expand these labels." + example = ExtraExample( + "/fixtures/facetable.json?_extra=expandable_columns", + note=( + "Each item is a ``[foreign_key, label_column]`` pair: the " + "foreign key relationship, then the column in the other table " + "that would be used as the label for each expanded value." + ), + ) scopes = {ExtraScope.TABLE} async def resolve(self, context): @@ -919,9 +1051,14 @@ class ExpandableColumnsExtra(Extra): class ForeignKeyTablesExtra(Extra): description = "Tables that link to this row using foreign keys" example = ExtraExample( - "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables", + note=( + "``count`` is the number of rows in the other table that " + "reference this row, and ``link`` is a URL to browse those rows." + ), ) scopes = {ExtraScope.ROW} + expensive = True async def resolve(self, context): return await context.foreign_key_tables( @@ -930,7 +1067,30 @@ class ForeignKeyTablesExtra(Extra): class ExtrasExtra(Extra): - description = "Available ?_extra= blocks" + description = "List of ?_extra= blocks that can be used on this page" + example = ExtraExample( + value=[ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": False, + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": True, + }, + ], + note=( + "Shape abbreviated from /fixtures/facetable.json?_extra=extras - " + "the full response lists every extra described on this page. " + "``toggle_url`` is the current URL with that extra added or " + "removed, and ``selected`` is ``true`` for extras included in " + "the current request." + ), + ) scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY} async def resolve(self, context): diff --git a/docs/json_api.rst b/docs/json_api.rst index 6b595577..fbc3cf60 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -276,7 +276,7 @@ The available table extras are listed below. "select count(*) from facetable " ``facet_results`` - Results of facets calculated against this data (May execute additional queries.) + Results of facets calculated against this data (May execute additional queries. See :ref:`facets` for details of how facets work.) Shape abbreviated from /fixtures/facetable.json?_facet=state&_extra=facet_results. @@ -309,12 +309,14 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_facet=state&_extra=facets_timed_out`` + A list of the names of any facets that exceeded the :ref:`setting_facet_time_limit_ms` time limit - an empty list if every facet calculation completed. + .. code-block:: json [] ``suggested_facets`` - Suggestions for facets that might return interesting results (May execute additional queries.) + Suggestions for facets that might return interesting results (May execute additional queries. Suggestions are controlled by the :ref:`setting_suggest_facets` setting.) Shape abbreviated from /fixtures/facetable.json?_extra=suggested_facets. @@ -341,6 +343,8 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_size=1&_extra=next_url`` + ``null`` if there are no more pages of results. See :ref:`json_api_pagination`. + .. code-block:: json "http://localhost/fixtures/facetable.json?_size=1&_extra=next_url&_next=1" @@ -426,7 +430,7 @@ The available table extras are listed below. ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item per row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -452,7 +456,7 @@ The available table extras are listed below. } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/facetable.json?_extra=debug`` @@ -501,28 +505,108 @@ The available table extras are listed below. } ``column_types`` - Column type assignments for this table + Column type assignments for this table (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) - .. code-block:: json + ``GET /fixtures/facetable.json?_size=0&_extra=column_types`` - {} - -``set_column_type_ui`` - Column type UI metadata for this table - -``metadata`` - Metadata about the table, database or stored query - - ``GET /fixtures/facetable.json?_extra=metadata`` + This example is from an instance where the ``tags`` column has been assigned the ``json`` column type. .. code-block:: json { - "columns": {} + "tags": { + "type": "json", + "config": null + } + } + +``set_column_type_ui`` + Information needed to build an interface for assigning column types (``null`` unless the current actor is allowed to use the :ref:`set column type API ` for this table.) + + Shape abbreviated to two columns, as seen by an actor with ``set-column-type`` permission. ``current`` is the column type currently assigned to each column and ``options`` lists the types that could be assigned to it. + + .. code-block:: json + + { + "path": "/fixtures/facetable/-/set-column-type", + "columns": { + "created": { + "current": null, + "options": [ + { + "name": "email", + "description": "Email address" + }, + { + "name": "json", + "description": "JSON data" + }, + { + "name": "url", + "description": "URL" + } + ] + }, + "tags": { + "current": { + "type": "json", + "config": null + }, + "options": [ + { + "name": "email", + "description": "Email address" + }, + { + "name": "json", + "description": "JSON data" + }, + { + "name": "url", + "description": "URL" + } + ] + } + } + } + +``metadata`` + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) + + ``GET /fixtures/facetable.json?_extra=metadata`` + + This example is from an instance where the ``facetable`` table has a metadata ``description`` and a :ref:`column description ` for its ``state`` column. The ``columns`` object is empty for tables with no column descriptions. + + .. code-block:: json + + { + "description": "A demo table of places, used to demonstrate facets", + "columns": { + "state": "Two letter US state code" + } } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -543,7 +627,7 @@ The available table extras are listed below. "facetable" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/facetable.json?_extra=database_color`` @@ -556,6 +640,8 @@ The available table extras are listed below. ``GET /fixtures/facetable.json?_extra=renderers`` + Each key is the name of an output format, each value the URL for this data in that format. Plugins can add additional formats using the :ref:`register_output_renderer() plugin hook `. + .. code-block:: json { @@ -563,7 +649,7 @@ The available table extras are listed below. } ``custom_table_templates`` - Custom template names considered for this table + Custom template names considered for this table (The first template in this list that exists will be used to render the table on the HTML version of this page. See :ref:`customization_custom_templates`.) ``GET /fixtures/facetable.json?_extra=custom_table_templates`` @@ -576,7 +662,7 @@ The available table extras are listed below. ] ``sorted_facet_results`` - Facet results sorted for display + Facet results sorted for display (The same data as ``facet_results``, as a list in the order used by the HTML interface: facets from :ref:`facet configuration ` first, then other facets ordered by their number of results.) ``GET /fixtures/facetable.json?_facet=state&_extra=sorted_facet_results`` @@ -643,7 +729,7 @@ The available table extras are listed below. true ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/facetable.json?_extra=private`` @@ -652,10 +738,12 @@ The available table extras are listed below. false ``expandable_columns`` - Foreign key columns that can be expanded with labels + Foreign key columns that can be expanded with labels (See :ref:`expand_foreign_keys` for how to expand these labels.) ``GET /fixtures/facetable.json?_extra=expandable_columns`` + Each item is a ``[foreign_key, label_column]`` pair: the foreign key relationship, then the column in the other table that would be used as the label for each expanded value. + .. code-block:: json [ @@ -720,7 +808,7 @@ The following extras are available for row JSON responses. ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item for the requested row. The object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -741,7 +829,7 @@ The following extras are available for row JSON responses. } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/simple_primary_key/1.json?_extra=debug`` @@ -803,17 +891,28 @@ The following extras are available for row JSON responses. } ``column_types`` - Column type assignments for this table + Column type assignments for this table (An empty object if no column types have been assigned. Column types can be assigned in :ref:`configuration ` or using the :ref:`set column type API `.) + + ``GET /fixtures/facetable/1.json?_extra=column_types`` + + This example is from an instance where the ``tags`` column has been assigned the ``json`` column type. .. code-block:: json - {} + { + "tags": { + "type": "json", + "config": null + } + } ``metadata`` - Metadata about the table, database or stored query + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) ``GET /fixtures/simple_primary_key/1.json?_extra=metadata`` + This table has no metadata, so only an empty ``columns`` object is returned. + .. code-block:: json { @@ -821,7 +920,26 @@ The following extras are available for row JSON responses. } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -842,7 +960,7 @@ The following extras are available for row JSON responses. "simple_primary_key" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/simple_primary_key/1.json?_extra=database_color`` @@ -851,7 +969,7 @@ The following extras are available for row JSON responses. "9403e5" ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/simple_primary_key/1.json?_extra=private`` @@ -860,10 +978,12 @@ The following extras are available for row JSON responses. false ``foreign_key_tables`` - Tables that link to this row using foreign keys + Tables that link to this row using foreign keys (May execute additional queries.) ``GET /fixtures/simple_primary_key/1.json?_extra=foreign_key_tables`` + ``count`` is the number of rows in the other table that reference this row, and ``link`` is a URL to browse those rows. + .. code-block:: json [ @@ -921,7 +1041,7 @@ The following extras are available for arbitrary SQL query responses and stored, ] ``render_cell`` - Rendered HTML for each cell using the render_cell plugin hook + Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) The ``render_cell`` array has one item per query result row, in the same order as the ``rows`` array. Each object is keyed by column name. Only columns whose rendered value differs from the default are included. @@ -941,7 +1061,7 @@ The following extras are available for arbitrary SQL query responses and stored, } ``debug`` - Extra debug information + Extra debug information (The contents of this block are not a stable part of the Datasette API and may change without warning.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=debug`` @@ -1000,10 +1120,12 @@ The following extras are available for arbitrary SQL query responses and stored, } ``metadata`` - Metadata about the table, database or stored query + Metadata about the table, database or stored query (See :ref:`metadata` for how to attach metadata to tables.) ``GET /fixtures/neighborhood_search.json?text=town&_extra=metadata`` + For stored queries this returns the full configuration of the query, including the :ref:`stored query options `. For ``?sql=`` queries it returns an empty object. + .. code-block:: json { @@ -1029,7 +1151,26 @@ The following extras are available for arbitrary SQL query responses and stored, } ``extras`` - Available ?_extra= blocks + List of ?_extra= blocks that can be used on this page + + Shape abbreviated from /fixtures/facetable.json?_extra=extras - the full response lists every extra described on this page. ``toggle_url`` is the current URL with that extra added or removed, and ``selected`` is ``true`` for extras included in the current request. + + .. code-block:: json + + [ + { + "name": "count", + "description": "Total count of rows matching these filters", + "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count", + "selected": false + }, + { + "name": "extras", + "description": "List of ?_extra= blocks that can be used on this page", + "toggle_url": "http://localhost/fixtures/facetable.json", + "selected": true + } + ] ``database`` Database name @@ -1041,7 +1182,7 @@ The following extras are available for arbitrary SQL query responses and stored, "fixtures" ``database_color`` - Color assigned to the database + Color assigned to the database (A six character hex color, without the leading ``#``, derived from a hash of the database name and used in the Datasette interface.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=database_color`` @@ -1050,7 +1191,7 @@ The following extras are available for arbitrary SQL query responses and stored, "9403e5" ``private`` - Whether this resource is private to the current actor + Whether this resource is private to the current actor (``true`` if the current actor can see this resource but an anonymous user could not. See :ref:`authentication_permissions`.) ``GET /fixtures/-/query.json?sql=select+1+as+one&_extra=private`` diff --git a/docs/json_api_doc.py b/docs/json_api_doc.py index 44ef4a42..422e67f4 100644 --- a/docs/json_api_doc.py +++ b/docs/json_api_doc.py @@ -93,9 +93,26 @@ async def _fetch_live_examples(scoped_classes): datasette = Datasette( [str(db_path)], settings={"num_sql_threads": 1}, + metadata={ + "databases": { + "fixtures": { + "tables": { + "facetable": { + "description": "A demo table of places, used to demonstrate facets", + "columns": {"state": "Two letter US state code"}, + } + } + } + } + }, config={ "databases": { "fixtures": { + "tables": { + "facetable": { + "column_types": {"tags": "json"}, + } + }, "queries": { "neighborhood_search": { "sql": textwrap.dedent(""" @@ -108,7 +125,7 @@ async def _fetch_live_examples(scoped_classes): """), "title": "Search neighborhoods", } - } + }, } } }, From 88878b418473dcb399de376658d2bd7423b66c97 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jun 2026 12:51:40 -0700 Subject: [PATCH 1111/1116] datasette.allowed_many() method --- datasette/app.py | 101 +++++++--- datasette/utils/actions_sql.py | 221 ++++++++++++++------- docs/internals.rst | 33 ++++ docs/plugin_hooks.rst | 6 + tests/conftest.py | 11 +- tests/test_allowed_many.py | 341 +++++++++++++++++++++++++++++++++ 6 files changed, 613 insertions(+), 100 deletions(-) create mode 100644 tests/test_allowed_many.py diff --git a/datasette/app.py b/datasette/app.py index 81d23acb..a6696ad9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2,7 +2,7 @@ from __future__ import annotations import asyncio import contextvars -from typing import TYPE_CHECKING, Any, Dict, Iterable, List +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence if TYPE_CHECKING: from datasette.permissions import Resource @@ -1817,46 +1817,97 @@ class Datasette: # For global actions, resource can be omitted: can_debug = await datasette.allowed(action="permissions-debug", actor=actor) """ - from datasette.utils.actions_sql import check_permission_for_resource + results = await self.allowed_many( + actions=[action], resource=resource, actor=actor + ) + return results[action] - # For global actions, resource remains None + async def allowed_many( + self, + *, + actions: Sequence[str], + resource: "Resource" = None, + actor: dict | None = None, + ) -> dict[str, bool]: + """ + Check several actions against one resource for one actor. - # Check if this action has also_requires - if so, check that action first - action_obj = self.actions.get(action) - if action_obj and action_obj.also_requires: - # Must have the required action first - if not await self.allowed( - action=action_obj.also_requires, - resource=resource, + Resolves every action (plus any also_requires dependencies) with a + single internal database query, instead of one or two queries per + action. + + Example: + from datasette.resources import TableResource + results = await datasette.allowed_many( + actions=["edit-schema", "drop-table", "insert-row"], + resource=TableResource(database="data", table="exercise"), actor=actor, - ): - return False + ) + # {"edit-schema": True, "drop-table": True, "insert-row": False} + """ + from datasette.utils.actions_sql import check_permissions_for_actions # For global actions, resource is None parent = resource.parent if resource else None child = resource.child if resource else None - result = await check_permission_for_resource( + # Expand also_requires dependencies (transitively) so that each + # dependency is resolved within the same batch + expanded = [] + + def add_action(name): + if name in expanded: + return + action_obj = self.actions.get(name) + if action_obj is None: + raise ValueError(f"Unknown action: {name}") + expanded.append(name) + if action_obj.also_requires: + add_action(action_obj.also_requires) + + requested = list(dict.fromkeys(actions)) + for name in requested: + add_action(name) + + raw = await check_permissions_for_actions( datasette=self, actor=actor, - action=action, + actions=expanded, parent=parent, child=child, ) + final = {} - # Log the permission check for debugging - self._permission_checks.append( - PermissionCheck( - when=datetime.datetime.now(datetime.timezone.utc).isoformat(), - actor=actor, - action=action, - parent=parent, - child=child, - result=result, + def resolve(name): + # final verdict = own rules AND verdict of also_requires chain + if name in final: + return final[name] + result = raw[name] + action_obj = self.actions.get(name) + if result and action_obj.also_requires: + result = resolve(action_obj.also_requires) + final[name] = result + return result + + for name in expanded: + resolve(name) + + # Log every check for the debug page, dependencies before the + # actions that required them + when = datetime.datetime.now(datetime.timezone.utc).isoformat() + for name in reversed(expanded): + self._permission_checks.append( + PermissionCheck( + when=when, + actor=actor, + action=name, + parent=parent, + child=child, + result=final[name], + ) ) - ) - return result + return {name: final[name] for name in requested} async def ensure_permission( self, diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index 891ee913..a422c1ed 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -21,6 +21,8 @@ The core pattern is: - Across levels, child beats parent beats global """ +import asyncio +import re from typing import TYPE_CHECKING from datasette.utils.permissions import gather_permission_sql_from_hooks @@ -495,6 +497,146 @@ async def build_permission_rules_sql( return rules_union, all_params, restriction_sqls +async def check_permissions_for_actions( + *, + datasette: "Datasette", + actor: dict | None, + actions: list[str], + parent: str | None, + child: str | None, +) -> dict[str, bool]: + """ + Check several actions for one actor and resource in a single query. + + Args: + datasette: The Datasette instance + actor: The actor dict (or None) + actions: List of action names to check + parent: The parent resource identifier (e.g., database name, or None) + child: The child resource identifier (e.g., table name, or None) + + Returns: + Dict mapping each action name to True (allowed) or False (denied) + + Each action contributes its own tagged block of permission rules + (gathered from the permission_resources_sql hook, with parameters + namespaced per action to avoid collisions) plus an optional + restriction allowlist CTE. One internal database query resolves + the winning rule per action using the same specificity-then-deny + ordering as the rest of the permission system. + + Note: this resolves each action independently - also_requires + dependencies are handled by the caller (Datasette.allowed_many). + """ + from datasette.utils.permissions import SKIP_PERMISSION_CHECKS + + for action in actions: + if not datasette.actions.get(action): + raise ValueError(f"Unknown action: {action}") + + # Dedupe while preserving order + unique_actions = list(dict.fromkeys(actions)) + if not unique_actions: + return {} + + # Gather hook results for each action concurrently - hooks within a + # single action still run sequentially, preserving existing semantics + gathered = await asyncio.gather( + *( + gather_permission_sql_from_hooks( + datasette=datasette, actor=actor, action=action + ) + for action in unique_actions + ) + ) + + if any(result is SKIP_PERMISSION_CHECKS for result in gathered): + return {action: True for action in unique_actions} + + params = {"_check_parent": parent, "_check_child": child} + ctes = [] + selects = [] + verdicts = {} + + for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)): + prefix = f"a{i}_" + rule_parts = [] + restriction_parts = [] + + for permission_sql in permission_sqls: + sql = permission_sql.sql + restriction_sql = permission_sql.restriction_sql + # Namespace this block's params so identical names used for + # different actions cannot collide + for key in permission_sql.params or {}: + new_key = prefix + key + params[new_key] = permission_sql.params[key] + pattern = re.compile(":" + re.escape(key) + r"(?![A-Za-z0-9_])") + if sql: + sql = pattern.sub(":" + new_key, sql) + if restriction_sql: + restriction_sql = pattern.sub(":" + new_key, restriction_sql) + + if restriction_sql: + restriction_parts.append(restriction_sql) + + # Skip plugins that only provide restriction_sql (no permission rules) + if sql is None: + continue + rule_parts.append( + f"SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)" + ) + + if not rule_parts: + # No rules from any plugin - default deny. Restrictions can + # only restrict, never grant, so no SQL is needed at all + verdicts[action] = False + continue + ctes.append(f"a{i}_rules AS (\n" + "\nUNION ALL\n".join(rule_parts) + "\n)") + + # Winning rule for this action: most specific depth first, then + # deny-beats-allow, then source_plugin as a stable tie-break + verdict_sql = f"""COALESCE(( + SELECT allow FROM ( + SELECT allow, source_plugin, + CASE + WHEN child IS NOT NULL THEN 2 + WHEN parent IS NOT NULL THEN 1 + ELSE 0 + END AS depth + FROM a{i}_rules + WHERE (parent IS NULL OR parent = :_check_parent) + AND (child IS NULL OR child = :_check_child) + ORDER BY + depth DESC, + CASE WHEN allow = 0 THEN 0 ELSE 1 END, + source_plugin + LIMIT 1 + ) +), 0)""" + + if restriction_parts: + # Database-level restrictions (parent, NULL) match all children + restriction_intersect = "\nINTERSECT\n".join( + f"SELECT * FROM ({sql})" for sql in restriction_parts + ) + ctes.append(f"a{i}_restriction AS (\n{restriction_intersect}\n)") + verdict_sql = f"""({verdict_sql}) AND EXISTS ( + SELECT 1 FROM a{i}_restriction r + WHERE (r.parent = :_check_parent OR r.parent IS NULL) + AND (r.child = :_check_child OR r.child IS NULL) +)""" + + selects.append(f"SELECT {i} AS action_idx, ({verdict_sql}) AS is_allowed") + + if selects: + query = "WITH\n" + ",\n".join(ctes) + "\n" + "\nUNION ALL\n".join(selects) + result = await datasette.get_internal_database().execute(query, params) + for row in result.rows: + verdicts[unique_actions[row[0]]] = bool(row[1]) + return verdicts + + async def check_permission_for_resource( *, datasette: "Datasette", @@ -515,77 +657,12 @@ async def check_permission_for_resource( Returns: True if the actor is allowed, False otherwise - - This builds the cascading permission query and checks if the specific - resource is in the allowed set. """ - rules_union, all_params, restriction_sqls = await build_permission_rules_sql( - datasette, actor, action + results = await check_permissions_for_actions( + datasette=datasette, + actor=actor, + actions=[action], + parent=parent, + child=child, ) - - # If no rules (empty SQL), default deny - if not rules_union: - return False - - # Add parameters for the resource we're checking - all_params["_check_parent"] = parent - all_params["_check_child"] = child - - # If there are restriction filters, check if the resource passes them first - if restriction_sqls: - # Check if resource is in restriction allowlist - # Database-level restrictions (parent, NULL) should match all children (parent, *) - # Wrap each restriction_sql in a subquery to avoid operator precedence issues - restriction_check = "\nINTERSECT\n".join( - f"SELECT * FROM ({sql})" for sql in restriction_sqls - ) - restriction_query = f""" -WITH restriction_list AS ( - {restriction_check} -) -SELECT EXISTS ( - SELECT 1 FROM restriction_list - WHERE (parent = :_check_parent OR parent IS NULL) - AND (child = :_check_child OR child IS NULL) -) AS in_allowlist -""" - result = await datasette.get_internal_database().execute( - restriction_query, all_params - ) - if result.rows and not result.rows[0][0]: - # Resource not in restriction allowlist - deny - return False - - query = f""" -WITH -all_rules AS ( - {rules_union} -), -matched_rules AS ( - SELECT ar.*, - CASE - WHEN ar.child IS NOT NULL THEN 2 -- child-level (most specific) - WHEN ar.parent IS NOT NULL THEN 1 -- parent-level - ELSE 0 -- root/global - END AS depth - FROM all_rules ar - WHERE (ar.parent IS NULL OR ar.parent = :_check_parent) - AND (ar.child IS NULL OR ar.child = :_check_child) -), -winner AS ( - SELECT * - FROM matched_rules - ORDER BY - depth DESC, -- specificity first (higher depth wins) - CASE WHEN allow=0 THEN 0 ELSE 1 END, -- then deny over allow - source_plugin -- stable tie-break - LIMIT 1 -) -SELECT COALESCE((SELECT allow FROM winner), 0) AS is_allowed -""" - - # Execute the query against the internal database - result = await datasette.get_internal_database().execute(query, all_params) - if result.rows: - return bool(result.rows[0][0]) - return False + return results[action] diff --git a/docs/internals.rst b/docs/internals.rst index f269155a..f3c1152a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -512,6 +512,39 @@ Example usage: The method returns ``True`` if the permission is granted, ``False`` if denied. +.. _datasette_allowed_many: + +await .allowed_many(\*, actions, resource, actor=None) +------------------------------------------------------ + +``actions`` - list of strings + The names of the actions to permission check. + +``resource`` - Resource object + A Resource object representing the database, table, or other resource that each action is checked against. Omit for global actions. + +``actor`` - dictionary, optional + The authenticated actor. This is usually ``request.actor``. Defaults to ``None`` for unauthenticated requests. + +Checks several actions against the same resource for the same actor, returning a dictionary mapping each action name to ``True`` or ``False``. The whole batch - including any actions pulled in through ``also_requires`` dependencies - is resolved with a single SQL query against the internal database, so this is much faster than calling :ref:`datasette.allowed() ` once per action. + +Example usage: + +.. code-block:: python + + from datasette.resources import TableResource + + results = await datasette.allowed_many( + actions=["insert-row", "delete-row", "drop-table"], + resource=TableResource( + database="fixtures", table="facetable" + ), + actor=request.actor, + ) + # {"insert-row": True, "delete-row": True, "drop-table": False} + +Actions for which no plugin provides any permission rules are resolved to ``False`` directly, without being included in the SQL query at all. + .. _datasette_allowed_resources: await .allowed_resources(action, actor=None, \*, parent=None, include_is_private=False, include_reasons=False, limit=100, next=None) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 2a0ddc93..0a55f8ec 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1458,6 +1458,12 @@ to avoid conflicts with other plugins. The recommended convention is to prefix p plugin's source name (e.g., ``myplugin_user_id``). The system reserves these parameter names: ``:actor``, ``:actor_id``, ``:action``, and ``:filter_parent``. +This hook may be called for many actions in rapid succession - for example +:ref:`datasette.allowed_many() ` gathers rules for every action in its batch +concurrently. Hook implementations must not assume that checks for different actions arrive one +page-render apart, and expensive work (such as network calls) should be cached independently of the +``action`` argument where possible. + You can also use return ``PermissionSQL.allow(reason="reason goes here")`` or ``PermissionSQL.deny(reason="reason goes here")`` as shortcuts for simple root-level allow or deny rules. These will create SQL snippets that look like this: .. code-block:: sql diff --git a/tests/conftest.py b/tests/conftest.py index b9b3c35e..27d6fa77 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -146,6 +146,7 @@ def restore_working_directory(tmpdir, request): @pytest.fixture(scope="session", autouse=True) def check_actions_are_documented(): from datasette.plugins import pm + from datasette.default_actions import register_actions as default_register_actions content = ( pathlib.Path(__file__).parent.parent / "docs" / "authentication.rst" @@ -154,6 +155,9 @@ def check_actions_are_documented(): documented_actions = set(permissions_re.findall(content)).union( UNDOCUMENTED_PERMISSIONS ) + # Only Datasette core actions need to be documented - actions registered + # by (test) plugins are checked for registration but not documentation + core_actions = {action.name for action in default_register_actions()} def before(hook_name, hook_impls, kwargs): if hook_name == "permission_resources_sql": @@ -165,9 +169,10 @@ def check_actions_are_documented(): + " (or maybe a test forgot to do await ds.invoke_startup())" ) action = kwargs.get("action").replace("-", "_") - assert ( - action in documented_actions - ), "Undocumented permission action: {}".format(action) + if kwargs["action"] in core_actions: + assert ( + action in documented_actions + ), "Undocumented permission action: {}".format(action) pm.add_hookcall_monitoring( before=before, after=lambda outcome, hook_name, hook_impls, kwargs: None diff --git a/tests/test_allowed_many.py b/tests/test_allowed_many.py new file mode 100644 index 00000000..53d0ffd9 --- /dev/null +++ b/tests/test_allowed_many.py @@ -0,0 +1,341 @@ +""" +Tests for the datasette.allowed_many() batch permission API, which +resolves multiple actions against one resource in a single internal +database query. datasette.allowed() is implemented on top of it, so +both entry points share one resolution code path. +""" + +import pytest +import pytest_asyncio +from datasette.app import Datasette +from datasette.permissions import PermissionSQL, SkipPermissions +from datasette.resources import DatabaseResource, TableResource +from datasette import hookimpl + + +@pytest_asyncio.fixture +async def ds(): + ds = Datasette() + await ds.invoke_startup() + db = ds.add_memory_database("analytics") + await db.execute_write("CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY)") + await db.execute_write("CREATE TABLE IF NOT EXISTS events (id INTEGER PRIMARY KEY)") + await ds._refresh_schemas() + return ds + + +class MatrixRulesPlugin: + """Different rules per action for actor carol, to exercise resolution.""" + + @hookimpl + def permission_resources_sql(self, datasette, actor, action): + if not actor or actor.get("id") != "carol": + return None + if action == "view-table": + return PermissionSQL(sql=""" + SELECT NULL AS parent, NULL AS child, 1 AS allow, 'global allow' AS reason + UNION ALL + SELECT 'analytics' AS parent, 'sensitive' AS child, 0 AS allow, 'deny sensitive' AS reason + """) + if action == "insert-row": + return PermissionSQL( + sql="SELECT 'analytics' AS parent, NULL AS child, 1 AS allow, 'analytics writes' AS reason" + ) + # Everything else: no opinion (implicit deny unless defaults allow) + return None + + +@pytest.mark.asyncio +async def test_allowed_many_basic(ds): + plugin = MatrixRulesPlugin() + ds.pm.register(plugin, name="matrix") + try: + results = await ds.allowed_many( + actions=["view-table", "insert-row", "drop-table"], + resource=TableResource("analytics", "users"), + actor={"id": "carol"}, + ) + assert results == { + "view-table": True, + "insert-row": True, + "drop-table": False, + } + # Child-level deny beats global allow + sensitive = await ds.allowed_many( + actions=["view-table"], + resource=TableResource("analytics", "sensitive"), + actor={"id": "carol"}, + ) + assert sensitive == {"view-table": False} + finally: + ds.pm.unregister(name="matrix") + + +@pytest.mark.asyncio +async def test_allowed_many_matches_allowed(ds): + """Every action resolved by allowed_many() must match allowed().""" + plugin = MatrixRulesPlugin() + ds.pm.register(plugin, name="matrix") + try: + all_actions = list(ds.actions) + for resource in ( + TableResource("analytics", "users"), + TableResource("analytics", "sensitive"), + DatabaseResource("analytics"), + ): + batched = await ds.allowed_many( + actions=all_actions, resource=resource, actor={"id": "carol"} + ) + assert set(batched) == set(all_actions) + for action in all_actions: + individual = await ds.allowed( + action=action, resource=resource, actor={"id": "carol"} + ) + assert ( + batched[action] == individual + ), f"Mismatch for {action} on {resource}" + finally: + ds.pm.unregister(name="matrix") + + +@pytest.mark.asyncio +async def test_allowed_many_unknown_action_raises(ds): + with pytest.raises(ValueError, match="Unknown action"): + await ds.allowed_many( + actions=["view-table", "no-such-action"], + resource=TableResource("analytics", "users"), + actor=None, + ) + + +@pytest.mark.asyncio +async def test_allowed_many_empty_actions(ds): + assert ( + await ds.allowed_many( + actions=[], resource=TableResource("analytics", "users"), actor=None + ) + == {} + ) + + +class AlsoRequiresRulesPlugin: + """dave: store-query allowed but execute-sql explicitly denied. + erin: store-query allowed (execute-sql stays default-allowed).""" + + @hookimpl + def permission_resources_sql(self, datasette, actor, action): + actor_id = actor.get("id") if actor else None + if actor_id == "dave": + if action == "store-query": + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'dave can store' AS reason" + ) + if action == "execute-sql": + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, 0 AS allow, 'dave no sql' AS reason" + ) + if actor_id == "erin" and action == "store-query": + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'erin can store' AS reason" + ) + return None + + +@pytest.mark.asyncio +async def test_allowed_many_also_requires(ds): + # store-query also_requires execute-sql, which also_requires view-database + plugin = AlsoRequiresRulesPlugin() + ds.pm.register(plugin, name="also_requires") + try: + resource = DatabaseResource("analytics") + dave = await ds.allowed_many( + actions=["store-query", "execute-sql", "view-database"], + resource=resource, + actor={"id": "dave"}, + ) + # execute-sql denied, so store-query must be denied too + assert dave == { + "store-query": False, + "execute-sql": False, + "view-database": True, + } + erin = await ds.allowed_many( + actions=["store-query"], resource=resource, actor={"id": "erin"} + ) + assert erin == {"store-query": True} + # Must match the single-check path + assert ( + await ds.allowed( + action="store-query", resource=resource, actor={"id": "dave"} + ) + is False + ) + assert ( + await ds.allowed( + action="store-query", resource=resource, actor={"id": "erin"} + ) + is True + ) + finally: + ds.pm.unregister(name="also_requires") + + +@pytest.mark.asyncio +async def test_allowed_many_respects_restrictions(ds): + """Token-style _r restrictions are enforced within the batch.""" + actor = {"id": "root", "_r": {"d": {"analytics": ["vt"]}}} + results = await ds.allowed_many( + actions=["view-table", "drop-table"], + resource=TableResource("analytics", "users"), + actor=actor, + ) + # root could normally do both, but the token only allows view-table + # on the analytics database + assert results == {"view-table": True, "drop-table": False} + other_db = await ds.allowed_many( + actions=["view-table"], + resource=TableResource("production", "stuff"), + actor=actor, + ) + assert other_db == {"view-table": False} + # Equivalence with allowed() + assert ( + await ds.allowed( + action="view-table", + resource=TableResource("analytics", "users"), + actor=actor, + ) + is True + ) + assert ( + await ds.allowed( + action="drop-table", + resource=TableResource("analytics", "users"), + actor=actor, + ) + is False + ) + + +class ParamCollisionPlugin: + """Same parameter name with a different value for every action.""" + + @hookimpl + def permission_resources_sql(self, datasette, actor, action): + if not actor or actor.get("id") != "paula": + return None + flag = 1 if action in ("drop-table", "insert-row") else 0 + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, :flag AS allow, 'flagged' AS reason", + params={"flag": flag}, + ) + + +@pytest.mark.asyncio +async def test_allowed_many_namespaces_params_across_actions(ds): + """Many actions whose rules use identical param names must not collide.""" + plugin = ParamCollisionPlugin() + ds.pm.register(plugin, name="collision") + try: + all_actions = list(ds.actions) + assert len(all_actions) >= 15 + resource = TableResource("analytics", "users") + results = await ds.allowed_many( + actions=all_actions, resource=resource, actor={"id": "paula"} + ) + # Spot-check: only the flagged actions resolve True + assert results["drop-table"] is True + assert results["create-table"] is False + # Full equivalence against single checks + for action in all_actions: + assert results[action] == await ds.allowed( + action=action, resource=resource, actor={"id": "paula"} + ), f"Mismatch for {action}" + finally: + ds.pm.unregister(name="collision") + + +@pytest.mark.asyncio +async def test_allowed_many_single_internal_db_query(ds): + internal_db = ds.get_internal_database() + calls = [] + original_execute = internal_db.execute + + async def counting_execute(sql, params=None, **kwargs): + calls.append(sql) + return await original_execute(sql, params, **kwargs) + + internal_db.execute = counting_execute + try: + results = await ds.allowed_many( + actions=["view-table", "insert-row", "delete-row", "drop-table"], + resource=TableResource("analytics", "users"), + actor={"id": "root", "_r": {"d": {"analytics": ["vt"]}}}, + ) + assert len(results) == 4 + assert len(calls) == 1 + finally: + internal_db.execute = original_execute + + +@pytest.mark.asyncio +async def test_allowed_many_no_query_when_no_rules(ds): + """Actions with no rules from any plugin are denied without SQL. + + Restrictions can only restrict, never grant, so an action with no + rule rows is always False - it should not contribute to the query, + and if no action has rules there should be no query at all.""" + internal_db = ds.get_internal_database() + calls = [] + original_execute = internal_db.execute + + async def counting_execute(sql, params=None, **kwargs): + calls.append(sql) + return await original_execute(sql, params, **kwargs) + + internal_db.execute = counting_execute + try: + # bob gets no rules at all for these write actions + results = await ds.allowed_many( + actions=["drop-table", "delete-row"], + resource=TableResource("analytics", "users"), + actor={"id": "bob"}, + ) + assert results == {"drop-table": False, "delete-row": False} + assert len(calls) == 0 + # A mixed batch still needs exactly one query + calls.clear() + results = await ds.allowed_many( + actions=["view-table", "drop-table"], + resource=TableResource("analytics", "users"), + actor={"id": "bob"}, + ) + assert results == {"view-table": True, "drop-table": False} + assert len(calls) == 1 + finally: + internal_db.execute = original_execute + + +@pytest.mark.asyncio +async def test_allowed_many_global_actions_without_resource(ds): + results = await ds.allowed_many( + actions=["view-instance", "permissions-debug"], + actor={"id": "root"}, + ) + assert results["view-instance"] is True + # Equivalence with single checks for global actions + for action in ("view-instance", "permissions-debug"): + assert results[action] == await ds.allowed(action=action, actor={"id": "root"}) + anon = await ds.allowed_many(actions=["permissions-debug"], actor=None) + assert anon == {"permissions-debug": False} + + +@pytest.mark.asyncio +async def test_allowed_many_skip_permission_checks(ds): + with SkipPermissions(): + results = await ds.allowed_many( + actions=["view-table", "drop-table"], + resource=TableResource("analytics", "users"), + actor=None, + ) + assert results == {"view-table": True, "drop-table": True} From bb59c61c9f9b6766199ce1434c7008739653f141 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jun 2026 13:11:09 -0700 Subject: [PATCH 1112/1116] Request-scoped permission check cache Adds a per-request cache for permission check results, plus wiring that resolves action permissions in bulk before plugin hooks need them: - New _permission_check_cache contextvar, set to a fresh dict for each request by DatasetteRouter and reset when the request ends. Keys include the full serialized actor, so actors differing in any field (e.g. token restrictions) never share entries. SkipPermissions mode bypasses the cache entirely. - datasette.allowed_many() now consults the cache and stores its results there, so repeated datasette.allowed() checks within one request resolve without further SQL. - Table pages resolve all registered table-level actions against the current table and all database-level actions against its database (database pages likewise) in batched queries before invoking the table_actions/database_actions plugin hooks - allowed() calls made inside those hooks are then served from the cache with no plugin changes required. Actions with no permission rules from any plugin are resolved to False without touching the database. Benchmarks (benchmarks/) with a simulated 12-plugin ecosystem making 18 checks per table page show 34 -> 13 internal-DB queries per page; with 2ms-per-query internal DB latency (modelling Datasette Cloud) table page time drops from 77.9ms to 27.6ms - the caching layer accounts for ~91% of that improvement over allowed_many() alone. Co-Authored-By: Claude Fable 5 --- datasette/app.py | 67 +++++-- datasette/permissions.py | 8 + datasette/views/database.py | 13 ++ datasette/views/table_extras.py | 26 ++- docs/internals.rst | 4 + docs/plugin_hooks.rst | 6 +- tests/test_allowed_many.py | 340 +++++++++++++++++++++++++++++++- 7 files changed, 443 insertions(+), 21 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index a6696ad9..9979b6c5 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -291,6 +291,15 @@ DEFAULT_NOT_SET = object() ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params")) +def _permission_cache_key(actor, action, parent, child): + # Key on the full serialized actor so actors differing in any field + # (e.g. token restrictions) never share cache entries + actor_key = ( + json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None + ) + return (actor_key, action, parent, child) + + async def favicon(request, send): await asgi_send_file( send, @@ -1834,7 +1843,9 @@ class Datasette: Resolves every action (plus any also_requires dependencies) with a single internal database query, instead of one or two queries per - action. + action. Results are stored in the request-scoped permission cache, + so subsequent datasette.allowed() calls for the same checks within + the same request are served from the cache. Example: from datasette.resources import TableResource @@ -1846,6 +1857,10 @@ class Datasette: # {"edit-schema": True, "drop-table": True, "insert-row": False} """ from datasette.utils.actions_sql import check_permissions_for_actions + from datasette.permissions import ( + _permission_check_cache, + _skip_permission_checks, + ) # For global actions, resource is None parent = resource.parent if resource else None @@ -1869,14 +1884,30 @@ class Datasette: for name in requested: add_action(name) - raw = await check_permissions_for_actions( - datasette=self, - actor=actor, - actions=expanded, - parent=parent, - child=child, - ) + # Consult the request-scoped cache, unless permission checks are + # being skipped (skip-mode verdicts must never be cached) + skip = _skip_permission_checks.get() + cache = None if skip else _permission_check_cache.get() + final = {} + to_check = [] + for name in expanded: + if cache is not None: + key = _permission_cache_key(actor, name, parent, child) + if key in cache: + final[name] = cache[key] + continue + to_check.append(name) + + raw = {} + if to_check: + raw = await check_permissions_for_actions( + datasette=self, + actor=actor, + actions=to_check, + parent=parent, + child=child, + ) def resolve(name): # final verdict = own rules AND verdict of also_requires chain @@ -1892,8 +1923,13 @@ class Datasette: for name in expanded: resolve(name) - # Log every check for the debug page, dependencies before the - # actions that required them + # Cache the freshly computed checks + if cache is not None: + for name in to_check: + cache[_permission_cache_key(actor, name, parent, child)] = final[name] + + # Log every check (including cache hits) for the debug page, + # dependencies before the actions that required them when = datetime.datetime.now(datetime.timezone.utc).isoformat() for name in reversed(expanded): self._permission_checks.append( @@ -2663,7 +2699,16 @@ class DatasetteRouter: if raw_path: path = raw_path.decode("ascii") path = path.partition("?")[0] - return await self.route_path(scope, receive, send, path) + # Give each request a fresh permission check cache, so repeated + # datasette.allowed() checks within the request are memoized but + # results never persist beyond it + from datasette.permissions import _permission_check_cache + + cache_token = _permission_check_cache.set({}) + try: + return await self.route_path(scope, receive, send, path) + finally: + _permission_check_cache.reset(cache_token) async def route_path(self, scope, receive, send, path): # Strip off base_url if present before routing diff --git a/datasette/permissions.py b/datasette/permissions.py index a9a3cc7c..786dc026 100644 --- a/datasette/permissions.py +++ b/datasette/permissions.py @@ -8,6 +8,14 @@ _skip_permission_checks = contextvars.ContextVar( "skip_permission_checks", default=False ) +# Request-scoped cache of permission check results. The ASGI router sets +# this to a fresh dict at the start of each request, so cached verdicts +# never outlive a request or leak between actors. Keys are +# (actor_json, action, parent, child) tuples, values are booleans. +_permission_check_cache: contextvars.ContextVar[dict | None] = contextvars.ContextVar( + "permission_check_cache", default=None +) + class SkipPermissions: """Context manager to temporarily skip permission checks. diff --git a/datasette/views/database.py b/datasette/views/database.py index f1756863..6afd9734 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -118,6 +118,19 @@ class DatabaseView(View): ) async def database_actions(): + # Resolve the registered database-level actions for this + # database in one batched query, seeding the request permission + # cache so that allowed() calls made inside the plugin hooks + # below are served from the cache + await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is DatabaseResource + ], + resource=DatabaseResource(database), + actor=request.actor, + ) links = [] for hook in pm.hook.database_actions( datasette=datasette, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 948f3daa..a0308e49 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from datasette.database import QueryInterrupted from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider from datasette.plugins import pm -from datasette.resources import TableResource +from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( await_me_maybe, call_with_supported_arguments, @@ -361,6 +361,30 @@ class ActionsExtra(Extra): else: kwargs["table"] = context.table_name method = pm.hook.table_actions + # Resolve the registered table-level actions for this table + # and the database-level actions for its database in two + # batched queries, seeding the request permission cache so + # that allowed() calls made inside the plugin hooks below + # are served from the cache + datasette = context.datasette + await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is TableResource + ], + resource=TableResource(context.database_name, context.table_name), + actor=context.request.actor, + ) + await datasette.allowed_many( + actions=[ + name + for name, action in datasette.actions.items() + if action.resource_class is DatabaseResource + ], + resource=DatabaseResource(context.database_name), + actor=context.request.actor, + ) for hook in method(**kwargs): extra_links = await await_me_maybe(hook) if extra_links: diff --git a/docs/internals.rst b/docs/internals.rst index f3c1152a..641286f8 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -512,6 +512,8 @@ Example usage: The method returns ``True`` if the permission is granted, ``False`` if denied. +Results are cached for the duration of the current request, so checking the same ``(actor, action, resource)`` combination twice within one request only does the underlying permission resolution work once. + .. _datasette_allowed_many: await .allowed_many(\*, actions, resource, actor=None) @@ -543,6 +545,8 @@ Example usage: ) # {"insert-row": True, "delete-row": True, "drop-table": False} +Each result is stored in the per-request permission check cache, so subsequent ``datasette.allowed()`` calls for the same checks within the same request are served from that cache. Datasette uses this before running the ``table_actions`` and ``database_actions`` plugin hooks: it resolves every registered table-level action against the current table and every database-level action against its database first, which means ``allowed()`` calls made by those plugin hooks are usually served from the cache instead of triggering additional queries. + Actions for which no plugin provides any permission rules are resolved to ``False`` directly, without being included in the SQL query at all. .. _datasette_allowed_resources: diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 0a55f8ec..580f7402 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1460,9 +1460,9 @@ plugin's source name (e.g., ``myplugin_user_id``). The system reserves these par This hook may be called for many actions in rapid succession - for example :ref:`datasette.allowed_many() ` gathers rules for every action in its batch -concurrently. Hook implementations must not assume that checks for different actions arrive one -page-render apart, and expensive work (such as network calls) should be cached independently of the -``action`` argument where possible. +concurrently before table and database pages render their action menus. Hook implementations must not +assume that checks for different actions arrive one page-render apart, and expensive work (such as +network calls) should be cached independently of the ``action`` argument where possible. You can also use return ``PermissionSQL.allow(reason="reason goes here")`` or ``PermissionSQL.deny(reason="reason goes here")`` as shortcuts for simple root-level allow or deny rules. These will create SQL snippets that look like this: diff --git a/tests/test_allowed_many.py b/tests/test_allowed_many.py index 53d0ffd9..3d0d0c9a 100644 --- a/tests/test_allowed_many.py +++ b/tests/test_allowed_many.py @@ -1,18 +1,52 @@ """ -Tests for the datasette.allowed_many() batch permission API, which -resolves multiple actions against one resource in a single internal -database query. datasette.allowed() is implemented on top of it, so -both entry points share one resolution code path. +Tests for request-scoped permission check memoization and the +datasette.allowed_many() batch permission API. + +Layer 1: per-request cache consulted by datasette.allowed() +Layer 2: allowed_many() resolves multiple actions in one internal-DB query +Layer 3: table/database views precompute all registered actions before + invoking table_actions/database_actions plugin hooks """ import pytest import pytest_asyncio from datasette.app import Datasette -from datasette.permissions import PermissionSQL, SkipPermissions +from datasette.permissions import ( + PermissionSQL, + SkipPermissions, + _permission_check_cache, +) from datasette.resources import DatabaseResource, TableResource from datasette import hookimpl +class CountingRulesPlugin: + """Counts permission_resources_sql gathers and grants rules for alice.""" + + def __init__(self): + self.calls = [] + + @hookimpl + def permission_resources_sql(self, datasette, actor, action): + actor_id = actor.get("id") if actor else None + self.calls.append((actor_id, action)) + if actor_id == "alice": + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'alice allowed' AS reason" + ) + return None + + def count(self, actor_id=None, action=None): + return len( + [ + (a, c) + for a, c in self.calls + if (actor_id is None or a == actor_id) + and (action is None or c == action) + ] + ) + + @pytest_asyncio.fixture async def ds(): ds = Datasette() @@ -24,6 +58,154 @@ async def ds(): return ds +@pytest_asyncio.fixture +async def counting_ds(ds): + plugin = CountingRulesPlugin() + ds.pm.register(plugin, name="counting") + try: + yield ds, plugin + finally: + ds.pm.unregister(name="counting") + + +# ---------------------------------------------------------------------- +# Layer 1: request-scoped memoization +# ---------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_allowed_memoized_when_cache_active(counting_ds): + ds, plugin = counting_ds + resource = TableResource("analytics", "users") + token = _permission_check_cache.set({}) + try: + first = await ds.allowed( + action="view-table", resource=resource, actor={"id": "alice"} + ) + gathers_after_first = plugin.count(actor_id="alice", action="view-table") + assert gathers_after_first > 0 + second = await ds.allowed( + action="view-table", resource=resource, actor={"id": "alice"} + ) + assert first is True + assert second is True + # The second identical check must not gather hooks again + assert plugin.count(actor_id="alice", action="view-table") == ( + gathers_after_first + ) + finally: + _permission_check_cache.reset(token) + + +@pytest.mark.asyncio +async def test_allowed_not_memoized_without_cache(counting_ds): + ds, plugin = counting_ds + resource = TableResource("analytics", "users") + assert _permission_check_cache.get() is None + await ds.allowed(action="view-table", resource=resource, actor={"id": "alice"}) + first_count = plugin.count(actor_id="alice", action="view-table") + await ds.allowed(action="view-table", resource=resource, actor={"id": "alice"}) + # No request cache active - hooks gathered again + assert plugin.count(actor_id="alice", action="view-table") == first_count * 2 + + +@pytest.mark.asyncio +async def test_cache_keyed_on_full_actor_identity(counting_ds): + """Interleaved checks for different actors never share cache entries.""" + # Uses drop-table because default permissions deny it to non-root actors + ds, plugin = counting_ds + resource = TableResource("analytics", "users") + token = _permission_check_cache.set({}) + try: + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "alice"} + ) + is True + ) + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "bob"} + ) + is False + ) + # Repeat interleaved - cached results must stay correct per actor + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "alice"} + ) + is True + ) + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "bob"} + ) + is False + ) + # Actors differing in fields beyond id must not collide either + assert ( + await ds.allowed( + action="drop-table", + resource=resource, + actor={"id": "alice", "_r": {"a": []}}, + ) + is False + ) + finally: + _permission_check_cache.reset(token) + + +@pytest.mark.asyncio +async def test_cache_keyed_on_resource(counting_ds): + ds, plugin = counting_ds + token = _permission_check_cache.set({}) + try: + await ds.allowed( + action="view-table", + resource=TableResource("analytics", "users"), + actor={"id": "alice"}, + ) + count = plugin.count(actor_id="alice", action="view-table") + # Different resource - must not be served from cache + await ds.allowed( + action="view-table", + resource=TableResource("analytics", "events"), + actor={"id": "alice"}, + ) + assert plugin.count(actor_id="alice", action="view-table") == count * 2 + finally: + _permission_check_cache.reset(token) + + +@pytest.mark.asyncio +async def test_skip_permission_checks_bypasses_cache(counting_ds): + ds, plugin = counting_ds + resource = TableResource("analytics", "users") + token = _permission_check_cache.set({}) + try: + with SkipPermissions(): + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "bob"} + ) + is True + ) + # The skip-mode True must not have been cached + assert ( + await ds.allowed( + action="drop-table", resource=resource, actor={"id": "bob"} + ) + is False + ) + finally: + _permission_check_cache.reset(token) + + +# ---------------------------------------------------------------------- +# Layer 2: allowed_many() +# ---------------------------------------------------------------------- + + class MatrixRulesPlugin: """Different rules per action for actor carol, to exercise resolution.""" @@ -233,7 +415,7 @@ class ParamCollisionPlugin: @pytest.mark.asyncio async def test_allowed_many_namespaces_params_across_actions(ds): - """Many actions whose rules use identical param names must not collide.""" + """40+ actions whose rules use identical param names must not collide.""" plugin = ParamCollisionPlugin() ds.pm.register(plugin, name="collision") try: @@ -330,6 +512,24 @@ async def test_allowed_many_global_actions_without_resource(ds): assert anon == {"permissions-debug": False} +@pytest.mark.asyncio +async def test_allowed_many_seeds_request_cache(counting_ds): + ds, plugin = counting_ds + resource = TableResource("analytics", "users") + actions = ["view-table", "insert-row", "drop-table"] + token = _permission_check_cache.set({}) + try: + await ds.allowed_many(actions=actions, resource=resource, actor={"id": "alice"}) + gathers = plugin.count(actor_id="alice") + assert gathers > 0 + for action in actions: + await ds.allowed(action=action, resource=resource, actor={"id": "alice"}) + # Every allowed() call must have been served from the seeded cache + assert plugin.count(actor_id="alice") == gathers + finally: + _permission_check_cache.reset(token) + + @pytest.mark.asyncio async def test_allowed_many_skip_permission_checks(ds): with SkipPermissions(): @@ -339,3 +539,131 @@ async def test_allowed_many_skip_permission_checks(ds): actor=None, ) assert results == {"view-table": True, "drop-table": True} + + +# ---------------------------------------------------------------------- +# Layer 3: precompute before table_actions / database_actions hooks +# ---------------------------------------------------------------------- + + +class ActionHooksPlugin: + """Plugin hooks that make allowed() checks, like real action plugins do.""" + + @hookimpl + def table_actions(self, datasette, actor, database, table): + async def inner(): + links = [] + if await datasette.allowed( + action="drop-table", + resource=TableResource(database, table), + actor=actor, + ): + links.append( + {"href": "/drop", "label": "Drop this table (test-plugin)"} + ) + if await datasette.allowed( + action="create-table", + resource=DatabaseResource(database), + actor=actor, + ): + links.append( + {"href": "/create", "label": "Create a table (test-plugin)"} + ) + return links + + return inner + + @hookimpl + def database_actions(self, datasette, actor, database): + async def inner(): + if await datasette.allowed( + action="create-table", + resource=DatabaseResource(database), + actor=actor, + ): + return [{"href": "/create", "label": "Create a table (test-plugin)"}] + return [] + + return inner + + +@pytest_asyncio.fixture +async def spying_ds(ds, monkeypatch): + """ds with the ActionHooksPlugin plus a spy recording every batch of + actions sent to check_permissions_for_actions.""" + from datasette.utils import actions_sql + + plugin = ActionHooksPlugin() + ds.pm.register(plugin, name="action_hooks") + ds.root_enabled = True + recorded = [] + original = actions_sql.check_permissions_for_actions + + async def spy(**kwargs): + recorded.append(kwargs["actions"]) + return await original(**kwargs) + + monkeypatch.setattr(actions_sql, "check_permissions_for_actions", spy) + try: + yield ds, recorded + finally: + ds.pm.unregister(name="action_hooks") + + +@pytest.mark.asyncio +async def test_table_page_precomputes_action_permissions(spying_ds): + ds, recorded = spying_ds + cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} + response = await ds.client.get("/analytics/users", cookies=cookies) + assert response.status_code == 200 + # The plugin's permission checks were served from the precomputed batch + assert "Drop this table (test-plugin)" in response.text + assert "Create a table (test-plugin)" in response.text + # One batch covered the table-level actions for the table resource, + # and one covered the database-level actions for the database resource + batches = [batch for batch in recorded if len(batch) > 1] + assert any("drop-table" in batch for batch in batches) + assert any("create-table" in batch for batch in batches) + # The precompute is scoped to actions relevant to each resource: + # no global or query-level actions in any batch, and no mixing of + # table-level and database-level actions + for batch in batches: + assert "view-instance" not in batch + assert "view-query" not in batch + assert not ("drop-table" in batch and "create-table" in batch) + # The hook's own allowed() calls hit the cache - no single-action + # fallback queries for the actions it checked + assert ["drop-table"] not in recorded + assert ["create-table"] not in recorded + + +@pytest.mark.asyncio +async def test_database_page_precomputes_action_permissions(spying_ds): + ds, recorded = spying_ds + cookies = {"ds_actor": ds.client.actor_cookie({"id": "root"})} + response = await ds.client.get("/analytics", cookies=cookies) + assert response.status_code == 200 + assert "Create a table (test-plugin)" in response.text + batches = [batch for batch in recorded if len(batch) > 1] + assert any("create-table" in batch for batch in batches) + # Scoped to database-level actions only + for batch in batches: + assert "view-instance" not in batch + assert "drop-table" not in batch + assert ["create-table"] not in recorded + + +@pytest.mark.asyncio +async def test_cache_does_not_leak_across_requests(counting_ds): + ds, plugin = counting_ds + cookies = {"ds_actor": ds.client.actor_cookie({"id": "alice"})} + response = await ds.client.get("/analytics/users.json", cookies=cookies) + assert response.status_code == 200 + first_request_gathers = plugin.count(actor_id="alice", action="view-table") + assert first_request_gathers > 0 + response = await ds.client.get("/analytics/users.json", cookies=cookies) + assert response.status_code == 200 + # Second request must re-gather (fresh cache), not reuse the first one + assert ( + plugin.count(actor_id="alice", action="view-table") == first_request_gathers * 2 + ) From d4cb8b464bf1cbe69a8921fc8c9315e04a5f49cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 12 Jun 2026 13:21:58 -0700 Subject: [PATCH 1113/1116] Fix for trace_child_tasks exception handling I had Claude Fable 5 review our use of contextvar and it spotted this place where exceptions were not correctly handled. --- datasette/tracer.py | 6 ++++-- tests/test_tracer.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/datasette/tracer.py b/datasette/tracer.py index 9e66613b..28f3cc09 100644 --- a/datasette/tracer.py +++ b/datasette/tracer.py @@ -27,8 +27,10 @@ def get_task_id(): @contextmanager def trace_child_tasks(): token = trace_task_id.set(get_task_id()) - yield - trace_task_id.reset(token) + try: + yield + finally: + trace_task_id.reset(token) @contextmanager diff --git a/tests/test_tracer.py b/tests/test_tracer.py index 6cc80fc4..9db211d3 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -70,6 +70,19 @@ def test_trace_query_errors(): assert trace_info["traces"][-1]["error"] == "no such table: non_existent_table" +@pytest.mark.asyncio +async def test_trace_child_tasks_resets_contextvar_on_exception(): + from datasette import tracer + + before = tracer.trace_task_id.get() + with pytest.raises(ValueError): + with tracer.trace_child_tasks(): + assert tracer.trace_task_id.get() is not None + raise ValueError("simulated error") + # The contextvar must be reset even though the block raised + assert tracer.trace_task_id.get() == before + + def test_trace_parallel_queries(): with make_app_client(settings={"trace_debug": True}) as client: response = client.get("/parallel-queries?_trace=1") From 86334d233dd2e668169e54fb2312cae2705e7ffc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jun 2026 11:09:28 -0700 Subject: [PATCH 1114/1116] Switch to CTE to handle 600+ actions at once GPT-5.5 xhigh in Codex spotted this problem and fixed it with a CTE: https://gisthost.github.io/?46076499ee685acddc988ff6b47a74b0 --- datasette/utils/actions_sql.py | 15 ++++++++++---- tests/test_allowed_many.py | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/datasette/utils/actions_sql.py b/datasette/utils/actions_sql.py index a422c1ed..c7137e6b 100644 --- a/datasette/utils/actions_sql.py +++ b/datasette/utils/actions_sql.py @@ -555,7 +555,7 @@ async def check_permissions_for_actions( params = {"_check_parent": parent, "_check_child": child} ctes = [] - selects = [] + result_rows = [] verdicts = {} for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)): @@ -627,10 +627,17 @@ async def check_permissions_for_actions( AND (r.child = :_check_child OR r.child IS NULL) )""" - selects.append(f"SELECT {i} AS action_idx, ({verdict_sql}) AS is_allowed") + result_rows.append(f"({i}, ({verdict_sql}))") - if selects: - query = "WITH\n" + ",\n".join(ctes) + "\n" + "\nUNION ALL\n".join(selects) + if result_rows: + ctes.append( + "results(action_idx, is_allowed) AS (VALUES\n" + + ",\n".join(result_rows) + + "\n)" + ) + query = ( + "WITH\n" + ",\n".join(ctes) + "\nSELECT action_idx, is_allowed FROM results" + ) result = await datasette.get_internal_database().execute(query, params) for row in result.rows: verdicts[unique_actions[row[0]]] = bool(row[1]) diff --git a/tests/test_allowed_many.py b/tests/test_allowed_many.py index 3d0d0c9a..08b952fb 100644 --- a/tests/test_allowed_many.py +++ b/tests/test_allowed_many.py @@ -12,6 +12,7 @@ import pytest import pytest_asyncio from datasette.app import Datasette from datasette.permissions import ( + Action, PermissionSQL, SkipPermissions, _permission_check_cache, @@ -541,6 +542,43 @@ async def test_allowed_many_skip_permission_checks(ds): assert results == {"view-table": True, "drop-table": True} +class ManyActionsPlugin: + """Registers enough actions to exceed SQLite's compound SELECT limit.""" + + def __init__(self, count): + self.action_names = [f"bulk-action-{i}" for i in range(count)] + self.action_names_set = set(self.action_names) + + @hookimpl + def register_actions(self, datasette): + return [ + Action(name=name, abbr=None, description="Bulk test action") + for name in self.action_names + ] + + @hookimpl + def permission_resources_sql(self, datasette, actor, action): + if action in self.action_names_set: + return PermissionSQL( + sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'bulk allow' AS reason", + params={}, + ) + + +@pytest.mark.asyncio +async def test_allowed_many_more_than_sqlite_compound_select_limit(): + plugin = ManyActionsPlugin(600) + ds = Datasette() + ds.pm.register(plugin, name="many_actions") + try: + await ds.invoke_startup() + results = await ds.allowed_many(actions=plugin.action_names, actor=None) + assert len(results) == 600 + assert all(results.values()) + finally: + ds.pm.unregister(name="many_actions") + + # ---------------------------------------------------------------------- # Layer 3: precompute before table_actions / database_actions hooks # ---------------------------------------------------------------------- From ab19b0382bc560ef7ae511f899aa7051935577af Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jun 2026 11:12:31 -0700 Subject: [PATCH 1115/1116] Removed note from permission_resources_sql Refs https://github.com/simonw/datasette/pull/2775/changes#r3408385197 --- docs/plugin_hooks.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 580f7402..2a0ddc93 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -1458,12 +1458,6 @@ to avoid conflicts with other plugins. The recommended convention is to prefix p plugin's source name (e.g., ``myplugin_user_id``). The system reserves these parameter names: ``:actor``, ``:actor_id``, ``:action``, and ``:filter_parent``. -This hook may be called for many actions in rapid succession - for example -:ref:`datasette.allowed_many() ` gathers rules for every action in its batch -concurrently before table and database pages render their action menus. Hook implementations must not -assume that checks for different actions arrive one page-render apart, and expensive work (such as -network calls) should be cached independently of the ``action`` argument where possible. - You can also use return ``PermissionSQL.allow(reason="reason goes here")`` or ``PermissionSQL.deny(reason="reason goes here")`` as shortcuts for simple root-level allow or deny rules. These will create SQL snippets that look like this: .. code-block:: sql From f2927a164746a1a2da3e14680948bfbdddfd626b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 13 Jun 2026 11:15:47 -0700 Subject: [PATCH 1116/1116] Fix for gen.throw(*sys.exc_info()) warning Closes #2776 --- datasette/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 6cd5d11e..e7fe1ed9 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -829,10 +829,10 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): # Execute the actual write try: result = fn(conn) - except Exception: + except Exception as e: # Throw exception into generator so it can handle it try: - gen.throw(*sys.exc_info()) + gen.throw(e) except StopIteration: pass # Re-raise the original exception