From 2638200d26b07701108fa6275e35c7c011535e4c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 15 Apr 2026 17:19:43 -0700 Subject: [PATCH 001/223] Link to datasette.io preview tool --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 635ca60e..5a109fda 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -296,7 +296,7 @@ Don't forget to create the release from the correct branch - usually ``main``, b While the release is running you can confirm that the correct commits made it into the release using the https://github.com/simonw/datasette/compare/0.64.6...0.64.7 URL. -Finally, post a news item about the release on `datasette.io `__ by editing the `news.yaml `__ file in that site's repository. +Finally, post a news item about the release on `datasette.io `__ by editing the `news.yaml `__ file in that site's repository. Use `this preview tool `__ to preview the edits to the YAML. .. _contributing_alpha_beta: From ade0ef8a60bad2a3e659d1cf1581bfe1fa96e289 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 19:14:32 -0700 Subject: [PATCH 002/223] Restore compatibility with existing execute_write_fn() callbacks Closes #2691 --- datasette/database.py | 19 +++++++++++++------ tests/test_internals_database.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 8b824462..7364ff7f 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,6 +1,7 @@ import asyncio import atexit from collections import namedtuple +import inspect import os from pathlib import Path import janus @@ -263,15 +264,21 @@ class Database: def _wrap_fn_with_hooks(self, fn, request, transaction, track_event): from .plugins import pm - # Wrap fn so it receives track_event if its signature supports it + # Wrap fn so it receives track_event if its signature supports it. + # Historically fn was called positionally, so any single-parameter + # name (conn, connection, db, ...) worked. Preserve that by only + # switching to keyword dependency injection when the callback + # explicitly opts in by declaring a `track_event` parameter. original_fn = fn - def fn_with_track_event(conn): - return call_with_supported_arguments( - original_fn, conn=conn, track_event=track_event - ) + if "track_event" in inspect.signature(original_fn).parameters: - fn = fn_with_track_event + def fn_with_track_event(conn): + return call_with_supported_arguments( + original_fn, conn=conn, track_event=track_event + ) + + fn = fn_with_track_event wrappers = pm.hook.write_wrapper( datasette=self.ds, diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index e3d35f57..0d565d61 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -539,6 +539,37 @@ async def test_execute_write_fn_exception(db): await db.execute_write_fn(write_fn) +@pytest.mark.asyncio +@pytest.mark.parametrize("param_name", ["conn", "connection", "db", "c"]) +async def test_execute_write_fn_accepts_any_single_param_name(db, param_name): + # Plugins historically relied on the fact that the callback was invoked + # positionally, so any parameter name worked. Preserve that contract. + scope = {} + exec( + "def write_fn({0}):\n" + " return {0}.execute('select 1 + 1').fetchone()[0]".format(param_name), + scope, + ) + write_fn = scope["write_fn"] + result = await db.execute_write_fn(write_fn) + assert result == 2 + + +@pytest.mark.asyncio +async def test_execute_write_fn_with_track_event(db): + # When the callback declares track_event it still receives both args + # via dependency injection. + seen = [] + + def write_fn(conn, track_event): + seen.append(track_event) + return conn.execute("select 1 + 1").fetchone()[0] + + result = await db.execute_write_fn(write_fn) + assert result == 2 + assert len(seen) == 1 and callable(seen[0]) + + @pytest.mark.asyncio @pytest.mark.timeout(1) async def test_execute_write_fn_connection_exception(tmpdir, app_client): From dabf8e4199cd4598697e538c495cc66aa429a262 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:08:46 -0700 Subject: [PATCH 003/223] Database.close() shuts down write thread and raises DatasetteClosedError After this commit, Database.close() sends a sentinel to the write queue so the background write thread exits cleanly, closes cached read/write connections, and marks the instance closed. Subsequent calls to execute*() raise DatasetteClosedError. close() remains idempotent and one-way. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) --- datasette/database.py | 79 +++++++++++++++++++++++++++++++- docs/internals.rst | 6 ++- tests/test_internals_database.py | 56 ++++++++++++++++++++++ 3 files changed, 138 insertions(+), 3 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 7364ff7f..e3c4bfec 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -34,6 +34,13 @@ connections = threading.local() AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) +class DatasetteClosedError(RuntimeError): + """Raised when using a Datasette or Database instance after close().""" + + +_SHUTDOWN = object() + + class Database: # For table counts stop at this many rows: count_limit = 10000 @@ -76,6 +83,7 @@ class Database: self._cached_table_counts = None self._write_thread = None self._write_queue = None + self._closed = False # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None @@ -84,6 +92,12 @@ class Database: if not is_temp_disk: self.mode = mode + def _check_not_closed(self): + if self._closed: + raise DatasetteClosedError( + "Database {!r} has been closed".format(self.name) + ) + @property def cached_table_counts(self): if self._cached_table_counts is not None: @@ -149,9 +163,53 @@ class Database: return conn def close(self): - # Close all connections - useful to avoid running out of file handles in tests + """Release all resources held by this database. + + Idempotent. After close() further calls to execute()/execute_fn()/ + execute_write()/execute_write_fn() raise DatasetteClosedError. + """ + if self._closed: + return + self._closed = True + # Shut down the write thread, if any, via a sentinel. The thread + # drains any writes already queued before the sentinel and then + # closes its own write connection and returns. + write_thread = self._write_thread + if write_thread is not None and self._write_queue is not None: + self._write_queue.put(_SHUTDOWN) + write_thread.join(timeout=10) + if write_thread.is_alive(): + sys.stderr.write( + "Datasette: write thread for {!r} did not exit within 10s\n".format( + self.name + ) + ) + sys.stderr.flush() + # Close anything still tracked in _all_file_connections for connection in self._all_file_connections: - connection.close() + try: + connection.close() + except Exception: + pass + self._all_file_connections = [] + # Drop per-thread cached read connections we can reach + try: + delattr(connections, self._thread_local_id) + except AttributeError: + pass + # Close non-threaded-mode cached connections if still open + if self._read_connection is not None: + try: + self._read_connection.close() + except Exception: + pass + self._read_connection = None + if self._write_connection is not None: + try: + self._write_connection.close() + except Exception: + pass + self._write_connection = None if self.is_temp_disk: self._cleanup_temp_file() @@ -164,6 +222,8 @@ class Database: pass async def execute_write(self, sql, params=None, block=True, request=None): + self._check_not_closed() + def _inner(conn): return conn.execute(sql, params or []) @@ -172,6 +232,8 @@ class Database: return results async def execute_write_script(self, sql, block=True, request=None): + self._check_not_closed() + def _inner(conn): return conn.executescript(sql) @@ -182,6 +244,8 @@ class Database: return results async def execute_write_many(self, sql, params_seq, block=True, request=None): + self._check_not_closed() + def _inner(conn): count = 0 @@ -203,6 +267,7 @@ class Database: return results async def execute_isolated_fn(self, fn): + self._check_not_closed() # Open a new connection just for the duration of this function # blocking the write queue to avoid any writes occurring during it if self.ds.executor is None: @@ -223,6 +288,7 @@ class Database: return await self._send_to_write_thread(fn, isolated_connection=True) async def execute_write_fn(self, fn, block=True, transaction=True, request=None): + self._check_not_closed() pending_events = [] def track_event(event): @@ -334,6 +400,13 @@ class Database: conn_exception = e while True: task = self._write_queue.get() + if task is _SHUTDOWN: + if conn is not None: + try: + conn.close() + except Exception: + pass + return if conn_exception is not None: result = conn_exception else: @@ -366,6 +439,7 @@ class Database: task.reply_queue.sync_q.put(result) async def execute_fn(self, fn): + self._check_not_closed() if self.ds.executor is None: # non-threaded mode if self._read_connection is None: @@ -396,6 +470,7 @@ class Database: log_sql_errors=True, ): """Executes sql against db_name in a thread""" + self._check_not_closed() page_size = page_size or self.ds.page_size def sql_operation_in_thread(conn): diff --git a/docs/internals.rst b/docs/internals.rst index ba9d3131..53c20106 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1830,7 +1830,11 @@ The return value of the function will be returned by this method. Any exceptions db.close() ---------- -Closes all of the open connections to file-backed databases. This is mainly intended to be used by large test suites, to avoid hitting limits on the number of open files. +Release all resources held by this ``Database`` instance. This shuts down the background write thread (if one was started by a previous call to :ref:`database_execute_write_fn` or similar), closes the write connection, and closes any cached read connections. + +After ``db.close()`` has been called, any further call to :ref:`database_execute`, :ref:`database_execute_fn`, :ref:`database_execute_write`, :ref:`database_execute_write_fn`, :ref:`database_execute_write_many`, :ref:`database_execute_write_script` or :ref:`database_execute_isolated_fn` will raise a ``datasette.database.DatasetteClosedError`` exception. + +``close()`` is idempotent — calling it a second time is a no-op. It is one-way: a closed ``Database`` cannot be reopened. .. _internals_database_introspection: diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 0d565d61..8ff74a83 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -4,6 +4,7 @@ Tests for the datasette.database.Database class from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues +from datasette.database import DatasetteClosedError from datasette.utils.sqlite import sqlite3, sqlite_version from datasette.utils import Column import pytest @@ -833,3 +834,58 @@ def test_repr_temp_disk(app_client): assert isinstance(db.size, int) assert isinstance(db.mtime_ns, int) db.close() + + +@pytest.mark.asyncio +async def test_database_close_shuts_down_write_thread(tmpdir): + path = str(tmpdir / "dbclose.db") + conn = sqlite3.connect(path) + conn.execute("create table t (id integer primary key)") + conn.close() + ds = Datasette([path]) + db = ds.get_database("dbclose") + # Trigger write thread creation + await db.execute_write("insert into t (id) values (1)") + assert db._write_thread is not None + assert db._write_thread.is_alive() + db.close() + # Wait briefly for the thread to exit — the sentinel should cause it to return. + db._write_thread.join(timeout=5) + assert not db._write_thread.is_alive() + ds._internal_database.close() + + +@pytest.mark.asyncio +async def test_database_close_raises_on_further_use(tmpdir): + path = str(tmpdir / "closed.db") + conn = sqlite3.connect(path) + conn.execute("create table t (id integer primary key)") + conn.close() + ds = Datasette([path]) + db = ds.get_database("closed") + await db.execute("select 1") + db.close() + with pytest.raises(DatasetteClosedError): + await db.execute("select 1") + with pytest.raises(DatasetteClosedError): + await db.execute_write("insert into t (id) values (1)") + with pytest.raises(DatasetteClosedError): + await db.execute_fn(lambda conn: conn.execute("select 1").fetchone()) + with pytest.raises(DatasetteClosedError): + await db.execute_write_fn(lambda conn: conn.execute("select 1")) + ds._internal_database.close() + + +@pytest.mark.asyncio +async def test_database_close_is_idempotent(tmpdir): + path = str(tmpdir / "idemp.db") + conn = sqlite3.connect(path) + conn.execute("create table t (id integer primary key)") + conn.close() + ds = Datasette([path]) + db = ds.get_database("idemp") + await db.execute_write("insert into t (id) values (1)") + db.close() + # Second call should be a no-op, not raise + db.close() + ds._internal_database.close() From 290f27158f1b53a9a90a36dbfb271ffbb6eef310 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:10:18 -0700 Subject: [PATCH 004/223] Datasette.close() closes databases, shuts down executor, unlinks temp file Datasette.close() iterates over every attached Database (including the internal database), calls Database.close() on each, then shuts down the ThreadPoolExecutor. Exceptions raised by one Database don't prevent the others from being closed; the first exception is re-raised afterwards. Idempotent. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) --- datasette/app.py | 28 +++++++++++++++ docs/internals.rst | 13 +++++++ tests/test_internals_datasette.py | 59 +++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 0f417ec9..367f38f9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -326,6 +326,7 @@ class Datasette: default_deny=False, ): self._startup_invoked = False + self._closed = False assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" @@ -834,6 +835,33 @@ class Datasette: new_databases.pop(name) self.databases = new_databases + def close(self): + """Release all resources held by this Datasette instance. + + Closes every attached Database (including the internal database), + shuts down the executor, and unlinks the temporary file used for + the internal database if one was created. Idempotent and one-way. + """ + if self._closed: + return + self._closed = True + first_exception = None + dbs = list(self.databases.values()) + [self._internal_database] + for db in dbs: + try: + db.close() + except Exception as e: + if first_exception is None: + first_exception = e + if self.executor is not None: + try: + self.executor.shutdown(wait=True, cancel_futures=True) + except Exception as e: + if first_exception is None: + first_exception = e + if first_exception is not None: + raise first_exception + def setting(self, key): return self._settings.get(key, None) diff --git a/docs/internals.rst b/docs/internals.rst index 53c20106..2710345b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1079,6 +1079,19 @@ The ``name`` and ``route`` parameters are optional and work the same way as they This removes a database that has been previously added. ``name=`` is the unique name of that database. +.. _datasette_close: + +.close() +-------- + +Release all resources held by this ``Datasette`` instance. This calls :ref:`database_close` on every attached database (including the internal database), shuts down the thread pool executor used to run SQL queries, and unlinks the temporary file used to back the internal database if one was created. + +``close()`` is synchronous, idempotent and one-way: after a call to ``close()`` any attempt to use the Datasette instance to execute SQL will raise a ``datasette.database.DatasetteClosedError`` exception. A closed ``Datasette`` cannot be reopened — callers that need a fresh instance should construct a new one. + +If a call to ``Database.close()`` on one of the attached databases raises an exception, ``Datasette.close()`` will continue trying to close the remaining databases and will re-raise the first exception after every database has been processed. + +When Datasette is being served over ASGI the ``close()`` method is wired up to the lifespan shutdown event, so resources are released cleanly on ``SIGTERM`` / ``SIGINT``. + .. _datasette_track_event: await .track_event(event) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index ec0180a7..5f773658 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -3,8 +3,10 @@ Tests for the datasette.app.Datasette class """ import dataclasses +import os from datasette import Context from datasette.app import Datasette, Database, ResourcesSQL +from datasette.database import DatasetteClosedError from datasette.resources import DatabaseResource from itsdangerous import BadSignature import pytest @@ -213,3 +215,60 @@ async def test_allowed_resources_sql(datasette): assert isinstance(result, ResourcesSQL) assert "all_rules AS" in result.sql assert result.params["action"] == "view-table" + + +@pytest.mark.asyncio +async def test_datasette_close_closes_all_databases_and_executor(): + ds = Datasette(memory=True) + await ds.invoke_startup() + # Confirm internal DB has write machinery running + assert ds._internal_database._write_thread is not None + assert ds._internal_database._write_thread.is_alive() + temp_path = ds._internal_database.path + assert os.path.exists(temp_path) + executor = ds.executor + ds.close() + # Executor is shut down + assert executor._shutdown + # All attached Database instances are closed + for db in ds.databases.values(): + assert db._closed + assert ds._internal_database._closed + # Temp internal DB file is unlinked + assert not os.path.exists(temp_path) + + +@pytest.mark.asyncio +async def test_datasette_close_is_idempotent(): + ds = Datasette(memory=True) + await ds.invoke_startup() + ds.close() + # Second call should be a no-op + ds.close() + + +@pytest.mark.asyncio +async def test_datasette_close_raises_on_use(): + ds = Datasette(memory=True) + await ds.invoke_startup() + ds.close() + with pytest.raises(DatasetteClosedError): + await ds.get_internal_database().execute("select 1") + + +@pytest.mark.asyncio +async def test_datasette_close_continues_past_db_error(): + # If one Database raises during close(), the others still get closed. + ds = Datasette(memory=True) + await ds.invoke_startup() + + class Boom(Database): + def close(self): + raise RuntimeError("boom") + + bad = ds.add_database(Boom(ds, is_memory=True), name="bad") + good = ds.add_database(Database(ds, is_memory=True), name="good") + with pytest.raises(RuntimeError, match="boom"): + ds.close() + assert good._closed + assert ds._internal_database._closed From d72dd3537850988fb24cd53d50c690dc7acb4332 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:11:02 -0700 Subject: [PATCH 005/223] Wire Datasette.close into ASGI lifespan shutdown AsgiLifespan now receives an on_shutdown callback that invokes Datasette.close(), so resources are released cleanly when the ASGI server delivers a lifespan.shutdown message (SIGTERM / SIGINT for uvicorn). Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) --- datasette/app.py | 5 ++++- tests/test_internals_datasette.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/datasette/app.py b/datasette/app.py index 367f38f9..358081ef 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2338,10 +2338,13 @@ class Datasette: if not database.is_mutable: await database.table_counts(limit=60 * 60 * 1000) + async def _close_on_shutdown(): + self.close() + asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan(asgi) + asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 5f773658..11463eda 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -256,6 +256,29 @@ async def test_datasette_close_raises_on_use(): await ds.get_internal_database().execute("select 1") +@pytest.mark.asyncio +async def test_asgi_lifespan_shutdown_closes_datasette(): + ds = Datasette(memory=True) + app = ds.app() + # Drive an ASGI lifespan: startup, then shutdown. + messages_sent = [] + inbox = [ + {"type": "lifespan.startup"}, + {"type": "lifespan.shutdown"}, + ] + + async def receive(): + return inbox.pop(0) + + async def send(message): + messages_sent.append(message) + + await app({"type": "lifespan"}, receive, send) + assert {"type": "lifespan.startup.complete"} in messages_sent + assert {"type": "lifespan.shutdown.complete"} in messages_sent + assert ds._closed + + @pytest.mark.asyncio async def test_datasette_close_continues_past_db_error(): # If one Database raises during close(), the others still get closed. From 34cc320eabb09d7d62f8a6045b868c746adfe9d2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:15:50 -0700 Subject: [PATCH 006/223] Pytest auto-close plugin for Datasette instances Installs a pytest11 entry point so that every Datasette() constructed inside a pytest_runtest_call phase is auto-closed at the end of the test. Fixture-scoped instances are untouched. Opt out via the datasette_autoclose = false ini option. This gives large test suites a safety net against FD exhaustion and leaked write threads from the now-default temp-disk internal database without requiring every existing test to be rewritten. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) --- datasette/_pytest_plugin.py | 78 ++++++++++++++++++++++ docs/testing_plugins.rst | 16 +++++ pyproject.toml | 3 + tests/test_pytest_autoclose_plugin.py | 93 +++++++++++++++++++++++++++ 4 files changed, 190 insertions(+) create mode 100644 datasette/_pytest_plugin.py create mode 100644 tests/test_pytest_autoclose_plugin.py diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py new file mode 100644 index 00000000..3f6c0d96 --- /dev/null +++ b/datasette/_pytest_plugin.py @@ -0,0 +1,78 @@ +""" +Pytest plugin that automatically closes any Datasette instances constructed +inside a test body. Fixture-scoped instances survive. + +Registered as a pytest11 entry point in pyproject.toml so that downstream +projects using Datasette get the same FD-safety net for their own tests. + +Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the +equivalent ini file). +""" + +from __future__ import annotations + +import contextvars +import weakref + +import pytest + +from datasette.app import Datasette + +_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar( + "datasette_active_instances", default=None +) + +_original_init = Datasette.__init__ + + +def _tracking_init(self, *args, **kwargs): + _original_init(self, *args, **kwargs) + instances = _active_instances.get() + if instances is not None: + instances.append(weakref.ref(self)) + + +Datasette.__init__ = _tracking_init + + +def pytest_addoption(parser): + parser.addini( + "datasette_autoclose", + help=( + "Automatically close Datasette instances created inside test " + "bodies (default: true)." + ), + default="true", + ) + + +def _enabled(config) -> bool: + value = config.getini("datasette_autoclose") + if isinstance(value, bool): + return value + return str(value).strip().lower() not in ("false", "0", "no", "off") + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_call(item): + if not _enabled(item.config): + yield + return + refs: list[weakref.ref] = [] + token = _active_instances.set(refs) + try: + yield + finally: + _active_instances.reset(token) + for ref in reversed(refs): + ds = ref() + if ds is None: + continue + try: + ds.close() + except Exception as e: + item.warn( + pytest.PytestUnraisableExceptionWarning( + f"Error closing Datasette instance: {e!r}" + ) + ) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 1b10c132..070ab6cf 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -82,6 +82,22 @@ This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepar If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request. +.. _testing_plugins_autoclose: + +Automatic cleanup of Datasette instances +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Installing Datasette also installs a small pytest plugin that automatically calls :ref:`datasette_close` on any ``Datasette()`` instance constructed inside the body of a test function. This helps prevent large test suites from running out of file descriptors or leaking background threads from the hundreds of instances they may build up across a session. + +Instances created inside a pytest fixture are **not** closed by this plugin — pytest fixtures often create a single ``Datasette`` that is shared across many tests, and closing it automatically would break those tests. If you need a per-test instance and want to share it between multiple tests, create it inside a fixture rather than at the top level of a test function. + +If you need to opt out of this behavior, add the following to your ``pytest.ini`` (or equivalent): + +.. code-block:: ini + + [pytest] + datasette_autoclose = false + .. _testing_datasette_client: Using datasette.client in tests diff --git a/pyproject.toml b/pyproject.toml index a0ee050c..4a4ed75e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,9 @@ CI = "https://github.com/simonw/datasette/actions?query=workflow%3ATest" [project.scripts] datasette = "datasette.cli:cli" +[project.entry-points.pytest11] +datasette = "datasette._pytest_plugin" + [dependency-groups] dev = [ "pytest>=9", diff --git a/tests/test_pytest_autoclose_plugin.py b/tests/test_pytest_autoclose_plugin.py new file mode 100644 index 00000000..78154ef5 --- /dev/null +++ b/tests/test_pytest_autoclose_plugin.py @@ -0,0 +1,93 @@ +""" +Tests for datasette._pytest_plugin — the pytest plugin that auto-closes +Datasette instances constructed inside test bodies. + +These tests drive a real pytest session in a subprocess so the plugin +operates exactly as it would for a downstream consumer. +""" + +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parent.parent + + +def _run_pytest(tmp_path: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-m", "pytest", "-v", str(tmp_path)], + cwd=str(tmp_path), + capture_output=True, + text=True, + ) + + +def test_auto_close_of_instances_made_in_test_body(tmp_path): + # Two ordered tests: + # test_a makes a Datasette() and stashes a hard reference + # test_b asserts that the hard-reffed instance was closed by the plugin + (tmp_path / "test_sample.py").write_text(textwrap.dedent(""" + from datasette.app import Datasette + + _stash = {} + + def test_a(): + ds = Datasette(memory=True) + _stash["ds"] = ds + assert ds._closed is False + + def test_b(): + assert _stash["ds"]._closed is True + """)) + result = _run_pytest(tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_fixture_scoped_instance_is_not_closed(tmp_path): + # A module-scoped fixture instance must survive across tests in the module. + (tmp_path / "test_fixture.py").write_text(textwrap.dedent(""" + import pytest + from datasette.app import Datasette + + @pytest.fixture(scope="module") + def ds(): + return Datasette(memory=True) + + def test_first(ds): + assert ds._closed is False + + def test_second(ds): + # Still alive because the plugin only tracks instances + # constructed during pytest_runtest_call, not during fixture + # setup. + assert ds._closed is False + """)) + result = _run_pytest(tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_opt_out_via_ini(tmp_path): + # datasette_autoclose = false should leave instances untouched. + (tmp_path / "pytest.ini").write_text(textwrap.dedent(""" + [pytest] + datasette_autoclose = false + """).strip()) + (tmp_path / "test_optout.py").write_text(textwrap.dedent(""" + from datasette.app import Datasette + + _stash = {} + + def test_a(): + ds = Datasette(memory=True) + _stash["ds"] = ds + + def test_b(): + # Opt-out: plugin must not have closed it. + assert _stash["ds"]._closed is False + _stash["ds"].close() + """)) + result = _run_pytest(tmp_path) + assert result.returncode == 0, result.stdout + result.stderr From c0153386ef20126a289da96204718570d571b4b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:18:05 -0700 Subject: [PATCH 007/223] FD-leak regression test for Datasette.close() Creates and disposes 50 Datasette instances in a loop and asserts that the number of open file descriptors and live threads does not grow, exercising the full close() path end to end. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) --- pyproject.toml | 1 + tests/test_fd_leak.py | 56 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tests/test_fd_leak.py diff --git a/pyproject.toml b/pyproject.toml index 4a4ed75e..e6007afd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ dev = [ "myst-parser", "sphinx-markdown-builder", "ruamel.yaml", + "psutil>=5.9", ] [project.optional-dependencies] diff --git a/tests/test_fd_leak.py b/tests/test_fd_leak.py new file mode 100644 index 00000000..926722a1 --- /dev/null +++ b/tests/test_fd_leak.py @@ -0,0 +1,56 @@ +""" +Regression test for https://github.com/simonw/datasette/issues/2692 — +confirm that creating and closing Datasette instances in a loop does not +leak open file descriptors. + +Each Datasette() with is_temp_disk internal DB opens a temp file and a +write thread with its own SQLite connection. Without Datasette.close() +nothing unwinds this state, and a large pytest run exhausts the process +FD limit. +""" + +import asyncio +import threading + +import pytest + +try: + import psutil +except ImportError: # pragma: no cover + psutil = None + +from datasette.app import Datasette + + +def _count_open_files(): + return len(psutil.Process().open_files()) + + +def _count_threads(): + return threading.active_count() + + +@pytest.mark.skipif(psutil is None, reason="psutil not installed") +def test_close_releases_file_descriptors(): + # Warm-up so Python/library caches don't skew the baseline + ds = Datasette(memory=True) + asyncio.run(ds.invoke_startup()) + ds.close() + + baseline_fds = _count_open_files() + baseline_threads = _count_threads() + + for _ in range(50): + ds = Datasette(memory=True) + asyncio.run(ds.invoke_startup()) + ds.close() + + after_fds = _count_open_files() + after_threads = _count_threads() + + assert ( + after_fds - baseline_fds <= 2 + ), f"Leaked FDs: baseline={baseline_fds}, after=50 iterations={after_fds}" + assert ( + after_threads - baseline_threads <= 2 + ), f"Leaked threads: baseline={baseline_threads}, after={after_threads}" From d23b32c3e57469f0c0de149aee8594205dfdb319 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:25:58 -0700 Subject: [PATCH 008/223] Call ds.close() in more places in tests Refs #2692 --- tests/test_api_write.py | 7 +------ tests/test_column_types.py | 10 ++-------- tests/test_docs_plugins.py | 1 + tests/test_write_wrapper.py | 1 + 4 files changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 9ba08848..64f91701 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -22,12 +22,7 @@ def ds_write(tmp_path_factory): ds = Datasette([db_path], immutables=[db_path_immutable]) ds.root_enabled = True yield ds - # Close both setup connections plus any Datasette-managed connections. - db1.close() - db2.close() - for database in ds.databases.values(): - if not database.is_memory: - database.close() + ds.close() def write_token(ds, actor_id="root", permissions=None): diff --git a/tests/test_column_types.py b/tests/test_column_types.py index 6e89acb9..d77f2cf5 100644 --- a/tests/test_column_types.py +++ b/tests/test_column_types.py @@ -52,10 +52,7 @@ def ds_ct(tmp_path_factory): ) ds.root_enabled = True yield ds - db.close() - for database in ds.databases.values(): - if not database.is_memory: - database.close() + ds.close() @pytest.fixture @@ -95,10 +92,7 @@ def ds_ct_editor_permission(tmp_path_factory): ) ds.root_enabled = True yield ds - db.close() - for database in ds.databases.values(): - if not database.is_memory: - database.close() + ds.close() def write_token(ds, actor_id="root", permissions=None): diff --git a/tests/test_docs_plugins.py b/tests/test_docs_plugins.py index c51858d3..613160ac 100644 --- a/tests/test_docs_plugins.py +++ b/tests/test_docs_plugins.py @@ -23,6 +23,7 @@ async def datasette_with_plugin(): yield datasette finally: datasette.pm.unregister(name="undo") + datasette.close() # -- end datasette_with_plugin_fixture -- diff --git a/tests/test_write_wrapper.py b/tests/test_write_wrapper.py index c2ceb344..48c964b4 100644 --- a/tests/test_write_wrapper.py +++ b/tests/test_write_wrapper.py @@ -505,6 +505,7 @@ def ds_with_event_tracking(tmp_path): ds.track_event = recording_track_event yield ds + ds.close() @pytest.mark.asyncio From df96e12737454077c707f469506be0ee96091965 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:32:19 -0700 Subject: [PATCH 009/223] Auto-close Datasette instances from function-scoped fixtures too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plugin now tracks instances across the full test protocol (setup, call, teardown) and closes all of them at the end — including ones created inside function-scoped pytest fixtures. Session-, module-, class- and package-scoped fixtures are still exempted by subtracting any instances their setup adds from the tracking list. This makes downstream projects like datasette-alerts work at low FD limits without every fixture needing an explicit ds.close() call. Refs #2692 See https://github.com/simonw/datasette/issues/2692#issuecomment-4265072230 Co-Authored-By: Claude Opus 4.7 (1M context) --- datasette/_pytest_plugin.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/datasette/_pytest_plugin.py b/datasette/_pytest_plugin.py index 3f6c0d96..5fb6b473 100644 --- a/datasette/_pytest_plugin.py +++ b/datasette/_pytest_plugin.py @@ -1,6 +1,9 @@ """ Pytest plugin that automatically closes any Datasette instances constructed -inside a test body. Fixture-scoped instances survive. +during a pytest test — both in the test body and in function-scoped +fixtures. Instances constructed by session-, module-, class- or package- +scoped fixtures are left alone, because other tests in the session will +still want to use them. Registered as a pytest11 entry point in pyproject.toml so that downstream projects using Datasette get the same FD-safety net for their own tests. @@ -40,7 +43,7 @@ def pytest_addoption(parser): "datasette_autoclose", help=( "Automatically close Datasette instances created inside test " - "bodies (default: true)." + "bodies and function-scoped fixtures (default: true)." ), default="true", ) @@ -54,7 +57,8 @@ def _enabled(config) -> bool: @pytest.hookimpl(hookwrapper=True) -def pytest_runtest_call(item): +def pytest_runtest_protocol(item, nextitem): + """Track Datasette instances across setup, call and teardown; close at end.""" if not _enabled(item.config): yield return @@ -76,3 +80,29 @@ def pytest_runtest_call(item): f"Error closing Datasette instance: {e!r}" ) ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_setup(fixturedef, request): + """Exempt instances created by non-function-scoped fixtures. + + Session-, module-, class- and package-scoped fixtures produce Datasette + instances that must survive beyond the current test — other tests in + the session will still use them. When such a fixture creates one or + more Datasette instances during its setup, we snapshot the tracking + list before the fixture runs and subtract off any instances that were + added during its setup, so they don't get closed at test teardown. + """ + refs = _active_instances.get() + if refs is None: + yield + return + before_ids = {id(ref) for ref in refs} + yield + if fixturedef.scope != "function": + new_refs = [ref for ref in refs if id(ref) not in before_ids] + for new_ref in new_refs: + try: + refs.remove(new_ref) + except ValueError: + pass From ede942a32e65191ccf554d481987f2d42f4a9a92 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:34:48 -0700 Subject: [PATCH 010/223] Fix ruff lints in close-related tests Drop unused `bad = ...` assignment and unused `import pytest`. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/test_internals_datasette.py | 2 +- tests/test_pytest_autoclose_plugin.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index 11463eda..d58c9a29 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -289,7 +289,7 @@ async def test_datasette_close_continues_past_db_error(): def close(self): raise RuntimeError("boom") - bad = ds.add_database(Boom(ds, is_memory=True), name="bad") + ds.add_database(Boom(ds, is_memory=True), name="bad") good = ds.add_database(Database(ds, is_memory=True), name="good") with pytest.raises(RuntimeError, match="boom"): ds.close() diff --git a/tests/test_pytest_autoclose_plugin.py b/tests/test_pytest_autoclose_plugin.py index 78154ef5..3af1aace 100644 --- a/tests/test_pytest_autoclose_plugin.py +++ b/tests/test_pytest_autoclose_plugin.py @@ -11,8 +11,6 @@ import sys import textwrap from pathlib import Path -import pytest - REPO_ROOT = Path(__file__).parent.parent From 03eeeb9d92e3821611931d9fa259811d95b646e8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:38:08 -0700 Subject: [PATCH 011/223] Docs: auto-close plugin now handles function-scoped fixtures Describe the updated scoping rule: instances from test bodies and function-scoped fixtures are closed automatically; session-, module-, class- and package-scoped fixtures are exempt. Refs #2692 --- docs/testing_plugins.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/testing_plugins.rst b/docs/testing_plugins.rst index 070ab6cf..b82a6e0c 100644 --- a/docs/testing_plugins.rst +++ b/docs/testing_plugins.rst @@ -87,9 +87,18 @@ If you are using ``await datasette.client.get()`` and similar methods then you d Automatic cleanup of Datasette instances ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Installing Datasette also installs a small pytest plugin that automatically calls :ref:`datasette_close` on any ``Datasette()`` instance constructed inside the body of a test function. This helps prevent large test suites from running out of file descriptors or leaking background threads from the hundreds of instances they may build up across a session. +Installing Datasette also installs a small pytest plugin that automatically calls :ref:`datasette_close` on any ``Datasette()`` instance constructed during a test. This helps prevent large test suites from running out of file descriptors or leaking background threads from the hundreds of instances they may build up across a session. -Instances created inside a pytest fixture are **not** closed by this plugin — pytest fixtures often create a single ``Datasette`` that is shared across many tests, and closing it automatically would break those tests. If you need a per-test instance and want to share it between multiple tests, create it inside a fixture rather than at the top level of a test function. +The plugin closes: + +- Instances created in the body of a test function. +- Instances created inside **function-scoped** pytest fixtures (the default scope — ``@pytest.fixture`` with no ``scope=`` argument, or ``scope="function"``). + +The plugin deliberately does **not** close: + +- Instances created inside higher-scoped fixtures (``scope="session"``, ``"module"``, ``"class"`` or ``"package"``). Those fixtures are typically designed to produce a single ``Datasette`` that is shared across many tests, and closing it automatically would break the tests that run after the first. + +In practice this means downstream projects rarely need to call ``ds.close()`` themselves — function-scoped fixtures and inline test code are both covered automatically, while long-lived shared fixtures keep working as before. If you need to opt out of this behavior, add the following to your ``pytest.ini`` (or equivalent): From c9a7dc9be21f586e86ae09e3e55376c5f9a5fd25 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:40:51 -0700 Subject: [PATCH 012/223] Declare ds_client as session-scoped so auto-close plugin spares it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ds_client already caches a single Datasette for the whole session via a module-level _ds_client global, so the declared fixture scope should match. With function scope the auto-close plugin correctly closes it after the first test that uses it, which then breaks every subsequent test that reuses the cached (now-closed) instance — as seen in the CI coverage job, which runs serially rather than under pytest-xdist. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3a3203fd..171a5433 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -53,7 +53,7 @@ def bare_ds(): return Datasette(memory=True) -@pytest_asyncio.fixture +@pytest_asyncio.fixture(scope="session") async def ds_client(): from datasette.app import Datasette from datasette.database import Database From b3001c1e5a5d5d5b2a04daf4a7445023f24806d5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:41:58 -0700 Subject: [PATCH 013/223] Drop redundant _ds_client global now that ds_client is session-scoped Session-scoped fixtures are cached per worker by pytest itself, so the manual _ds_client module global is no longer needed. Refs #2692 Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/conftest.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 171a5433..5f1cc587 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,9 +28,6 @@ UNDOCUMENTED_PERMISSIONS = { "view_document", } -_ds_client = None - - def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): start = time.time() while time.time() - start < timeout: @@ -60,10 +57,6 @@ async def ds_client(): from .fixtures import CONFIG, METADATA, PLUGINS_DIR import secrets - global _ds_client - if _ds_client is not None: - return _ds_client - ds = Datasette( metadata=METADATA, config=CONFIG, @@ -95,8 +88,7 @@ async def ds_client(): await db.execute_write_fn(prepare) await ds.invoke_startup() - _ds_client = ds.client - return _ds_client + return ds.client def pytest_report_header(config): From 630e557cdb7cce7ac05b4b3f8067990211e5477c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:44:21 -0700 Subject: [PATCH 014/223] Ran black --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 5f1cc587..4ea89458 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,7 @@ UNDOCUMENTED_PERMISSIONS = { "view_document", } + def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): start = time.time() while time.time() - start < timeout: From a6031c98476487ec2aa5830bea75df9e6615262d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:59:21 -0700 Subject: [PATCH 015/223] Release 1.0a28 Refs #2691, #2692, #2693 --- datasette/version.py | 2 +- docs/changelog.rst | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index e2c80e50..cf908bb2 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a27" +__version__ = "1.0a28" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9cd7a7d6..7c1da152 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v1_0_a28: + +1.0a28 (2026-04-16) +------------------- + +- Fixed a compatibility bug introduced in 1.0a27 where ``execute_write_fn()`` callbacks with a parameter name other than ``conn`` were seeing errors. (:issue:`2691`) +- The :ref:`database.close() ` method now also shuts down the write connection for that database. +- New :ref:`datasette.close() ` method for closing down all databases and resources associated with a Datasette instance. This is called automatically when the server shuts down. (:pr:`2693`) +- Datasette now includes a pytest plugin which automatically calls ``datasette.close()`` on temporary instances created in function-scoped fixtures and during tests. See :ref:`testing_plugins_autoclose` for details. This helps avoid running out of file descriptors in plugin test suites that were written before the ``Database(is_temp_disk=True)`` feature introduced in Datasette 1.0a27. (:issue:`2692`) + .. _v1_0_a27: 1.0a27 (2026-04-15) From b15ce18ddc463a537a52879381ad929d1867143d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 17 Apr 2026 08:44:43 -0700 Subject: [PATCH 016/223] TokenRestrictions.abbreviated(datasette) utility method for creating _r dicts (#2696) Closes #2695 Refs https://github.com/simonw/datasette-auth-tokens/pull/42 --- datasette/tokens.py | 59 ++++++++++++++++++++++--------------- docs/internals.rst | 24 +++++++++++++++ tests/test_token_handler.py | 37 +++++++++++++++++++++++ 3 files changed, 97 insertions(+), 23 deletions(-) diff --git a/datasette/tokens.py b/datasette/tokens.py index 5a12d8e0..38a55529 100644 --- a/datasette/tokens.py +++ b/datasette/tokens.py @@ -52,6 +52,38 @@ class TokenRestrictions: self.resource.setdefault(database, {}).setdefault(resource, []).append(action) return self + def abbreviated(self, datasette: "Datasette") -> Optional[dict]: + """ + Return the abbreviated ``_r`` dictionary shape for this set of + restrictions, using action abbreviations registered with ``datasette``. + Returns ``None`` if no restrictions are set. + """ + if not (self.all or self.database or self.resource): + return None + + def abbreviate_action(action): + action_obj = datasette.actions.get(action) + if not action_obj: + return action + return action_obj.abbr or action + + result: dict = {} + if self.all: + result["a"] = [abbreviate_action(a) for a in self.all] + if self.database: + result["d"] = { + database: [abbreviate_action(a) for a in actions] + for database, actions in self.database.items() + } + if self.resource: + result["r"] = {} + for database, resources in self.resource.items(): + for resource, actions in resources.items(): + result["r"].setdefault(database, {})[resource] = [ + abbreviate_action(a) for a in actions + ] + return result + class TokenHandler: """ @@ -104,31 +136,12 @@ class SignedTokenHandler(TokenHandler): token = {"a": actor_id, "t": int(time.time())} - def abbreviate_action(action): - action_obj = datasette.actions.get(action) - if not action_obj: - return action - return action_obj.abbr or action - if expires_after: token["d"] = expires_after - if restrictions and ( - restrictions.all or restrictions.database or restrictions.resource - ): - token["_r"] = {} - if restrictions.all: - token["_r"]["a"] = [abbreviate_action(a) for a in restrictions.all] - if restrictions.database: - token["_r"]["d"] = {} - for database, actions in restrictions.database.items(): - token["_r"]["d"][database] = [abbreviate_action(a) for a in actions] - if restrictions.resource: - token["_r"]["r"] = {} - for database, resources in restrictions.resource.items(): - for resource, actions in resources.items(): - token["_r"]["r"].setdefault(database, {})[resource] = [ - abbreviate_action(a) for a in actions - ] + if restrictions is not None: + abbreviated = restrictions.abbreviated(datasette) + if abbreviated is not None: + token["_r"] = abbreviated return "dstok_{}".format(datasette.sign(token, namespace="token")) async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: diff --git a/docs/internals.rst b/docs/internals.rst index 2710345b..e0123a7b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -729,6 +729,30 @@ The builder methods are: Each method returns the ``TokenRestrictions`` instance so calls can be chained. +``TokenRestrictions`` also provides an ``abbreviated(datasette)`` method which returns the restrictions as a dictionary using the compact format described in :ref:`authentication_cli_create_token_restrict`, with action names replaced by their registered abbreviations. It returns the inner dictionary only - the ``"_r"`` wrapping key shown in that section is not included. Returns ``None`` if no restrictions are set. This is useful when writing a custom :ref:`plugin_hook_register_token_handler` that needs to embed restrictions in a token payload. + +For example, the following restrictions: + +.. code-block:: python + + restrictions = ( + TokenRestrictions() + .allow_all("view-instance") + .allow_database("docs", "view-query") + .allow_resource("docs", "attachments", "insert-row") + ) + restrictions.abbreviated(datasette) + +Returns this dictionary, using the abbreviations registered for each action: + +.. code-block:: python + + { + "a": ["vi"], + "d": {"docs": ["vq"]}, + "r": {"docs": {"attachments": ["ir"]}}, + } + The following example creates a token that can access ``view-instance`` and ``view-table`` across everything, can additionally use ``view-query`` for anything in the ``docs`` database and is allowed to execute ``insert-row`` and ``update-row`` in the ``attachments`` table in that database: .. code-block:: python diff --git a/tests/test_token_handler.py b/tests/test_token_handler.py index 83f09046..5c87f577 100644 --- a/tests/test_token_handler.py +++ b/tests/test_token_handler.py @@ -291,6 +291,43 @@ async def test_expires_after_round_trip(datasette): assert "token_expires" in actor +@pytest.mark.asyncio +@pytest.mark.parametrize( + "build_restrictions,expected", + [ + (lambda r: r, None), + (lambda r: r.allow_all("view-instance"), {"a": ["vi"]}), + ( + lambda r: r.allow_database("docs", "view-query"), + {"d": {"docs": ["vq"]}}, + ), + ( + lambda r: r.allow_resource("docs", "attachments", "insert-row"), + {"r": {"docs": {"attachments": ["ir"]}}}, + ), + ( + lambda r: r.allow_all("view-instance") + .allow_database("docs", "view-query") + .allow_resource("docs", "attachments", "insert-row"), + { + "a": ["vi"], + "d": {"docs": ["vq"]}, + "r": {"docs": {"attachments": ["ir"]}}, + }, + ), + ( + lambda r: r.allow_all("not-a-real-action"), + {"a": ["not-a-real-action"]}, + ), + ], + ids=["empty", "all", "database", "resource", "combined", "unknown_action"], +) +async def test_token_restrictions_abbreviated(datasette, build_restrictions, expected): + await datasette.invoke_startup() + restrictions = build_restrictions(TokenRestrictions()) + assert restrictions.abbreviated(datasette) == expected + + @pytest.mark.asyncio async def test_signed_tokens_disabled(): """create_token and verify_token should fail/skip when signed tokens are disabled.""" From 0dc7bb19d9a95df9d9c6bd00e943d407fc11f49e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 22 Apr 2026 22:22:47 -0700 Subject: [PATCH 017/223] Table headers and column options visible for 0 rows Closes #2701 --- datasette/templates/_table.html | 5 +++-- datasette/templates/table.html | 2 -- tests/test_html.py | 2 +- tests/test_table_html.py | 26 ++++++++++++++++++++++++-- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index ba34b60f..f47a325f 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -1,6 +1,6 @@
-{% if display_rows %} +{% if display_columns %}
@@ -31,6 +31,7 @@
-{% else %} +{% endif %} +{% if not display_rows %}

0 records

{% endif %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 2919d306..c841e1be 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -141,7 +141,6 @@ {% if all_columns %} -{% if display_rows %} -{% endif %} diff --git a/tests/test_html.py b/tests/test_html.py index e38898da..7425692d 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -805,7 +805,7 @@ async def test_blob_download_invalid_messages(ds_client, path, expected_message) async def test_zero_results(ds_client, path): response = await ds_client.get(path) soup = Soup(response.text, "html.parser") - assert 0 == len(soup.select("table")) + assert 0 == len(soup.select("table tbody tr")) assert 1 == len(soup.select("p.zero-results")) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index d8dde593..86b9a4eb 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -752,8 +752,11 @@ async def test_column_chooser_present(ds_client): @pytest.mark.asyncio -async def test_mobile_column_actions_present(ds_client): - response = await ds_client.get("/fixtures/facetable") +@pytest.mark.parametrize( + "path", ["/fixtures/facetable", "/fixtures/123_starts_with_digits"] +) +async def test_mobile_column_actions_present(ds_client, path): + response = await ds_client.get(path) assert response.status_code == 200 soup = Soup(response.text, "html.parser") button = soup.select_one("button.column-actions-mobile.small-screen-only") @@ -764,6 +767,25 @@ async def test_mobile_column_actions_present(ds_client): "mobile-column-actions.js" in (script.get("src") or "") for script in soup.find_all("script") ) + # mobile-column-actions.js builds its dialog from elements, + # so the thead must render even when the table has no rows. + ths = soup.select("table.rows-and-columns thead th[data-column]") + assert len(ths) >= 1 + + +@pytest.mark.asyncio +async def test_zero_row_table_renders_thead(ds_client): + response = await ds_client.get("/fixtures/123_starts_with_digits") + assert response.status_code == 200 + soup = Soup(response.text, "html.parser") + table = soup.select_one("table.rows-and-columns") + assert table is not None + column_names = [ + th.get("data-column") for th in table.select("thead th[data-column]") + ] + assert "content" in column_names + assert table.select_one("tbody tr") is None + assert soup.select_one("p.zero-results") is not None @pytest.mark.asyncio From aa84fe008d7c6263bd8712adcfe9be53a9f207ea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 May 2026 16:05:12 -0700 Subject: [PATCH 018/223] Fix for column actions on Mobile Safari, closes #2708 --- datasette/static/app.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 26717c43..1ce84bc8 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -818,7 +818,8 @@ dialog.mobile-column-actions-dialog::backdrop { } .mobile-column-actions-dialog .list-wrap { - flex: 1; + flex: 1 1 auto; + min-height: 0; overflow-y: auto; overflow-x: hidden; position: relative; From 345f910043bebd4fb829c1f8c248a17e38856191 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 16:31:36 -0700 Subject: [PATCH 019/223] Fix for Database.close()/Datasette.close() order (#2710) Closes: - #2709 The key behavior change: after close() starts, no new execute work can be submitted, but already-running execute work is allowed to finish before SQLite connections are closed. --- datasette/database.py | 26 +++++++++++++--- datasette/version.py | 2 +- tests/test_internals_datasette.py | 49 +++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index e3c4bfec..657adfa5 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -84,6 +84,8 @@ class Database: self._write_thread = None self._write_queue = None self._closed = False + self._pending_execute_futures = set() + self._pending_execute_futures_lock = threading.Lock() # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None @@ -98,6 +100,10 @@ class Database: "Database {!r} has been closed".format(self.name) ) + def _remove_pending_execute_future(self, future): + with self._pending_execute_futures_lock: + self._pending_execute_futures.discard(future) + @property def cached_table_counts(self): if self._cached_table_counts is not None: @@ -170,7 +176,11 @@ class Database: """ if self._closed: return - self._closed = True + with self._pending_execute_futures_lock: + if self._closed: + return + self._closed = True + pending_execute_futures = tuple(self._pending_execute_futures) # Shut down the write thread, if any, via a sentinel. The thread # drains any writes already queued before the sentinel and then # closes its own write connection and returns. @@ -185,6 +195,11 @@ class Database: ) ) sys.stderr.flush() + for future in pending_execute_futures: + try: + future.result() + except Exception: + pass # Close anything still tracked in _all_file_connections for connection in self._all_file_connections: try: @@ -456,9 +471,12 @@ class Database: setattr(connections, self._thread_local_id, conn) return fn(conn) - return await asyncio.get_event_loop().run_in_executor( - self.ds.executor, in_thread - ) + with self._pending_execute_futures_lock: + self._check_not_closed() + future = self.ds.executor.submit(in_thread) + self._pending_execute_futures.add(future) + future.add_done_callback(self._remove_pending_execute_future) + return await asyncio.wrap_future(future) async def execute( self, diff --git a/datasette/version.py b/datasette/version.py index cf908bb2..898d388c 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a28" +__version__ = "1.0a28.post1" __version_info__ = tuple(__version__.split(".")) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index d58c9a29..3f867eb0 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -2,8 +2,11 @@ Tests for the datasette.app.Datasette class """ +import asyncio import dataclasses import os +import sqlite3 +import time from datasette import Context from datasette.app import Datasette, Database, ResourcesSQL from datasette.database import DatasetteClosedError @@ -256,6 +259,52 @@ async def test_datasette_close_raises_on_use(): await ds.get_internal_database().execute("select 1") +async def _datasette_with_sleeping_execute(tmp_path, sleep_ms=200): + db_path = tmp_path / "data.db" + internal_path = tmp_path / "internal.db" + sqlite3.connect(db_path).close() + ds = Datasette([str(db_path)], internal=str(internal_path)) + loop = asyncio.get_running_loop() + sql_started = asyncio.Event() + original_prepare_connection = ds._prepare_connection + + def prepare_connection(conn, name): + original_prepare_connection(conn, name) + + def sleep_ms(ms): + loop.call_soon_threadsafe(sql_started.set) + time.sleep(ms / 1000) + return ms + + conn.create_function("sleep_ms", 1, sleep_ms) + + ds._prepare_connection = prepare_connection + task = asyncio.create_task( + ds.get_database().execute( + f"select sleep_ms({sleep_ms})", custom_time_limit=1000 + ) + ) + await asyncio.wait_for(sql_started.wait(), timeout=5) + return ds, task + + +@pytest.mark.asyncio +async def test_datasette_close_waits_for_in_flight_execute(tmp_path): + ds, task = await _datasette_with_sleeping_execute(tmp_path) + ds.close() + results = await task + assert [tuple(row) for row in results.rows] == [(200,)] + + +@pytest.mark.asyncio +async def test_datasette_close_waits_for_cancelled_in_flight_execute(tmp_path): + ds, task = await _datasette_with_sleeping_execute(tmp_path) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + ds.close() + + @pytest.mark.asyncio async def test_asgi_lifespan_shutdown_closes_datasette(): ds = Datasette(memory=True) From db16003865dee862d63895dbc156461e8b89372b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 16:39:06 -0700 Subject: [PATCH 020/223] Release 1.0a29 Refs #2695, #2701, #2708, #2709 --- datasette/version.py | 2 +- docs/changelog.rst | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 898d388c..e661e76d 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a28.post1" +__version__ = "1.0a29" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7c1da152..dd9273ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v1_0_a29: + +1.0a29 (2026-05-12) +------------------- + +- New ``TokenRestrictions.abbreviated(datasette)`` :ref:`utility method ` for creating ``"_r"`` dictionaries. (:issue:`2695`) +- Table headers and column options are now visible even if a table contains zero rows. (:issue:`2701`) +- Fixed bug with display of column actions dialog on Mobile Safari. (:issue:`2708`) +- Fixed bug where tests could crash with a segfault due to a race condition between ``Datasette.close()`` and ``Datasette.close()``. (:issue:`2709`) +- .. _v1_0_a28: 1.0a28 (2026-04-16) From 036aa6aa2ef71350d5e99cd12620f9bcd70d7a19 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 16:39:46 -0700 Subject: [PATCH 021/223] Removed a rogue hyphen --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index dd9273ca..4f26066c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,7 @@ Changelog - Table headers and column options are now visible even if a table contains zero rows. (:issue:`2701`) - Fixed bug with display of column actions dialog on Mobile Safari. (:issue:`2708`) - Fixed bug where tests could crash with a segfault due to a race condition between ``Datasette.close()`` and ``Datasette.close()``. (:issue:`2709`) -- + .. _v1_0_a28: 1.0a28 (2026-04-16) From 46d90a0b887bfa23f986a28499eb5f85cd7eed04 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 16:46:56 -0700 Subject: [PATCH 022/223] Bump to actions/checkout@v6 --- .github/workflows/publish.yml | 8 ++++---- .github/workflows/test.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2e8cea9c..87300593 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -35,7 +35,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: @@ -56,7 +56,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: @@ -92,7 +92,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a0f5477b..a1b2e9d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: From 3110faa0bab8cdeb4e4e042e87fefa434f64162f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 May 2026 11:45:43 -0700 Subject: [PATCH 023/223] Replace Janus queue with asyncio.Future Closes #1752 AI generated patch explanation: https://gisthost.github.io/?e2b8d9c7666e988b5c003ff5e5ef3098 --- datasette/database.py | 117 +++++++++++++++++++------------ docs/changelog.rst | 8 +++ pyproject.toml | 1 - tests/test_internals_database.py | 34 +++++++++ tests/test_write_wrapper.py | 27 ++++++- 5 files changed, 140 insertions(+), 47 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 657adfa5..66d50ffa 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -4,7 +4,6 @@ from collections import namedtuple import inspect import os from pathlib import Path -import janus import queue import sqlite_utils import sys @@ -330,13 +329,16 @@ class Database: else: # For non-blocking writes, spawn a background task to # dispatch events after the write thread completes - task_id, reply_queue = result + task_id, reply_future = result async def _dispatch_events_after_write(): - write_result = await reply_queue.async_q.get() - if not isinstance(write_result, Exception): - for event in pending_events: - await self.ds.track_event(event) + try: + await reply_future + except Exception: + # if the write failed, don't emit success events + return + for event in pending_events: + await self.ds.track_event(event) asyncio.ensure_future(_dispatch_events_after_write()) result = task_id @@ -390,18 +392,15 @@ class Database: ) self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") - reply_queue = janus.Queue() + loop = asyncio.get_running_loop() + reply_future = loop.create_future() self._write_queue.put( - WriteTask(fn, task_id, reply_queue, isolated_connection, transaction) + WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) ) if block: - result = await reply_queue.async_q.get() - if isinstance(result, Exception): - raise result - else: - return result + return await reply_future else: - return task_id, reply_queue + return task_id, reply_future def _execute_writes(self): # Infinite looping thread that protects the single write connection @@ -422,36 +421,37 @@ class Database: except Exception: pass return + exception = None + result = None if conn_exception is not None: - result = conn_exception + exception = conn_exception + elif task.isolated_connection: + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + exception = e + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass else: - if task.isolated_connection: - isolated_connection = self.connect(write=True) - try: - result = task.fn(isolated_connection) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - result = e - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass - else: - try: - if task.transaction: - with conn: - result = task.fn(conn) - else: + try: + if task.transaction: + with conn: result = task.fn(conn) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - result = e - task.reply_queue.sync_q.put(result) + else: + result = task.fn(conn) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + exception = e + _deliver_write_result(task, result, exception) async def execute_fn(self, fn): self._check_not_closed() @@ -892,16 +892,45 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: - __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection", "transaction") + __slots__ = ( + "fn", + "task_id", + "loop", + "reply_future", + "isolated_connection", + "transaction", + ) - def __init__(self, fn, task_id, reply_queue, isolated_connection, transaction): + def __init__( + self, fn, task_id, loop, reply_future, isolated_connection, transaction + ): self.fn = fn self.task_id = task_id - self.reply_queue = reply_queue + self.loop = loop + self.reply_future = reply_future self.isolated_connection = isolated_connection self.transaction = transaction +def _deliver_write_result(task, result, exception): + # Called from the write thread. Delivers the result back to the + # awaiting coroutine on its event loop via call_soon_threadsafe. + def _set(): + if task.reply_future.done(): + # Awaiter was cancelled; nothing to do. + return + if exception is not None: + task.reply_future.set_exception(exception) + else: + task.reply_future.set_result(result) + + try: + task.loop.call_soon_threadsafe(_set) + except RuntimeError: + # Event loop has been closed; the awaiter is gone. + pass + + class QueryInterrupted(Exception): def __init__(self, e, sql, params): self.e = e diff --git a/docs/changelog.rst b/docs/changelog.rst index 4f26066c..5b637797 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,14 @@ Changelog ========= +.. _unreleased: + +Unreleased +---------- + +- Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) + + .. _v1_0_a29: 1.0a29 (2026-05-12) diff --git a/pyproject.toml b/pyproject.toml index e6007afd..c50c720a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,6 @@ dependencies = [ "pluggy>=1.0", "uvicorn>=0.11", "aiofiles>=0.4", - "janus>=0.6.2", "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 8ff74a83..75ae8d39 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -2,9 +2,12 @@ Tests for the datasette.database.Database class """ +import asyncio +from types import SimpleNamespace from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues from datasette.database import DatasetteClosedError +from datasette.database import _deliver_write_result from datasette.utils.sqlite import sqlite3, sqlite_version from datasette.utils import Column import pytest @@ -590,6 +593,37 @@ async def test_execute_write_fn_connection_exception(tmpdir, app_client): app_client.ds.remove_database("immutable-db") +@pytest.mark.asyncio +async def test_deliver_write_result_leaves_done_future_alone(): + loop = asyncio.get_running_loop() + reply_future = loop.create_future() + reply_future.set_result("original") + task = SimpleNamespace(loop=loop, reply_future=reply_future) + + # The write thread can finish after the caller has stopped waiting for the + # result. Delivery should notice that the future is already resolved and + # leave the caller's outcome alone instead of raising InvalidStateError. + _deliver_write_result(task, "replacement", None) + await asyncio.sleep(0) + + assert reply_future.result() == "original" + + +@pytest.mark.asyncio +async def test_deliver_write_result_ignores_closed_loop(): + closed_loop = asyncio.new_event_loop() + closed_loop.close() + reply_future = asyncio.get_running_loop().create_future() + task = SimpleNamespace(loop=closed_loop, reply_future=reply_future) + + # If the event loop that submitted the write has gone away, the write + # thread should drop the result rather than crash while reporting back to + # that closed loop. + _deliver_write_result(task, "result", None) + + assert not reply_future.done() + + def table_exists(conn, name): return bool( conn.execute( diff --git a/tests/test_write_wrapper.py b/tests/test_write_wrapper.py index 48c964b4..88ce5520 100644 --- a/tests/test_write_wrapper.py +++ b/tests/test_write_wrapper.py @@ -2,6 +2,7 @@ Tests for the write_wrapper plugin hook. """ +import asyncio from dataclasses import dataclass from datasette.app import Datasette from datasette.events import Event @@ -633,8 +634,6 @@ async def test_track_event_with_block_false(ds_with_event_tracking): assert task_id is not None # Give the background task time to complete - import asyncio - for _ in range(50): if ds._tracked_events: break @@ -644,6 +643,30 @@ async def test_track_event_with_block_false(ds_with_event_tracking): assert ds._tracked_events[0].message == "non-blocking" +@pytest.mark.asyncio +async def test_track_event_with_block_false_discarded_on_exception( + ds_with_event_tracking, +): + """Events queued by a non-blocking write are discarded if the write fails.""" + ds = ds_with_event_tracking + db = ds.get_database("test") + + def my_write(conn, track_event): + track_event(DummyEvent(actor=None, message="should not fire")) + raise ValueError("deliberate error") + + task_id = await db.execute_write_fn(my_write, block=False) + assert task_id is not None + + # A following blocking write proves the failed non-blocking task has + # completed; one more loop turn lets its event-dispatch task observe the + # exception and exit. + await db.execute_write_fn(lambda conn: conn.execute("select 1")) + await asyncio.sleep(0) + + assert ds._tracked_events == [] + + # --- Tests for RenameTableEvent detection --- From 10a1caac53be3d4b8344500f676c49bf82dd5384 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 May 2026 16:38:49 -0700 Subject: [PATCH 024/223] Upgrade a whole lot of GitHum Actions references --- .github/workflows/deploy-branch-preview.yml | 2 +- .github/workflows/deploy-latest.yml | 2 +- .github/workflows/prettier.yml | 4 ++-- .github/workflows/push_docker_tag.yml | 2 +- .github/workflows/spellcheck.yml | 2 +- .github/workflows/stable-docs.yml | 2 +- .github/workflows/test-coverage.yml | 2 +- .github/workflows/test-pyodide.yml | 4 ++-- .github/workflows/test-sqlite-support.yml | 2 +- .github/workflows/tmate-mac.yml | 2 +- .github/workflows/tmate.yml | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml index e56d9c27..4aa676da 100644 --- a/.github/workflows/deploy-branch-preview.yml +++ b/.github/workflows/deploy-branch-preview.yml @@ -12,7 +12,7 @@ jobs: deploy-branch-preview: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Set up Python 3.11 uses: actions/setup-python@v6 with: diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 7349a1ab..18c01fdc 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 77cce7d1..735e14e9 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -10,8 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v4 - - uses: actions/cache@v4 + uses: actions/checkout@v6 + - uses: actions/cache@v5 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index afe8d6b2..e622ef4c 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -13,7 +13,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index d42ae96b..9a808194 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,7 +9,7 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml index 3119d617..59b5fbc0 100644 --- a/.github/workflows/stable-docs.yml +++ b/.github/workflows/stable-docs.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 # We need all commits to find docs/ changes - name: Set up Git user diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 1b3d2f2c..c514048e 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index b490a9bf..5162c47a 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -12,7 +12,7 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python 3.10 uses: actions/setup-python@v6 with: @@ -20,7 +20,7 @@ jobs: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - name: Cache Playwright browsers - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-browsers diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index c81a3c0b..23fce459 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -25,7 +25,7 @@ jobs: #"3.23.1" # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index fcee0f21..a033cd92 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -10,6 +10,6 @@ jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 123f6c71..72af1eec 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -11,7 +11,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 env: From c1b30818633e5cbb43d32bf163f99b0aacc391b7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 May 2026 16:44:28 -0700 Subject: [PATCH 025/223] Removed obsolete workflow --- .github/workflows/deploy-branch-preview.yml | 35 --------------------- 1 file changed, 35 deletions(-) delete mode 100644 .github/workflows/deploy-branch-preview.yml diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml deleted file mode 100644 index 4aa676da..00000000 --- a/.github/workflows/deploy-branch-preview.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Deploy a Datasette branch preview to Vercel - -on: - workflow_dispatch: - inputs: - branch: - description: "Branch to deploy" - required: true - type: string - -jobs: - deploy-branch-preview: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Set up Python 3.11 - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - name: Install dependencies - run: | - pip install datasette-publish-vercel - - name: Deploy the preview - env: - VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} - run: | - export BRANCH="${{ github.event.inputs.branch }}" - wget https://latest.datasette.io/fixtures.db - datasette publish vercel fixtures.db \ - --branch $BRANCH \ - --project "datasette-preview-$BRANCH" \ - --token $VERCEL_TOKEN \ - --scope datasette \ - --about "Preview of $BRANCH" \ - --about_url "https://github.com/simonw/datasette/tree/$BRANCH" From 40e78e09277ae547dc63172f6b2bad50b0f24b64 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 May 2026 16:48:10 -0700 Subject: [PATCH 026/223] Change pull_request_target to pull_request event --- .github/workflows/documentation-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index a54bd83a..b8fb8aaa 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -1,6 +1,6 @@ name: Read the Docs Pull Request Preview on: - pull_request_target: + pull_request: types: - opened From 7a914f8c656de2ffa3f662e49bc95b24dd36b854 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 May 2026 12:14:50 -0700 Subject: [PATCH 027/223] Clear stale tables/other resources when DB removed, closes #2723 --- datasette/app.py | 23 +++++++++++++++---- docs/changelog.rst | 2 +- tests/test_internal_db.py | 48 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 358081ef..218d40c6 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -618,11 +618,24 @@ class Datasette: stale_databases = set(current_schema_versions.keys()) - set( self.databases.keys() ) - for stale_db_name in stale_databases: - await internal_db.execute_write( - "DELETE FROM catalog_databases WHERE database_name = ?", - [stale_db_name], - ) + if stale_databases: + + def delete_stale_database_catalog(conn): + for stale_db_name in stale_databases: + for table in ( + "catalog_columns", + "catalog_foreign_keys", + "catalog_indexes", + "catalog_views", + "catalog_tables", + "catalog_databases", + ): + conn.execute( + "DELETE FROM {} WHERE database_name = ?".format(table), + [stale_db_name], + ) + + await internal_db.execute_write_fn(delete_stale_database_catalog) for database_name, db in self.databases.items(): schema_version = (await db.execute("PRAGMA schema_version")).first()[0] # Compare schema versions to see if we should skip it diff --git a/docs/changelog.rst b/docs/changelog.rst index 5b637797..eb408287 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ Unreleased ---------- - Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) - +- Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) .. _v1_0_a29: diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index 7a0d1630..ec013b43 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -139,3 +139,51 @@ async def test_stale_catalog_entry_database_fix(tmp_path): f"Index page should return 200, not {response.status_code}. " "This fails due to stale catalog entries causing KeyError." ) + + +@pytest.mark.asyncio +async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path): + from datasette.app import Datasette + + import sqlite3 + + internal_db_path = str(tmp_path / "internal.db") + alpha_db_path = str(tmp_path / "alpha.db") + bravo_db_path = str(tmp_path / "bravo.db") + + for db_path, table_name in ( + (alpha_db_path, "alpha_table"), + (bravo_db_path, "bravo_table"), + (bravo_db_path, "bravo_table_2"), + ): + conn = sqlite3.connect(db_path) + conn.execute(f"CREATE TABLE {table_name} (id INTEGER PRIMARY KEY)") + conn.close() + + ds1 = Datasette(files=[alpha_db_path, bravo_db_path], internal=internal_db_path) + await ds1.invoke_startup() + + catalog_tables = await ds1.get_internal_database().execute(""" + SELECT database_name, table_name + FROM catalog_tables + ORDER BY database_name, table_name + """) + assert [tuple(row) for row in catalog_tables.rows] == [ + ("alpha", "alpha_table"), + ("bravo", "bravo_table"), + ("bravo", "bravo_table_2"), + ] + + ds1.close() + + ds2 = Datasette(files=[alpha_db_path], internal=internal_db_path) + await ds2.invoke_startup() + + catalog_tables = await ds2.get_internal_database().execute(""" + SELECT database_name, table_name + FROM catalog_tables + ORDER BY database_name, table_name + """) + assert [tuple(row) for row in catalog_tables.rows] == [("alpha", "alpha_table")] + + ds2.close() From 5d6de0154d18d0ed07e7ac7cf02cd3c37324ddbc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 May 2026 12:18:01 -0700 Subject: [PATCH 028/223] Bump Black to black==26.3.1 Refs https://github.com/advisories/GHSA-3936-cmfr-pm3m --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c50c720a..38085476 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ dev = [ "pytest-xdist>=2.2.1", "pytest-asyncio>=1.2.0", "beautifulsoup4>=4.8.1", - "black==26.1.0", + "black==26.3.1", "blacken-docs==1.20.0", "pytest-timeout>=1.4.2", "trustme>=0.7", From bbbc1cd59620c7c53a2eff6138ef338c8901ba6e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 May 2026 12:33:33 -0700 Subject: [PATCH 029/223] Remove height: 100% to fix Safari bug, closes #2724 --- datasette/static/navigation-search.js | 1 - docs/changelog.rst | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index 95e7dfc5..d2c300e2 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -54,7 +54,6 @@ class NavigationSearch extends HTMLElement { .search-container { display: flex; flex-direction: column; - height: 100%; } .search-input-wrapper { diff --git a/docs/changelog.rst b/docs/changelog.rst index eb408287..56c49ea3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased - Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) - Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) +- Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`) .. _v1_0_a29: From 54b272baf61cb014e6d262fea1b01dc84981c1d0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 May 2026 12:39:54 -0700 Subject: [PATCH 030/223] Remove existing stale catalog_ tables, refs #2723 Now if there are any existing stale records in internal.db those will be removed as well. --- datasette/app.py | 30 ++++++++++++++++---------- tests/test_internal_db.py | 45 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 218d40c6..b1f9b2f7 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -614,22 +614,30 @@ class Datasette: "select database_name, schema_version from catalog_databases" ) } - # Delete stale entries for databases that are no longer attached - stale_databases = set(current_schema_versions.keys()) - set( - self.databases.keys() + catalog_table_names = ( + "catalog_columns", + "catalog_foreign_keys", + "catalog_indexes", + "catalog_views", + "catalog_tables", + "catalog_databases", ) + # Delete stale entries for databases that are no longer attached + catalog_database_names = set(current_schema_versions.keys()) + for table in catalog_table_names[:-1]: + catalog_database_names.update( + row["database_name"] + for row in await internal_db.execute( + "select distinct database_name from {}".format(table) + ) + if row["database_name"] is not None + ) + stale_databases = catalog_database_names - set(self.databases.keys()) if stale_databases: def delete_stale_database_catalog(conn): for stale_db_name in stale_databases: - for table in ( - "catalog_columns", - "catalog_foreign_keys", - "catalog_indexes", - "catalog_views", - "catalog_tables", - "catalog_databases", - ): + for table in catalog_table_names: conn.execute( "DELETE FROM {} WHERE database_name = ?".format(table), [stale_db_name], diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index ec013b43..dcf14126 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -187,3 +187,48 @@ async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path assert [tuple(row) for row in catalog_tables.rows] == [("alpha", "alpha_table")] ds2.close() + + +@pytest.mark.asyncio +async def test_orphan_stale_catalog_child_entries_removed(tmp_path): + from datasette.app import Datasette + + import sqlite3 + + internal_db_path = str(tmp_path / "internal.db") + alpha_db_path = str(tmp_path / "alpha.db") + + conn = sqlite3.connect(alpha_db_path) + conn.execute("CREATE TABLE alpha_table (id INTEGER PRIMARY KEY)") + conn.close() + + ds1 = Datasette(files=[alpha_db_path], internal=internal_db_path) + await ds1.invoke_startup() + ds1.close() + + # Simulate the state left behind by old cleanup code: the parent database + # row was deleted, but child catalog rows survived because foreign key + # enforcement is not enabled for these internal catalog writes. + conn = sqlite3.connect(internal_db_path) + conn.execute("DELETE FROM catalog_databases WHERE database_name = 'fixtures'") + conn.execute(""" + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) + VALUES ('fixtures', 'stale_table', 1, 'CREATE TABLE stale_table (id INTEGER)') + """) + conn.commit() + conn.close() + + ds2 = Datasette(files=[alpha_db_path], internal=internal_db_path) + await ds2.invoke_startup() + + catalog_tables = await ds2.get_internal_database().execute(""" + SELECT database_name, table_name + FROM catalog_tables + ORDER BY database_name, table_name + """) + assert [tuple(row) for row in catalog_tables.rows] == [("alpha", "alpha_table")] + + response = await ds2.client.get("/-/tables.json") + assert response.status_code == 200 + + ds2.close() From d3330695fa42ad1cdb2f2b1b80470e95bea8ed12 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 May 2026 13:23:05 -0700 Subject: [PATCH 031/223] Always show 'Jump to...' menu item, closes #2725 --- datasette/static/app.css | 26 ++++++++++++++++++++++++++ datasette/static/navigation-search.js | 13 ++++++++++++- datasette/templates/base.html | 7 +++---- docs/changelog.rst | 1 + tests/test_html.py | 18 +++++++++++++++--- 5 files changed, 57 insertions(+), 8 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 1ce84bc8..c21d0dc4 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -362,6 +362,32 @@ form.nav-menu-logout { .nav-menu-inner a { display: block; } +.nav-menu-inner button.button-as-link { + display: block; + width: 100%; + text-align: left; + font: inherit; +} +.nav-menu-inner .keyboard-shortcut { + float: right; + box-sizing: border-box; + min-width: 1.4em; + margin-left: 0.75rem; + padding: 0 0.35em; + border: 1px solid rgba(255,255,244,0.6); + border-radius: 3px; + background: rgba(255,255,244,0.12); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.85em; + line-height: 1.35; + text-align: center; + text-decoration: none; +} +@media (max-width: 640px) { + .nav-menu-inner .keyboard-shortcut { + display: none; + } +} /* Table/database actions menu */ .page-action-menu { diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index d2c300e2..09d58898 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -199,6 +199,15 @@ class NavigationSearch extends HTMLElement { } }); + document.addEventListener("click", (e) => { + const trigger = e.target.closest("[data-navigation-search-open]"); + if (trigger) { + e.preventDefault(); + trigger.closest("details")?.removeAttribute("open"); + this.openMenu(); + } + }); + // Input event input.addEventListener("input", (e) => { this.handleSearch(e.target.value); @@ -390,7 +399,9 @@ class NavigationSearch extends HTMLElement { const dialog = this.shadowRoot.querySelector("dialog"); const input = this.shadowRoot.querySelector(".search-input"); - dialog.showModal(); + if (!dialog.open) { + dialog.showModal(); + } input.value = ""; input.focus(); diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 21f8c693..b4fecf70 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -20,7 +20,7 @@