From 0df6555ef3f9395863264a335ba47530f670ef64 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:59:21 -0700 Subject: [PATCH 001/113] Release 1.0a28 Refs #2691, #2692, #2693 --- datasette/version.py | 2 +- docs/changelog.rst | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index e2c80e50..cf908bb2 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a27" +__version__ = "1.0a28" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9cd7a7d6..7c1da152 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v1_0_a28: + +1.0a28 (2026-04-16) +------------------- + +- Fixed a compatibility bug introduced in 1.0a27 where ``execute_write_fn()`` callbacks with a parameter name other than ``conn`` were seeing errors. (:issue:`2691`) +- The :ref:`database.close() ` method now also shuts down the write connection for that database. +- New :ref:`datasette.close() ` method for closing down all databases and resources associated with a Datasette instance. This is called automatically when the server shuts down. (:pr:`2693`) +- Datasette now includes a pytest plugin which automatically calls ``datasette.close()`` on temporary instances created in function-scoped fixtures and during tests. See :ref:`testing_plugins_autoclose` for details. This helps avoid running out of file descriptors in plugin test suites that were written before the ``Database(is_temp_disk=True)`` feature introduced in Datasette 1.0a27. (:issue:`2692`) + .. _v1_0_a27: 1.0a27 (2026-04-15) From a6031c98476487ec2aa5830bea75df9e6615262d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 16 Apr 2026 20:59:21 -0700 Subject: [PATCH 002/113] Release 1.0a28 Refs #2691, #2692, #2693 --- datasette/version.py | 2 +- docs/changelog.rst | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index e2c80e50..cf908bb2 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a27" +__version__ = "1.0a28" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9cd7a7d6..7c1da152 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v1_0_a28: + +1.0a28 (2026-04-16) +------------------- + +- Fixed a compatibility bug introduced in 1.0a27 where ``execute_write_fn()`` callbacks with a parameter name other than ``conn`` were seeing errors. (:issue:`2691`) +- The :ref:`database.close() ` method now also shuts down the write connection for that database. +- New :ref:`datasette.close() ` method for closing down all databases and resources associated with a Datasette instance. This is called automatically when the server shuts down. (:pr:`2693`) +- Datasette now includes a pytest plugin which automatically calls ``datasette.close()`` on temporary instances created in function-scoped fixtures and during tests. See :ref:`testing_plugins_autoclose` for details. This helps avoid running out of file descriptors in plugin test suites that were written before the ``Database(is_temp_disk=True)`` feature introduced in Datasette 1.0a27. (:issue:`2692`) + .. _v1_0_a27: 1.0a27 (2026-04-15) From b15ce18ddc463a537a52879381ad929d1867143d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 17 Apr 2026 08:44:43 -0700 Subject: [PATCH 003/113] TokenRestrictions.abbreviated(datasette) utility method for creating _r dicts (#2696) Closes #2695 Refs https://github.com/simonw/datasette-auth-tokens/pull/42 --- datasette/tokens.py | 59 ++++++++++++++++++++++--------------- docs/internals.rst | 24 +++++++++++++++ tests/test_token_handler.py | 37 +++++++++++++++++++++++ 3 files changed, 97 insertions(+), 23 deletions(-) diff --git a/datasette/tokens.py b/datasette/tokens.py index 5a12d8e0..38a55529 100644 --- a/datasette/tokens.py +++ b/datasette/tokens.py @@ -52,6 +52,38 @@ class TokenRestrictions: self.resource.setdefault(database, {}).setdefault(resource, []).append(action) return self + def abbreviated(self, datasette: "Datasette") -> Optional[dict]: + """ + Return the abbreviated ``_r`` dictionary shape for this set of + restrictions, using action abbreviations registered with ``datasette``. + Returns ``None`` if no restrictions are set. + """ + if not (self.all or self.database or self.resource): + return None + + def abbreviate_action(action): + action_obj = datasette.actions.get(action) + if not action_obj: + return action + return action_obj.abbr or action + + result: dict = {} + if self.all: + result["a"] = [abbreviate_action(a) for a in self.all] + if self.database: + result["d"] = { + database: [abbreviate_action(a) for a in actions] + for database, actions in self.database.items() + } + if self.resource: + result["r"] = {} + for database, resources in self.resource.items(): + for resource, actions in resources.items(): + result["r"].setdefault(database, {})[resource] = [ + abbreviate_action(a) for a in actions + ] + return result + class TokenHandler: """ @@ -104,31 +136,12 @@ class SignedTokenHandler(TokenHandler): token = {"a": actor_id, "t": int(time.time())} - def abbreviate_action(action): - action_obj = datasette.actions.get(action) - if not action_obj: - return action - return action_obj.abbr or action - if expires_after: token["d"] = expires_after - if restrictions and ( - restrictions.all or restrictions.database or restrictions.resource - ): - token["_r"] = {} - if restrictions.all: - token["_r"]["a"] = [abbreviate_action(a) for a in restrictions.all] - if restrictions.database: - token["_r"]["d"] = {} - for database, actions in restrictions.database.items(): - token["_r"]["d"][database] = [abbreviate_action(a) for a in actions] - if restrictions.resource: - token["_r"]["r"] = {} - for database, resources in restrictions.resource.items(): - for resource, actions in resources.items(): - token["_r"]["r"].setdefault(database, {})[resource] = [ - abbreviate_action(a) for a in actions - ] + if restrictions is not None: + abbreviated = restrictions.abbreviated(datasette) + if abbreviated is not None: + token["_r"] = abbreviated return "dstok_{}".format(datasette.sign(token, namespace="token")) async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]: diff --git a/docs/internals.rst b/docs/internals.rst index 2710345b..e0123a7b 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -729,6 +729,30 @@ The builder methods are: Each method returns the ``TokenRestrictions`` instance so calls can be chained. +``TokenRestrictions`` also provides an ``abbreviated(datasette)`` method which returns the restrictions as a dictionary using the compact format described in :ref:`authentication_cli_create_token_restrict`, with action names replaced by their registered abbreviations. It returns the inner dictionary only - the ``"_r"`` wrapping key shown in that section is not included. Returns ``None`` if no restrictions are set. This is useful when writing a custom :ref:`plugin_hook_register_token_handler` that needs to embed restrictions in a token payload. + +For example, the following restrictions: + +.. code-block:: python + + restrictions = ( + TokenRestrictions() + .allow_all("view-instance") + .allow_database("docs", "view-query") + .allow_resource("docs", "attachments", "insert-row") + ) + restrictions.abbreviated(datasette) + +Returns this dictionary, using the abbreviations registered for each action: + +.. code-block:: python + + { + "a": ["vi"], + "d": {"docs": ["vq"]}, + "r": {"docs": {"attachments": ["ir"]}}, + } + The following example creates a token that can access ``view-instance`` and ``view-table`` across everything, can additionally use ``view-query`` for anything in the ``docs`` database and is allowed to execute ``insert-row`` and ``update-row`` in the ``attachments`` table in that database: .. code-block:: python diff --git a/tests/test_token_handler.py b/tests/test_token_handler.py index 83f09046..5c87f577 100644 --- a/tests/test_token_handler.py +++ b/tests/test_token_handler.py @@ -291,6 +291,43 @@ async def test_expires_after_round_trip(datasette): assert "token_expires" in actor +@pytest.mark.asyncio +@pytest.mark.parametrize( + "build_restrictions,expected", + [ + (lambda r: r, None), + (lambda r: r.allow_all("view-instance"), {"a": ["vi"]}), + ( + lambda r: r.allow_database("docs", "view-query"), + {"d": {"docs": ["vq"]}}, + ), + ( + lambda r: r.allow_resource("docs", "attachments", "insert-row"), + {"r": {"docs": {"attachments": ["ir"]}}}, + ), + ( + lambda r: r.allow_all("view-instance") + .allow_database("docs", "view-query") + .allow_resource("docs", "attachments", "insert-row"), + { + "a": ["vi"], + "d": {"docs": ["vq"]}, + "r": {"docs": {"attachments": ["ir"]}}, + }, + ), + ( + lambda r: r.allow_all("not-a-real-action"), + {"a": ["not-a-real-action"]}, + ), + ], + ids=["empty", "all", "database", "resource", "combined", "unknown_action"], +) +async def test_token_restrictions_abbreviated(datasette, build_restrictions, expected): + await datasette.invoke_startup() + restrictions = build_restrictions(TokenRestrictions()) + assert restrictions.abbreviated(datasette) == expected + + @pytest.mark.asyncio async def test_signed_tokens_disabled(): """create_token and verify_token should fail/skip when signed tokens are disabled.""" From 0dc7bb19d9a95df9d9c6bd00e943d407fc11f49e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 22 Apr 2026 22:22:47 -0700 Subject: [PATCH 004/113] Table headers and column options visible for 0 rows Closes #2701 --- datasette/templates/_table.html | 5 +++-- datasette/templates/table.html | 2 -- tests/test_html.py | 2 +- tests/test_table_html.py | 26 ++++++++++++++++++++++++-- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/datasette/templates/_table.html b/datasette/templates/_table.html index ba34b60f..f47a325f 100644 --- a/datasette/templates/_table.html +++ b/datasette/templates/_table.html @@ -1,6 +1,6 @@
-{% if display_rows %} +{% if display_columns %}
@@ -31,6 +31,7 @@
-{% else %} +{% endif %} +{% if not display_rows %}

0 records

{% endif %} diff --git a/datasette/templates/table.html b/datasette/templates/table.html index 2919d306..c841e1be 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -141,7 +141,6 @@ {% if all_columns %} -{% if display_rows %} -{% endif %} diff --git a/tests/test_html.py b/tests/test_html.py index e38898da..7425692d 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -805,7 +805,7 @@ async def test_blob_download_invalid_messages(ds_client, path, expected_message) async def test_zero_results(ds_client, path): response = await ds_client.get(path) soup = Soup(response.text, "html.parser") - assert 0 == len(soup.select("table")) + assert 0 == len(soup.select("table tbody tr")) assert 1 == len(soup.select("p.zero-results")) diff --git a/tests/test_table_html.py b/tests/test_table_html.py index d8dde593..86b9a4eb 100644 --- a/tests/test_table_html.py +++ b/tests/test_table_html.py @@ -752,8 +752,11 @@ async def test_column_chooser_present(ds_client): @pytest.mark.asyncio -async def test_mobile_column_actions_present(ds_client): - response = await ds_client.get("/fixtures/facetable") +@pytest.mark.parametrize( + "path", ["/fixtures/facetable", "/fixtures/123_starts_with_digits"] +) +async def test_mobile_column_actions_present(ds_client, path): + response = await ds_client.get(path) assert response.status_code == 200 soup = Soup(response.text, "html.parser") button = soup.select_one("button.column-actions-mobile.small-screen-only") @@ -764,6 +767,25 @@ async def test_mobile_column_actions_present(ds_client): "mobile-column-actions.js" in (script.get("src") or "") for script in soup.find_all("script") ) + # mobile-column-actions.js builds its dialog from elements, + # so the thead must render even when the table has no rows. + ths = soup.select("table.rows-and-columns thead th[data-column]") + assert len(ths) >= 1 + + +@pytest.mark.asyncio +async def test_zero_row_table_renders_thead(ds_client): + response = await ds_client.get("/fixtures/123_starts_with_digits") + assert response.status_code == 200 + soup = Soup(response.text, "html.parser") + table = soup.select_one("table.rows-and-columns") + assert table is not None + column_names = [ + th.get("data-column") for th in table.select("thead th[data-column]") + ] + assert "content" in column_names + assert table.select_one("tbody tr") is None + assert soup.select_one("p.zero-results") is not None @pytest.mark.asyncio From aa84fe008d7c6263bd8712adcfe9be53a9f207ea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 5 May 2026 16:05:12 -0700 Subject: [PATCH 005/113] Fix for column actions on Mobile Safari, closes #2708 --- datasette/static/app.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 26717c43..1ce84bc8 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -818,7 +818,8 @@ dialog.mobile-column-actions-dialog::backdrop { } .mobile-column-actions-dialog .list-wrap { - flex: 1; + flex: 1 1 auto; + min-height: 0; overflow-y: auto; overflow-x: hidden; position: relative; From 345f910043bebd4fb829c1f8c248a17e38856191 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 16:31:36 -0700 Subject: [PATCH 006/113] Fix for Database.close()/Datasette.close() order (#2710) Closes: - #2709 The key behavior change: after close() starts, no new execute work can be submitted, but already-running execute work is allowed to finish before SQLite connections are closed. --- datasette/database.py | 26 +++++++++++++--- datasette/version.py | 2 +- tests/test_internals_datasette.py | 49 +++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index e3c4bfec..657adfa5 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -84,6 +84,8 @@ class Database: self._write_thread = None self._write_queue = None self._closed = False + self._pending_execute_futures = set() + self._pending_execute_futures_lock = threading.Lock() # These are used when in non-threaded mode: self._read_connection = None self._write_connection = None @@ -98,6 +100,10 @@ class Database: "Database {!r} has been closed".format(self.name) ) + def _remove_pending_execute_future(self, future): + with self._pending_execute_futures_lock: + self._pending_execute_futures.discard(future) + @property def cached_table_counts(self): if self._cached_table_counts is not None: @@ -170,7 +176,11 @@ class Database: """ if self._closed: return - self._closed = True + with self._pending_execute_futures_lock: + if self._closed: + return + self._closed = True + pending_execute_futures = tuple(self._pending_execute_futures) # Shut down the write thread, if any, via a sentinel. The thread # drains any writes already queued before the sentinel and then # closes its own write connection and returns. @@ -185,6 +195,11 @@ class Database: ) ) sys.stderr.flush() + for future in pending_execute_futures: + try: + future.result() + except Exception: + pass # Close anything still tracked in _all_file_connections for connection in self._all_file_connections: try: @@ -456,9 +471,12 @@ class Database: setattr(connections, self._thread_local_id, conn) return fn(conn) - return await asyncio.get_event_loop().run_in_executor( - self.ds.executor, in_thread - ) + with self._pending_execute_futures_lock: + self._check_not_closed() + future = self.ds.executor.submit(in_thread) + self._pending_execute_futures.add(future) + future.add_done_callback(self._remove_pending_execute_future) + return await asyncio.wrap_future(future) async def execute( self, diff --git a/datasette/version.py b/datasette/version.py index cf908bb2..898d388c 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a28" +__version__ = "1.0a28.post1" __version_info__ = tuple(__version__.split(".")) diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index d58c9a29..3f867eb0 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -2,8 +2,11 @@ Tests for the datasette.app.Datasette class """ +import asyncio import dataclasses import os +import sqlite3 +import time from datasette import Context from datasette.app import Datasette, Database, ResourcesSQL from datasette.database import DatasetteClosedError @@ -256,6 +259,52 @@ async def test_datasette_close_raises_on_use(): await ds.get_internal_database().execute("select 1") +async def _datasette_with_sleeping_execute(tmp_path, sleep_ms=200): + db_path = tmp_path / "data.db" + internal_path = tmp_path / "internal.db" + sqlite3.connect(db_path).close() + ds = Datasette([str(db_path)], internal=str(internal_path)) + loop = asyncio.get_running_loop() + sql_started = asyncio.Event() + original_prepare_connection = ds._prepare_connection + + def prepare_connection(conn, name): + original_prepare_connection(conn, name) + + def sleep_ms(ms): + loop.call_soon_threadsafe(sql_started.set) + time.sleep(ms / 1000) + return ms + + conn.create_function("sleep_ms", 1, sleep_ms) + + ds._prepare_connection = prepare_connection + task = asyncio.create_task( + ds.get_database().execute( + f"select sleep_ms({sleep_ms})", custom_time_limit=1000 + ) + ) + await asyncio.wait_for(sql_started.wait(), timeout=5) + return ds, task + + +@pytest.mark.asyncio +async def test_datasette_close_waits_for_in_flight_execute(tmp_path): + ds, task = await _datasette_with_sleeping_execute(tmp_path) + ds.close() + results = await task + assert [tuple(row) for row in results.rows] == [(200,)] + + +@pytest.mark.asyncio +async def test_datasette_close_waits_for_cancelled_in_flight_execute(tmp_path): + ds, task = await _datasette_with_sleeping_execute(tmp_path) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + ds.close() + + @pytest.mark.asyncio async def test_asgi_lifespan_shutdown_closes_datasette(): ds = Datasette(memory=True) From db16003865dee862d63895dbc156461e8b89372b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 16:39:06 -0700 Subject: [PATCH 007/113] Release 1.0a29 Refs #2695, #2701, #2708, #2709 --- datasette/version.py | 2 +- docs/changelog.rst | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datasette/version.py b/datasette/version.py index 898d388c..e661e76d 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a28.post1" +__version__ = "1.0a29" __version_info__ = tuple(__version__.split(".")) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7c1da152..dd9273ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v1_0_a29: + +1.0a29 (2026-05-12) +------------------- + +- New ``TokenRestrictions.abbreviated(datasette)`` :ref:`utility method ` for creating ``"_r"`` dictionaries. (:issue:`2695`) +- Table headers and column options are now visible even if a table contains zero rows. (:issue:`2701`) +- Fixed bug with display of column actions dialog on Mobile Safari. (:issue:`2708`) +- Fixed bug where tests could crash with a segfault due to a race condition between ``Datasette.close()`` and ``Datasette.close()``. (:issue:`2709`) +- .. _v1_0_a28: 1.0a28 (2026-04-16) From 036aa6aa2ef71350d5e99cd12620f9bcd70d7a19 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 16:39:46 -0700 Subject: [PATCH 008/113] Removed a rogue hyphen --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index dd9273ca..4f26066c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,7 @@ Changelog - Table headers and column options are now visible even if a table contains zero rows. (:issue:`2701`) - Fixed bug with display of column actions dialog on Mobile Safari. (:issue:`2708`) - Fixed bug where tests could crash with a segfault due to a race condition between ``Datasette.close()`` and ``Datasette.close()``. (:issue:`2709`) -- + .. _v1_0_a28: 1.0a28 (2026-04-16) From 46d90a0b887bfa23f986a28499eb5f85cd7eed04 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 12 May 2026 16:46:56 -0700 Subject: [PATCH 009/113] Bump to actions/checkout@v6 --- .github/workflows/publish.yml | 8 ++++---- .github/workflows/test.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2e8cea9c..87300593 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -35,7 +35,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: @@ -56,7 +56,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: @@ -92,7 +92,7 @@ jobs: needs: [deploy] if: "!github.event.release.prerelease" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a0f5477b..a1b2e9d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: From 3110faa0bab8cdeb4e4e042e87fefa434f64162f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 May 2026 11:45:43 -0700 Subject: [PATCH 010/113] Replace Janus queue with asyncio.Future Closes #1752 AI generated patch explanation: https://gisthost.github.io/?e2b8d9c7666e988b5c003ff5e5ef3098 --- datasette/database.py | 117 +++++++++++++++++++------------ docs/changelog.rst | 8 +++ pyproject.toml | 1 - tests/test_internals_database.py | 34 +++++++++ tests/test_write_wrapper.py | 27 ++++++- 5 files changed, 140 insertions(+), 47 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 657adfa5..66d50ffa 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -4,7 +4,6 @@ from collections import namedtuple import inspect import os from pathlib import Path -import janus import queue import sqlite_utils import sys @@ -330,13 +329,16 @@ class Database: else: # For non-blocking writes, spawn a background task to # dispatch events after the write thread completes - task_id, reply_queue = result + task_id, reply_future = result async def _dispatch_events_after_write(): - write_result = await reply_queue.async_q.get() - if not isinstance(write_result, Exception): - for event in pending_events: - await self.ds.track_event(event) + try: + await reply_future + except Exception: + # if the write failed, don't emit success events + return + for event in pending_events: + await self.ds.track_event(event) asyncio.ensure_future(_dispatch_events_after_write()) result = task_id @@ -390,18 +392,15 @@ class Database: ) self._write_thread.start() task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") - reply_queue = janus.Queue() + loop = asyncio.get_running_loop() + reply_future = loop.create_future() self._write_queue.put( - WriteTask(fn, task_id, reply_queue, isolated_connection, transaction) + WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) ) if block: - result = await reply_queue.async_q.get() - if isinstance(result, Exception): - raise result - else: - return result + return await reply_future else: - return task_id, reply_queue + return task_id, reply_future def _execute_writes(self): # Infinite looping thread that protects the single write connection @@ -422,36 +421,37 @@ class Database: except Exception: pass return + exception = None + result = None if conn_exception is not None: - result = conn_exception + exception = conn_exception + elif task.isolated_connection: + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + exception = e + finally: + isolated_connection.close() + try: + self._all_file_connections.remove(isolated_connection) + except ValueError: + # Was probably a memory connection + pass else: - if task.isolated_connection: - isolated_connection = self.connect(write=True) - try: - result = task.fn(isolated_connection) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - result = e - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass - else: - try: - if task.transaction: - with conn: - result = task.fn(conn) - else: + try: + if task.transaction: + with conn: result = task.fn(conn) - except Exception as e: - sys.stderr.write("{}\n".format(e)) - sys.stderr.flush() - result = e - task.reply_queue.sync_q.put(result) + else: + result = task.fn(conn) + except Exception as e: + sys.stderr.write("{}\n".format(e)) + sys.stderr.flush() + exception = e + _deliver_write_result(task, result, exception) async def execute_fn(self, fn): self._check_not_closed() @@ -892,16 +892,45 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: - __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection", "transaction") + __slots__ = ( + "fn", + "task_id", + "loop", + "reply_future", + "isolated_connection", + "transaction", + ) - def __init__(self, fn, task_id, reply_queue, isolated_connection, transaction): + def __init__( + self, fn, task_id, loop, reply_future, isolated_connection, transaction + ): self.fn = fn self.task_id = task_id - self.reply_queue = reply_queue + self.loop = loop + self.reply_future = reply_future self.isolated_connection = isolated_connection self.transaction = transaction +def _deliver_write_result(task, result, exception): + # Called from the write thread. Delivers the result back to the + # awaiting coroutine on its event loop via call_soon_threadsafe. + def _set(): + if task.reply_future.done(): + # Awaiter was cancelled; nothing to do. + return + if exception is not None: + task.reply_future.set_exception(exception) + else: + task.reply_future.set_result(result) + + try: + task.loop.call_soon_threadsafe(_set) + except RuntimeError: + # Event loop has been closed; the awaiter is gone. + pass + + class QueryInterrupted(Exception): def __init__(self, e, sql, params): self.e = e diff --git a/docs/changelog.rst b/docs/changelog.rst index 4f26066c..5b637797 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,14 @@ Changelog ========= +.. _unreleased: + +Unreleased +---------- + +- Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) + + .. _v1_0_a29: 1.0a29 (2026-05-12) diff --git a/pyproject.toml b/pyproject.toml index e6007afd..c50c720a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,6 @@ dependencies = [ "pluggy>=1.0", "uvicorn>=0.11", "aiofiles>=0.4", - "janus>=0.6.2", "PyYAML>=5.3", "mergedeep>=1.1.1", "itsdangerous>=1.1", diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 8ff74a83..75ae8d39 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -2,9 +2,12 @@ Tests for the datasette.database.Database class """ +import asyncio +from types import SimpleNamespace from datasette.app import Datasette from datasette.database import Database, Results, MultipleValues from datasette.database import DatasetteClosedError +from datasette.database import _deliver_write_result from datasette.utils.sqlite import sqlite3, sqlite_version from datasette.utils import Column import pytest @@ -590,6 +593,37 @@ async def test_execute_write_fn_connection_exception(tmpdir, app_client): app_client.ds.remove_database("immutable-db") +@pytest.mark.asyncio +async def test_deliver_write_result_leaves_done_future_alone(): + loop = asyncio.get_running_loop() + reply_future = loop.create_future() + reply_future.set_result("original") + task = SimpleNamespace(loop=loop, reply_future=reply_future) + + # The write thread can finish after the caller has stopped waiting for the + # result. Delivery should notice that the future is already resolved and + # leave the caller's outcome alone instead of raising InvalidStateError. + _deliver_write_result(task, "replacement", None) + await asyncio.sleep(0) + + assert reply_future.result() == "original" + + +@pytest.mark.asyncio +async def test_deliver_write_result_ignores_closed_loop(): + closed_loop = asyncio.new_event_loop() + closed_loop.close() + reply_future = asyncio.get_running_loop().create_future() + task = SimpleNamespace(loop=closed_loop, reply_future=reply_future) + + # If the event loop that submitted the write has gone away, the write + # thread should drop the result rather than crash while reporting back to + # that closed loop. + _deliver_write_result(task, "result", None) + + assert not reply_future.done() + + def table_exists(conn, name): return bool( conn.execute( diff --git a/tests/test_write_wrapper.py b/tests/test_write_wrapper.py index 48c964b4..88ce5520 100644 --- a/tests/test_write_wrapper.py +++ b/tests/test_write_wrapper.py @@ -2,6 +2,7 @@ Tests for the write_wrapper plugin hook. """ +import asyncio from dataclasses import dataclass from datasette.app import Datasette from datasette.events import Event @@ -633,8 +634,6 @@ async def test_track_event_with_block_false(ds_with_event_tracking): assert task_id is not None # Give the background task time to complete - import asyncio - for _ in range(50): if ds._tracked_events: break @@ -644,6 +643,30 @@ async def test_track_event_with_block_false(ds_with_event_tracking): assert ds._tracked_events[0].message == "non-blocking" +@pytest.mark.asyncio +async def test_track_event_with_block_false_discarded_on_exception( + ds_with_event_tracking, +): + """Events queued by a non-blocking write are discarded if the write fails.""" + ds = ds_with_event_tracking + db = ds.get_database("test") + + def my_write(conn, track_event): + track_event(DummyEvent(actor=None, message="should not fire")) + raise ValueError("deliberate error") + + task_id = await db.execute_write_fn(my_write, block=False) + assert task_id is not None + + # A following blocking write proves the failed non-blocking task has + # completed; one more loop turn lets its event-dispatch task observe the + # exception and exit. + await db.execute_write_fn(lambda conn: conn.execute("select 1")) + await asyncio.sleep(0) + + assert ds._tracked_events == [] + + # --- Tests for RenameTableEvent detection --- From 10a1caac53be3d4b8344500f676c49bf82dd5384 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 May 2026 16:38:49 -0700 Subject: [PATCH 011/113] Upgrade a whole lot of GitHum Actions references --- .github/workflows/deploy-branch-preview.yml | 2 +- .github/workflows/deploy-latest.yml | 2 +- .github/workflows/prettier.yml | 4 ++-- .github/workflows/push_docker_tag.yml | 2 +- .github/workflows/spellcheck.yml | 2 +- .github/workflows/stable-docs.yml | 2 +- .github/workflows/test-coverage.yml | 2 +- .github/workflows/test-pyodide.yml | 4 ++-- .github/workflows/test-sqlite-support.yml | 2 +- .github/workflows/tmate-mac.yml | 2 +- .github/workflows/tmate.yml | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml index e56d9c27..4aa676da 100644 --- a/.github/workflows/deploy-branch-preview.yml +++ b/.github/workflows/deploy-branch-preview.yml @@ -12,7 +12,7 @@ jobs: deploy-branch-preview: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Set up Python 3.11 uses: actions/setup-python@v6 with: diff --git a/.github/workflows/deploy-latest.yml b/.github/workflows/deploy-latest.yml index 7349a1ab..18c01fdc 100644 --- a/.github/workflows/deploy-latest.yml +++ b/.github/workflows/deploy-latest.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 77cce7d1..735e14e9 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -10,8 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v4 - - uses: actions/cache@v4 + uses: actions/checkout@v6 + - uses: actions/cache@v5 name: Configure npm caching with: path: ~/.npm diff --git a/.github/workflows/push_docker_tag.yml b/.github/workflows/push_docker_tag.yml index afe8d6b2..e622ef4c 100644 --- a/.github/workflows/push_docker_tag.yml +++ b/.github/workflows/push_docker_tag.yml @@ -13,7 +13,7 @@ jobs: deploy_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Build and push to Docker Hub env: DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index d42ae96b..9a808194 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -9,7 +9,7 @@ jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/stable-docs.yml b/.github/workflows/stable-docs.yml index 3119d617..59b5fbc0 100644 --- a/.github/workflows/stable-docs.yml +++ b/.github/workflows/stable-docs.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 # We need all commits to find docs/ changes - name: Set up Git user diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 1b3d2f2c..c514048e 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out datasette - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: diff --git a/.github/workflows/test-pyodide.yml b/.github/workflows/test-pyodide.yml index b490a9bf..5162c47a 100644 --- a/.github/workflows/test-pyodide.yml +++ b/.github/workflows/test-pyodide.yml @@ -12,7 +12,7 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python 3.10 uses: actions/setup-python@v6 with: @@ -20,7 +20,7 @@ jobs: cache: 'pip' cache-dependency-path: '**/pyproject.toml' - name: Cache Playwright browsers - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/ms-playwright/ key: ${{ runner.os }}-browsers diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index c81a3c0b..23fce459 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -25,7 +25,7 @@ jobs: #"3.23.1" # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: diff --git a/.github/workflows/tmate-mac.yml b/.github/workflows/tmate-mac.yml index fcee0f21..a033cd92 100644 --- a/.github/workflows/tmate-mac.yml +++ b/.github/workflows/tmate-mac.yml @@ -10,6 +10,6 @@ jobs: build: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/tmate.yml b/.github/workflows/tmate.yml index 123f6c71..72af1eec 100644 --- a/.github/workflows/tmate.yml +++ b/.github/workflows/tmate.yml @@ -11,7 +11,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Setup tmate session uses: mxschmitt/action-tmate@v3 env: From c1b30818633e5cbb43d32bf163f99b0aacc391b7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 May 2026 16:44:28 -0700 Subject: [PATCH 012/113] Removed obsolete workflow --- .github/workflows/deploy-branch-preview.yml | 35 --------------------- 1 file changed, 35 deletions(-) delete mode 100644 .github/workflows/deploy-branch-preview.yml diff --git a/.github/workflows/deploy-branch-preview.yml b/.github/workflows/deploy-branch-preview.yml deleted file mode 100644 index 4aa676da..00000000 --- a/.github/workflows/deploy-branch-preview.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Deploy a Datasette branch preview to Vercel - -on: - workflow_dispatch: - inputs: - branch: - description: "Branch to deploy" - required: true - type: string - -jobs: - deploy-branch-preview: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Set up Python 3.11 - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - name: Install dependencies - run: | - pip install datasette-publish-vercel - - name: Deploy the preview - env: - VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }} - run: | - export BRANCH="${{ github.event.inputs.branch }}" - wget https://latest.datasette.io/fixtures.db - datasette publish vercel fixtures.db \ - --branch $BRANCH \ - --project "datasette-preview-$BRANCH" \ - --token $VERCEL_TOKEN \ - --scope datasette \ - --about "Preview of $BRANCH" \ - --about_url "https://github.com/simonw/datasette/tree/$BRANCH" From 40e78e09277ae547dc63172f6b2bad50b0f24b64 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 16 May 2026 16:48:10 -0700 Subject: [PATCH 013/113] Change pull_request_target to pull_request event --- .github/workflows/documentation-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index a54bd83a..b8fb8aaa 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -1,6 +1,6 @@ name: Read the Docs Pull Request Preview on: - pull_request_target: + pull_request: types: - opened From 7a914f8c656de2ffa3f662e49bc95b24dd36b854 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 May 2026 12:14:50 -0700 Subject: [PATCH 014/113] Clear stale tables/other resources when DB removed, closes #2723 --- datasette/app.py | 23 +++++++++++++++---- docs/changelog.rst | 2 +- tests/test_internal_db.py | 48 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 358081ef..218d40c6 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -618,11 +618,24 @@ class Datasette: stale_databases = set(current_schema_versions.keys()) - set( self.databases.keys() ) - for stale_db_name in stale_databases: - await internal_db.execute_write( - "DELETE FROM catalog_databases WHERE database_name = ?", - [stale_db_name], - ) + if stale_databases: + + def delete_stale_database_catalog(conn): + for stale_db_name in stale_databases: + for table in ( + "catalog_columns", + "catalog_foreign_keys", + "catalog_indexes", + "catalog_views", + "catalog_tables", + "catalog_databases", + ): + conn.execute( + "DELETE FROM {} WHERE database_name = ?".format(table), + [stale_db_name], + ) + + await internal_db.execute_write_fn(delete_stale_database_catalog) for database_name, db in self.databases.items(): schema_version = (await db.execute("PRAGMA schema_version")).first()[0] # Compare schema versions to see if we should skip it diff --git a/docs/changelog.rst b/docs/changelog.rst index 5b637797..eb408287 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ Unreleased ---------- - Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) - +- Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) .. _v1_0_a29: diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index 7a0d1630..ec013b43 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -139,3 +139,51 @@ async def test_stale_catalog_entry_database_fix(tmp_path): f"Index page should return 200, not {response.status_code}. " "This fails due to stale catalog entries causing KeyError." ) + + +@pytest.mark.asyncio +async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path): + from datasette.app import Datasette + + import sqlite3 + + internal_db_path = str(tmp_path / "internal.db") + alpha_db_path = str(tmp_path / "alpha.db") + bravo_db_path = str(tmp_path / "bravo.db") + + for db_path, table_name in ( + (alpha_db_path, "alpha_table"), + (bravo_db_path, "bravo_table"), + (bravo_db_path, "bravo_table_2"), + ): + conn = sqlite3.connect(db_path) + conn.execute(f"CREATE TABLE {table_name} (id INTEGER PRIMARY KEY)") + conn.close() + + ds1 = Datasette(files=[alpha_db_path, bravo_db_path], internal=internal_db_path) + await ds1.invoke_startup() + + catalog_tables = await ds1.get_internal_database().execute(""" + SELECT database_name, table_name + FROM catalog_tables + ORDER BY database_name, table_name + """) + assert [tuple(row) for row in catalog_tables.rows] == [ + ("alpha", "alpha_table"), + ("bravo", "bravo_table"), + ("bravo", "bravo_table_2"), + ] + + ds1.close() + + ds2 = Datasette(files=[alpha_db_path], internal=internal_db_path) + await ds2.invoke_startup() + + catalog_tables = await ds2.get_internal_database().execute(""" + SELECT database_name, table_name + FROM catalog_tables + ORDER BY database_name, table_name + """) + assert [tuple(row) for row in catalog_tables.rows] == [("alpha", "alpha_table")] + + ds2.close() From 5d6de0154d18d0ed07e7ac7cf02cd3c37324ddbc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 May 2026 12:18:01 -0700 Subject: [PATCH 015/113] Bump Black to black==26.3.1 Refs https://github.com/advisories/GHSA-3936-cmfr-pm3m --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c50c720a..38085476 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ dev = [ "pytest-xdist>=2.2.1", "pytest-asyncio>=1.2.0", "beautifulsoup4>=4.8.1", - "black==26.1.0", + "black==26.3.1", "blacken-docs==1.20.0", "pytest-timeout>=1.4.2", "trustme>=0.7", From bbbc1cd59620c7c53a2eff6138ef338c8901ba6e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 May 2026 12:33:33 -0700 Subject: [PATCH 016/113] Remove height: 100% to fix Safari bug, closes #2724 --- datasette/static/navigation-search.js | 1 - docs/changelog.rst | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index 95e7dfc5..d2c300e2 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -54,7 +54,6 @@ class NavigationSearch extends HTMLElement { .search-container { display: flex; flex-direction: column; - height: 100%; } .search-input-wrapper { diff --git a/docs/changelog.rst b/docs/changelog.rst index eb408287..56c49ea3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased - Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`) - Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`) +- Fixed a Safari bug with the table search mechanism triggered by pressing ``/``. (:issue:`2724`) .. _v1_0_a29: From 54b272baf61cb014e6d262fea1b01dc84981c1d0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 May 2026 12:39:54 -0700 Subject: [PATCH 017/113] Remove existing stale catalog_ tables, refs #2723 Now if there are any existing stale records in internal.db those will be removed as well. --- datasette/app.py | 30 ++++++++++++++++---------- tests/test_internal_db.py | 45 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 218d40c6..b1f9b2f7 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -614,22 +614,30 @@ class Datasette: "select database_name, schema_version from catalog_databases" ) } - # Delete stale entries for databases that are no longer attached - stale_databases = set(current_schema_versions.keys()) - set( - self.databases.keys() + catalog_table_names = ( + "catalog_columns", + "catalog_foreign_keys", + "catalog_indexes", + "catalog_views", + "catalog_tables", + "catalog_databases", ) + # Delete stale entries for databases that are no longer attached + catalog_database_names = set(current_schema_versions.keys()) + for table in catalog_table_names[:-1]: + catalog_database_names.update( + row["database_name"] + for row in await internal_db.execute( + "select distinct database_name from {}".format(table) + ) + if row["database_name"] is not None + ) + stale_databases = catalog_database_names - set(self.databases.keys()) if stale_databases: def delete_stale_database_catalog(conn): for stale_db_name in stale_databases: - for table in ( - "catalog_columns", - "catalog_foreign_keys", - "catalog_indexes", - "catalog_views", - "catalog_tables", - "catalog_databases", - ): + for table in catalog_table_names: conn.execute( "DELETE FROM {} WHERE database_name = ?".format(table), [stale_db_name], diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index ec013b43..dcf14126 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -187,3 +187,48 @@ async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path assert [tuple(row) for row in catalog_tables.rows] == [("alpha", "alpha_table")] ds2.close() + + +@pytest.mark.asyncio +async def test_orphan_stale_catalog_child_entries_removed(tmp_path): + from datasette.app import Datasette + + import sqlite3 + + internal_db_path = str(tmp_path / "internal.db") + alpha_db_path = str(tmp_path / "alpha.db") + + conn = sqlite3.connect(alpha_db_path) + conn.execute("CREATE TABLE alpha_table (id INTEGER PRIMARY KEY)") + conn.close() + + ds1 = Datasette(files=[alpha_db_path], internal=internal_db_path) + await ds1.invoke_startup() + ds1.close() + + # Simulate the state left behind by old cleanup code: the parent database + # row was deleted, but child catalog rows survived because foreign key + # enforcement is not enabled for these internal catalog writes. + conn = sqlite3.connect(internal_db_path) + conn.execute("DELETE FROM catalog_databases WHERE database_name = 'fixtures'") + conn.execute(""" + INSERT INTO catalog_tables (database_name, table_name, rootpage, sql) + VALUES ('fixtures', 'stale_table', 1, 'CREATE TABLE stale_table (id INTEGER)') + """) + conn.commit() + conn.close() + + ds2 = Datasette(files=[alpha_db_path], internal=internal_db_path) + await ds2.invoke_startup() + + catalog_tables = await ds2.get_internal_database().execute(""" + SELECT database_name, table_name + FROM catalog_tables + ORDER BY database_name, table_name + """) + assert [tuple(row) for row in catalog_tables.rows] == [("alpha", "alpha_table")] + + response = await ds2.client.get("/-/tables.json") + assert response.status_code == 200 + + ds2.close() From d3330695fa42ad1cdb2f2b1b80470e95bea8ed12 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 20 May 2026 13:23:05 -0700 Subject: [PATCH 018/113] Always show 'Jump to...' menu item, closes #2725 --- datasette/static/app.css | 26 ++++++++++++++++++++++++++ datasette/static/navigation-search.js | 13 ++++++++++++- datasette/templates/base.html | 7 +++---- docs/changelog.rst | 1 + tests/test_html.py | 18 +++++++++++++++--- 5 files changed, 57 insertions(+), 8 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index 1ce84bc8..c21d0dc4 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -362,6 +362,32 @@ form.nav-menu-logout { .nav-menu-inner a { display: block; } +.nav-menu-inner button.button-as-link { + display: block; + width: 100%; + text-align: left; + font: inherit; +} +.nav-menu-inner .keyboard-shortcut { + float: right; + box-sizing: border-box; + min-width: 1.4em; + margin-left: 0.75rem; + padding: 0 0.35em; + border: 1px solid rgba(255,255,244,0.6); + border-radius: 3px; + background: rgba(255,255,244,0.12); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.85em; + line-height: 1.35; + text-align: center; + text-decoration: none; +} +@media (max-width: 640px) { + .nav-menu-inner .keyboard-shortcut { + display: none; + } +} /* Table/database actions menu */ .page-action-menu { diff --git a/datasette/static/navigation-search.js b/datasette/static/navigation-search.js index d2c300e2..09d58898 100644 --- a/datasette/static/navigation-search.js +++ b/datasette/static/navigation-search.js @@ -199,6 +199,15 @@ class NavigationSearch extends HTMLElement { } }); + document.addEventListener("click", (e) => { + const trigger = e.target.closest("[data-navigation-search-open]"); + if (trigger) { + e.preventDefault(); + trigger.closest("details")?.removeAttribute("open"); + this.openMenu(); + } + }); + // Input event input.addEventListener("input", (e) => { this.handleSearch(e.target.value); @@ -390,7 +399,9 @@ class NavigationSearch extends HTMLElement { const dialog = this.shadowRoot.querySelector("dialog"); const input = this.shadowRoot.querySelector(".search-input"); - dialog.showModal(); + if (!dialog.open) { + dialog.showModal(); + } input.value = ""; input.focus(); diff --git a/datasette/templates/base.html b/datasette/templates/base.html index 21f8c693..b4fecf70 100644 --- a/datasette/templates/base.html +++ b/datasette/templates/base.html @@ -20,7 +20,7 @@