From 14815cb092c96d30411a215cd5256518b7ddd1bb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 06:06:57 +0000 Subject: [PATCH] Add busy_timeout_ms setting The SQLite busy timeout was previously an implicit policy - every connection inherited the sqlite3 driver's silent 5 second default. It is now an explicit, documented setting passed as timeout= to every sqlite3.connect() call. The default remains 5000ms. This matters for deployments where external processes write to the same database files Datasette is serving. Refs #2831 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01N76afGMhBRQk528VF1LTpR --- datasette/app.py | 5 +++++ datasette/database.py | 4 ++++ docs/changelog.rst | 1 + docs/cli-reference.rst | 2 ++ docs/settings.rst | 11 +++++++++++ tests/test_internals_database.py | 33 ++++++++++++++++++++++++++++++++ 6 files changed, 56 insertions(+) diff --git a/datasette/app.py b/datasette/app.py index 987c77df..53c41282 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -220,6 +220,11 @@ SETTINGS = ( "Number of threads in the thread pool for executing SQLite queries", ), Setting("sql_time_limit_ms", 1000, "Time limit for a SQL query in milliseconds"), + Setting( + "busy_timeout_ms", + 5000, + "How long SQLite waits for a locked database file before giving up", + ), Setting( "default_facet_size", 30, "Number of values to return for requested facets" ), diff --git a/datasette/database.py b/datasette/database.py index 6c1b3a9e..877c373b 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -210,6 +210,10 @@ class Database: extra_kwargs = {} if write: extra_kwargs["isolation_level"] = "IMMEDIATE" + # An explicit busy timeout policy rather than the driver's silent + # 5 second default - matters when external processes write to the + # same database files + extra_kwargs["timeout"] = self.ds.setting("busy_timeout_ms") / 1000 if self.memory_name: uri = "file:{}?mode=memory&cache=shared".format(self.memory_name) conn = sqlite3.connect( diff --git a/docs/changelog.rst b/docs/changelog.rst index caa1050f..0ab86365 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,6 +15,7 @@ Unreleased - The JSON write API is now atomic per request: ``/db/-/create`` with initial rows, multi-operation ``/db/table/-/alter`` calls and inserts using ``"return": true`` now either fully apply or roll back entirely if any part fails. Previously a failure part way through could leave earlier writes from the same request permanently committed. (:issue:`2831`) - Rebuilding the internal database catalog for a database is now a single atomic write. Previously the rebuild used six separate transactions, so queries against the internal database could observe a database with missing catalog rows while a rebuild was in progress. (:issue:`2831`) - sqlite-utils plugins no longer have their ``prepare_connection()`` hooks executed against Datasette's database connections - use Datasette's own :ref:`prepare_connection() ` plugin hook to customize connections. ``PRAGMA recursive_triggers=on`` is now applied consistently to every connection Datasette opens - previously it was enabled just on the write connection, as a side effect of the first sqlite-utils based write. (:issue:`2831`) +- New :ref:`setting_busy_timeout_ms` setting controlling how long SQLite waits for a locked database file before giving up, previously hard-wired to the ``sqlite3`` driver's silent 5 second default. This matters when external processes write to the same database files Datasette is serving. (:issue:`2831`) .. _v1_0_a36: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 2302f742..c495a865 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -251,6 +251,8 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam executing SQLite queries (default=3) sql_time_limit_ms Time limit for a SQL query in milliseconds (default=1000) + busy_timeout_ms How long SQLite waits for a locked database file + before giving up (default=5000) default_facet_size Number of values to return for requested facets (default=30) facet_time_limit_ms Time limit for calculating a requested facet diff --git a/docs/settings.rst b/docs/settings.rst index 9c114e4a..d7b78798 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -103,6 +103,17 @@ You can optionally set a lower time limit for an individual query using the ``?_ This would set the time limit to 100ms for that specific query. This feature is useful if you are working with databases of unknown size and complexity - a query that might make perfect sense for a smaller table could take too long to execute on a table with millions of rows. By setting custom time limits you can execute queries "optimistically" - e.g. give me an exact count of rows matching this query but only if it takes less than 100ms to calculate. +.. _setting_busy_timeout_ms: + +busy_timeout_ms +~~~~~~~~~~~~~~~ + +How long SQLite should wait when a database file is locked by another connection or process before giving up with a ``database is locked`` error, in milliseconds. The default is 5000 (five seconds), matching the default used by Python's ``sqlite3`` module. + +This mostly matters when other processes write to the same database files that Datasette is serving - a common pattern is a separate script (using `sqlite-utils `__ or similar) that periodically updates a database while Datasette serves it. A larger value makes Datasette more patient with long write transactions from those processes:: + + datasette mydatabase.db --setting busy_timeout_ms 10000 + .. _setting_max_returned_rows: max_returned_rows diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index fb1adcba..53f5e453 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -1323,3 +1323,36 @@ async def test_recursive_triggers_enabled_on_all_connections(tmp_path): ) assert write_value == 1 assert read_value == 1 + + +@pytest.mark.asyncio +async def test_busy_timeout_ms_setting(tmp_path): + # https://github.com/simonw/datasette/issues/2831 + # The SQLite busy timeout should be an explicit, configurable policy + # instead of the sqlite3 driver's inherited 5 second default + path = str(tmp_path / "test.db") + sqlite3.connect(path).close() + datasette = Datasette([path], settings={"busy_timeout_ms": 250}) + db = datasette.get_database("test") + read_value = await db.execute_fn( + lambda conn: conn.execute("PRAGMA busy_timeout").fetchone()[0] + ) + write_value = await db.execute_write_fn( + lambda conn: conn.execute("PRAGMA busy_timeout").fetchone()[0], + transaction=False, + ) + assert read_value == 250 + assert write_value == 250 + + +@pytest.mark.asyncio +async def test_busy_timeout_ms_default(tmp_path): + # Default matches the sqlite3 driver's historical 5 second default + path = str(tmp_path / "test.db") + sqlite3.connect(path).close() + datasette = Datasette([path]) + db = datasette.get_database("test") + read_value = await db.execute_fn( + lambda conn: conn.execute("PRAGMA busy_timeout").fetchone()[0] + ) + assert read_value == 5000