From 3713d6c0d59dc03685a0af5318b9f826dbbeb7d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 06:10:15 +0000 Subject: [PATCH] Add opt-in journal_mode setting, enable WAL on persistent internal DB New journal_mode setting lets deployments opt mutable database files into WAL mode (or delete/truncate/persist), applied on the write connection. WAL is paired with PRAGMA synchronous=NORMAL. Datasette does not change the journal mode of database files by default. Also fixes an inconsistency: a persistent internal database passed via --internal now gets WAL enabled, matching the temporary internal database default (which was moved to a temp disk file specifically so it could use WAL). Refs #2831 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01N76afGMhBRQk528VF1LTpR --- datasette/app.py | 11 ++++++- datasette/database.py | 24 ++++++++++++--- docs/changelog.rst | 2 ++ docs/cli-reference.rst | 3 ++ docs/settings.rst | 22 ++++++++++++++ tests/test_internals_database.py | 50 ++++++++++++++++++++++++++++++++ 6 files changed, 107 insertions(+), 5 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 53c41282..7ab1aa66 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -225,6 +225,11 @@ SETTINGS = ( 5000, "How long SQLite waits for a locked database file before giving up", ), + Setting( + "journal_mode", + "", + "Journal mode to set on mutable database files, e.g. wal - leave blank to use each file's existing mode", + ), Setting( "default_facet_size", 30, "Number of values to return for requested facets" ), @@ -483,7 +488,11 @@ class Datasette: if internal is None: self._internal_database = Database(self, is_temp_disk=True) else: - self._internal_database = Database(self, path=internal, mode="rwc") + # WAL for the same reason as the temporary internal database: + # the catalog can be read while it is being rewritten + self._internal_database = Database( + self, path=internal, mode="rwc", enable_wal=True + ) self._internal_database.name = INTERNAL_DB_NAME self.cache_headers = cache_headers diff --git a/datasette/database.py b/datasette/database.py index 877c373b..8963d210 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -36,6 +36,8 @@ EXECUTE_WRITE_RETURNING_LIMIT = 10 AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file")) +ALLOWED_JOURNAL_MODES = {"delete", "truncate", "persist", "wal"} + class DatasetteClosedError(RuntimeError): """Raised when using a Datasette or Database instance after close().""" @@ -129,6 +131,7 @@ class Database: memory_name=None, mode=None, is_temp_disk=False, + enable_wal=False, ): self.name = None self._thread_local_id = f"x{self._thread_local_id_counter}" @@ -140,6 +143,8 @@ class Database: self.is_memory = is_memory self.memory_name = memory_name self.is_temp_disk = is_temp_disk + self.enable_wal = enable_wal or is_temp_disk + self._wal_enabled = False if memory_name is not None: self.is_memory = True if is_temp_disk: @@ -148,10 +153,7 @@ class Database: self.path = temp_path self.is_mutable = True self.mode = "rwc" - self._wal_enabled = False atexit.register(self._cleanup_temp_file) - else: - self._wal_enabled = False self.cached_hash = None self.cached_size = None self._cached_table_counts = None @@ -241,9 +243,23 @@ class Database: f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs ) self._all_file_connections.append(conn) - if self.is_temp_disk and not self._wal_enabled: + if self.enable_wal and not self._wal_enabled: conn.execute("PRAGMA journal_mode=WAL") self._wal_enabled = True + if write and self.is_mutable and not self.enable_wal: + journal_mode = self.ds.setting("journal_mode") + if journal_mode: + if journal_mode not in ALLOWED_JOURNAL_MODES: + raise ValueError( + "journal_mode setting must be one of: {}".format( + ", ".join(sorted(ALLOWED_JOURNAL_MODES)) + ) + ) + conn.execute("PRAGMA journal_mode={}".format(journal_mode)) + if journal_mode == "wal": + # The standard WAL pairing - fewer fsyncs, application + # level consistency guarantees are unchanged + conn.execute("PRAGMA synchronous=NORMAL") return conn def close(self): diff --git a/docs/changelog.rst b/docs/changelog.rst index 0ab86365..515c4610 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,8 @@ Unreleased - Rebuilding the internal database catalog for a database is now a single atomic write. Previously the rebuild used six separate transactions, so queries against the internal database could observe a database with missing catalog rows while a rebuild was in progress. (:issue:`2831`) - sqlite-utils plugins no longer have their ``prepare_connection()`` hooks executed against Datasette's database connections - use Datasette's own :ref:`prepare_connection() ` plugin hook to customize connections. ``PRAGMA recursive_triggers=on`` is now applied consistently to every connection Datasette opens - previously it was enabled just on the write connection, as a side effect of the first sqlite-utils based write. (:issue:`2831`) - New :ref:`setting_busy_timeout_ms` setting controlling how long SQLite waits for a locked database file before giving up, previously hard-wired to the ``sqlite3`` driver's silent 5 second default. This matters when external processes write to the same database files Datasette is serving. (:issue:`2831`) +- New :ref:`setting_journal_mode` setting for opting mutable database files into SQLite's `WAL mode `__ (or another journal mode), allowing reads and writes to proceed concurrently. Datasette does not change the journal mode of database files by default. (:issue:`2831`) +- A persistent internal database specified with ``--internal`` now has WAL mode enabled, matching the behavior of the default temporary internal database. (:issue:`2831`) .. _v1_0_a36: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c495a865..abf94f49 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -253,6 +253,9 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam (default=1000) busy_timeout_ms How long SQLite waits for a locked database file before giving up (default=5000) + journal_mode Journal mode to set on mutable database files, + e.g. wal - leave blank to use each file's + existing mode (default=) default_facet_size Number of values to return for requested facets (default=30) facet_time_limit_ms Time limit for calculating a requested facet diff --git a/docs/settings.rst b/docs/settings.rst index d7b78798..bdbd1834 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -114,6 +114,28 @@ This mostly matters when other processes write to the same database files that D datasette mydatabase.db --setting busy_timeout_ms 10000 +.. _setting_journal_mode: + +journal_mode +~~~~~~~~~~~~ + +`Journal mode `__ to set on mutable database files. Leave blank (the default) to use whatever journal mode each database file already uses. + +Setting this to ``wal`` enables SQLite's `Write-Ahead Logging `__ mode, which allows reads and writes to proceed concurrently - in the default rollback journal mode each commit blocks readers, and long reads block the writer:: + + datasette mydatabase.db --setting journal_mode wal + +When ``wal`` is selected Datasette also sets ``PRAGMA synchronous=NORMAL`` on the write connection, the standard pairing for WAL which reduces the number of ``fsync`` operations without weakening application-level consistency guarantees. + +Other allowed values are ``delete``, ``truncate`` and ``persist``. + +Things to be aware of before enabling WAL: + +- WAL mode is persistent - it is recorded in the database file and stays in effect when other tools open the same file later. +- SQLite creates ``-wal`` and ``-shm`` files alongside the database file. +- WAL does not work reliably on network filesystems such as NFS. +- The directory containing the database must be writable by Datasette. + .. _setting_max_returned_rows: max_returned_rows diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index 53f5e453..2e928e44 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -1356,3 +1356,53 @@ async def test_busy_timeout_ms_default(tmp_path): lambda conn: conn.execute("PRAGMA busy_timeout").fetchone()[0] ) assert read_value == 5000 + + +@pytest.mark.asyncio +async def test_journal_mode_setting_applies_wal(tmp_path): + # https://github.com/simonw/datasette/issues/2831 + # Opt-in WAL support for mutable database files - not the default + path = str(tmp_path / "test.db") + sqlite3.connect(path).close() + datasette = Datasette([path], settings={"journal_mode": "wal"}) + db = datasette.get_database("test") + mode = await db.execute_write_fn( + lambda conn: conn.execute("PRAGMA journal_mode").fetchone()[0], + transaction=False, + ) + assert mode == "wal" + # WAL is paired with synchronous=NORMAL (1) on the write connection + synchronous = await db.execute_write_fn( + lambda conn: conn.execute("PRAGMA synchronous").fetchone()[0], + transaction=False, + ) + assert synchronous == 1 + + +@pytest.mark.asyncio +async def test_journal_mode_defaults_to_leaving_files_alone(tmp_path): + path = str(tmp_path / "test.db") + sqlite3.connect(path).close() + datasette = Datasette([path]) + db = datasette.get_database("test") + mode = await db.execute_write_fn( + lambda conn: conn.execute("PRAGMA journal_mode").fetchone()[0], + transaction=False, + ) + assert mode == "delete" + + +@pytest.mark.asyncio +async def test_persistent_internal_database_gets_wal(tmp_path): + # https://github.com/simonw/datasette/issues/2831 + # The temporary internal database enables WAL - a persistent one passed + # via --internal should get the same treatment + internal_path = str(tmp_path / "internal.db") + datasette = Datasette(memory=True, internal=internal_path) + await datasette.invoke_startup() + internal_db = datasette.get_internal_database() + mode = await internal_db.execute_write_fn( + lambda conn: conn.execute("PRAGMA journal_mode").fetchone()[0], + transaction=False, + ) + assert mode == "wal"