remote/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2026-09-11 11:04:07 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add opt-in journal_mode setting, enable WAL on persistent internal DB

Browse source 
New journal_mode setting lets deployments opt mutable database files
into WAL mode (or delete/truncate/persist), applied on the write
connection. WAL is paired with PRAGMA synchronous=NORMAL. Datasette
does not change the journal mode of database files by default.

Also fixes an inconsistency: a persistent internal database passed via
--internal now gets WAL enabled, matching the temporary internal
database default (which was moved to a temp disk file specifically so
it could use WAL).

Refs #2831

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01N76afGMhBRQk528VF1LTpR
This commit is contained in:
Claude 2026-07-09 06:10:15 +00:00
commit 3713d6c0d5
No known key found for this signature in database
6 changed files with 107 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

11
datasette/app.py
View file

@ -225,6 +225,11 @@ SETTINGS = (
5000,
"How long SQLite waits for a locked database file before giving up",
),
Setting(
"journal_mode",
"",
"Journal mode to set on mutable database files, e.g. wal - leave blank to use each file's existing mode",
),
Setting(
"default_facet_size", 30, "Number of values to return for requested facets"
),
@ -483,7 +488,11 @@ class Datasette:
if internal is None:
self._internal_database = Database(self, is_temp_disk=True)
else:
self._internal_database = Database(self, path=internal, mode="rwc")
# WAL for the same reason as the temporary internal database:
# the catalog can be read while it is being rewritten
self._internal_database = Database(
self, path=internal, mode="rwc", enable_wal=True
)
self._internal_database.name = INTERNAL_DB_NAME
self.cache_headers = cache_headers

24
datasette/database.py
View file

@ -36,6 +36,8 @@ EXECUTE_WRITE_RETURNING_LIMIT = 10
AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file"))
ALLOWED_JOURNAL_MODES = {"delete", "truncate", "persist", "wal"}
class DatasetteClosedError(RuntimeError):
"""Raised when using a Datasette or Database instance after close()."""
@ -129,6 +131,7 @@ class Database:
memory_name=None,
mode=None,
is_temp_disk=False,
enable_wal=False,
):
self.name = None
self._thread_local_id = f"x{self._thread_local_id_counter}"
@ -140,6 +143,8 @@ class Database:
self.is_memory = is_memory
self.memory_name = memory_name
self.is_temp_disk = is_temp_disk
self.enable_wal = enable_wal or is_temp_disk
self._wal_enabled = False
if memory_name is not None:
self.is_memory = True
if is_temp_disk:
@ -148,10 +153,7 @@ class Database:
self.path = temp_path
self.is_mutable = True
self.mode = "rwc"
self._wal_enabled = False
atexit.register(self._cleanup_temp_file)
else:
self._wal_enabled = False
self.cached_hash = None
self.cached_size = None
self._cached_table_counts = None
@ -241,9 +243,23 @@ class Database:
f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs
)
self._all_file_connections.append(conn)
if self.is_temp_disk and not self._wal_enabled:
if self.enable_wal and not self._wal_enabled:
conn.execute("PRAGMA journal_mode=WAL")
self._wal_enabled = True
if write and self.is_mutable and not self.enable_wal:
journal_mode = self.ds.setting("journal_mode")
if journal_mode:
if journal_mode not in ALLOWED_JOURNAL_MODES:
raise ValueError(
"journal_mode setting must be one of: {}".format(
", ".join(sorted(ALLOWED_JOURNAL_MODES))
)
)
conn.execute("PRAGMA journal_mode={}".format(journal_mode))
if journal_mode == "wal":
# The standard WAL pairing - fewer fsyncs, application
# level consistency guarantees are unchanged
conn.execute("PRAGMA synchronous=NORMAL")
return conn
def close(self):

2
docs/changelog.rst
View file

@ -16,6 +16,8 @@ Unreleased
- Rebuilding the internal database catalog for a database is now a single atomic write. Previously the rebuild used six separate transactions, so queries against the internal database could observe a database with missing catalog rows while a rebuild was in progress. (:issue:`2831`)
- sqlite-utils plugins no longer have their ``prepare_connection()`` hooks executed against Datasette's database connections - use Datasette's own :ref:`prepare_connection() <plugin_hook_prepare_connection>` plugin hook to customize connections. ``PRAGMA recursive_triggers=on`` is now applied consistently to every connection Datasette opens - previously it was enabled just on the write connection, as a side effect of the first sqlite-utils based write. (:issue:`2831`)
- New :ref:`setting_busy_timeout_ms` setting controlling how long SQLite waits for a locked database file before giving up, previously hard-wired to the ``sqlite3`` driver's silent 5 second default. This matters when external processes write to the same database files Datasette is serving. (:issue:`2831`)
- New :ref:`setting_journal_mode` setting for opting mutable database files into SQLite's `WAL mode <https://www.sqlite.org/wal.html>`__ (or another journal mode), allowing reads and writes to proceed concurrently. Datasette does not change the journal mode of database files by default. (:issue:`2831`)
- A persistent internal database specified with ``--internal`` now has WAL mode enabled, matching the behavior of the default temporary internal database. (:issue:`2831`)
.. _v1_0_a36:

3
docs/cli-reference.rst
View file

@ -253,6 +253,9 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam
(default=1000)
busy_timeout_ms How long SQLite waits for a locked database file
before giving up (default=5000)
journal_mode Journal mode to set on mutable database files,
e.g. wal - leave blank to use each file's
existing mode (default=)
default_facet_size Number of values to return for requested facets
(default=30)
facet_time_limit_ms Time limit for calculating a requested facet

22
docs/settings.rst
View file

@ -114,6 +114,28 @@ This mostly matters when other processes write to the same database files that D
datasette mydatabase.db --setting busy_timeout_ms 10000
.. _setting_journal_mode:
journal_mode
~~~~~~~~~~~~
`Journal mode <https://www.sqlite.org/pragma.html#pragma_journal_mode>`__ to set on mutable database files. Leave blank (the default) to use whatever journal mode each database file already uses.
Setting this to ``wal`` enables SQLite's `Write-Ahead Logging <https://www.sqlite.org/wal.html>`__ mode, which allows reads and writes to proceed concurrently - in the default rollback journal mode each commit blocks readers, and long reads block the writer::
datasette mydatabase.db --setting journal_mode wal
When ``wal`` is selected Datasette also sets ``PRAGMA synchronous=NORMAL`` on the write connection, the standard pairing for WAL which reduces the number of ``fsync`` operations without weakening application-level consistency guarantees.
Other allowed values are ``delete``, ``truncate`` and ``persist``.
Things to be aware of before enabling WAL:
- WAL mode is persistent - it is recorded in the database file and stays in effect when other tools open the same file later.
- SQLite creates ``-wal`` and ``-shm`` files alongside the database file.
- WAL does not work reliably on network filesystems such as NFS.
- The directory containing the database must be writable by Datasette.
.. _setting_max_returned_rows:
max_returned_rows

50
tests/test_internals_database.py
View file

@ -1356,3 +1356,53 @@ async def test_busy_timeout_ms_default(tmp_path):
lambda conn: conn.execute("PRAGMA busy_timeout").fetchone()[0]
)
assert read_value == 5000
@pytest.mark.asyncio
async def test_journal_mode_setting_applies_wal(tmp_path):
# https://github.com/simonw/datasette/issues/2831
# Opt-in WAL support for mutable database files - not the default
path = str(tmp_path / "test.db")
sqlite3.connect(path).close()
datasette = Datasette([path], settings={"journal_mode": "wal"})
db = datasette.get_database("test")
mode = await db.execute_write_fn(
lambda conn: conn.execute("PRAGMA journal_mode").fetchone()[0],
transaction=False,
)
assert mode == "wal"
# WAL is paired with synchronous=NORMAL (1) on the write connection
synchronous = await db.execute_write_fn(
lambda conn: conn.execute("PRAGMA synchronous").fetchone()[0],
transaction=False,
)
assert synchronous == 1
@pytest.mark.asyncio
async def test_journal_mode_defaults_to_leaving_files_alone(tmp_path):
path = str(tmp_path / "test.db")
sqlite3.connect(path).close()
datasette = Datasette([path])
db = datasette.get_database("test")
mode = await db.execute_write_fn(
lambda conn: conn.execute("PRAGMA journal_mode").fetchone()[0],
transaction=False,
)
assert mode == "delete"
@pytest.mark.asyncio
async def test_persistent_internal_database_gets_wal(tmp_path):
# https://github.com/simonw/datasette/issues/2831
# The temporary internal database enables WAL - a persistent one passed
# via --internal should get the same treatment
internal_path = str(tmp_path / "internal.db")
datasette = Datasette(memory=True, internal=internal_path)
await datasette.invoke_startup()
internal_db = datasette.get_internal_database()
mode = await internal_db.execute_write_fn(
lambda conn: conn.execute("PRAGMA journal_mode").fetchone()[0],
transaction=False,
)
assert mode == "wal"