From 6060a065b5a0caa3eed3d5fbc9868d089b0fe782 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 8 Oct 2026 15:31:48 -0700 Subject: [PATCH] Support SQLite resource error recovery on Python 3.10 --- datasette/schema_watcher.py | 8 ++- datasette/write_budget.py | 13 ++++- tests/test_sqlite_error_compatibility.py | 70 ++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 tests/test_sqlite_error_compatibility.py diff --git a/datasette/schema_watcher.py b/datasette/schema_watcher.py index 85a9c64b..c7dfd8f5 100644 --- a/datasette/schema_watcher.py +++ b/datasette/schema_watcher.py @@ -258,8 +258,11 @@ def _is_transient(error): if isinstance(error, OSError): return error.errno in (errno.EMFILE, errno.ENFILE, errno.ENOMEM, errno.EAGAIN) if isinstance(error, sqlite3.OperationalError): - code = getattr(error, "sqlite_errorcode", 0) & 0xFF - if code in (sqlite3.SQLITE_CANTOPEN, sqlite3.SQLITE_NOMEM): + code = getattr(error, "sqlite_errorcode", None) + if code is not None and code & 0xFF in ( + getattr(sqlite3, "SQLITE_CANTOPEN", None), + getattr(sqlite3, "SQLITE_NOMEM", None), + ): return True message = str(error).lower() return any( @@ -269,6 +272,7 @@ def _is_transient(error): "busy", "unable to open database file", "too many open files", + "out of memory", ) ) return False diff --git a/datasette/write_budget.py b/datasette/write_budget.py index 08f87da8..4ce29a13 100644 --- a/datasette/write_budget.py +++ b/datasette/write_budget.py @@ -65,8 +65,17 @@ def connect_with_retry(db, connect): errno.ENFILE, errno.ENOMEM, ) - cantopen = ( - getattr(error, "sqlite_errorcode", None) == sqlite3.SQLITE_CANTOPEN + # Python 3.10 exposes neither SQLite result constants nor + # sqlite_errorcode. Keep its open-error recovery path working. + code = getattr(error, "sqlite_errorcode", None) + cantopen_code = getattr(sqlite3, "SQLITE_CANTOPEN", None) + cantopen = isinstance(error, sqlite3.OperationalError) and ( + ( + code is not None + and cantopen_code is not None + and code & 0xFF == cantopen_code + ) + or "unable to open database file" in str(error).lower() ) if not pressure and (not cantopen or not os.path.exists(db.path)): raise diff --git a/tests/test_sqlite_error_compatibility.py b/tests/test_sqlite_error_compatibility.py new file mode 100644 index 00000000..b7da5153 --- /dev/null +++ b/tests/test_sqlite_error_compatibility.py @@ -0,0 +1,70 @@ +"""Error recovery must also work without Python 3.11's SQLite error metadata.""" + +import errno +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from datasette.schema_watcher import _is_transient +from datasette.utils import sqlite3 +from datasette.write_budget import DatabaseResourceError, connect_with_retry + + +@pytest.fixture +def old_sqlite_errors(monkeypatch): + for name in ("SQLITE_CANTOPEN", "SQLITE_NOMEM"): + monkeypatch.delattr(sqlite3, name, raising=False) + + +@pytest.mark.parametrize( + "error,expected", + [ + (sqlite3.OperationalError("unable to open database file"), True), + (sqlite3.OperationalError("out of memory"), True), + (sqlite3.OperationalError("database is locked"), True), + (sqlite3.OperationalError("no such table: missing"), False), + (OSError(errno.EMFILE, "too many open files"), True), + ], +) +def test_transient_errors_without_sqlite_constants(old_sqlite_errors, error, expected): + assert _is_transient(error) is expected + + +@pytest.mark.parametrize("os_error", [False, True]) +def test_open_recovers_without_sqlite_constants(old_sqlite_errors, tmp_path, os_error): + path = tmp_path / "db.sqlite" + path.touch() + pool = Mock() + db = SimpleNamespace(path=str(path), ds=SimpleNamespace(_read_pool_or_none=pool)) + error = ( + OSError(errno.EMFILE, "too many open files") + if os_error + else sqlite3.OperationalError("unable to open database file") + ) + connection = object() + connect = Mock(side_effect=[error, connection]) + assert connect_with_retry(db, connect) is connection + assert connect.call_count == 2 + pool.evict_idle.assert_called_once_with() + + +@pytest.mark.parametrize("os_error", [False, True]) +def test_open_failure_preserved_without_sqlite_constants( + old_sqlite_errors, tmp_path, os_error +): + path = tmp_path / "db.sqlite" + path.touch() + db = SimpleNamespace(path=str(path), ds=SimpleNamespace(_read_pool_or_none=None)) + error = ( + OSError(errno.EMFILE, "too many open files") + if os_error + else sqlite3.OperationalError("unable to open database file") + ) + connect = Mock(side_effect=error) + with pytest.raises( + DatabaseResourceError if os_error else sqlite3.OperationalError + ) as exc: + connect_with_retry(db, connect) + assert (exc.value.__cause__ if os_error else exc.value) is error + assert connect.call_count == 2