Support sqlite3.connect(autocommit=False) connections too

Database() now accepts all three Python transaction handling modes and
behaves identically in each. For autocommit=False connections - where
the driver holds an implicit transaction open at all times - the
library tracks transaction ownership itself:

- A new _explicit_transaction flag distinguishes transactions opened
  with begin()/atomic() from the driver's implicit one, exposed as a
  new db.in_transaction property (conn.in_transaction is always True
  in this mode)
- begin() claims the driver's implicit transaction instead of
  executing BEGIN, which the driver would reject; BEGIN/COMMIT/
  ROLLBACK passed to db.execute() are routed through begin()/commit()/
  rollback()
- Writes outside a user transaction commit the implicit transaction
  immediately, preserving the library's auto-commit contract
- Row-returning statements outside a transaction fetch eagerly and
  commit, so the implicit read transaction does not hold a shared
  lock that blocks writes from other connections
- PRAGMA and VACUUM run in temporary driver autocommit mode
  (ensure_autocommit_on() now flips conn.autocommit), since PRAGMAs
  are silently ignored and VACUUM refused inside the implicit
  transaction

A new pytest --sqlite-autocommit-false option runs the entire suite
in this mode, wired into CI alongside --sqlite-autocommit. Tests that
asserted on conn.in_transaction or wrote through db.conn without
committing now use the mode-aware library API instead.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PFkrS2nuP8mo1jmT594VCd
This commit is contained in:
Claude 2026-07-09 22:06:46 +00:00
commit adfcbb4731
No known key found for this signature in database
15 changed files with 323 additions and 94 deletions

View file

@ -18,6 +18,15 @@ def pytest_addoption(parser):
"sqlite3.connect(autocommit=True) mode"
),
)
parser.addoption(
"--sqlite-autocommit-false",
action="store_true",
default=False,
help=(
"Run every test against connections created with the Python 3.12+ "
"sqlite3.connect(autocommit=False) mode"
),
)
def pytest_configure(config):
@ -25,15 +34,22 @@ def pytest_configure(config):
sys._called_from_test = True # type: ignore[attr-defined]
if config.getoption("--sqlite-autocommit"):
autocommit_true = config.getoption("--sqlite-autocommit")
autocommit_false = config.getoption("--sqlite-autocommit-false")
if autocommit_true and autocommit_false:
raise pytest.UsageError(
"--sqlite-autocommit and --sqlite-autocommit-false are mutually exclusive"
)
if autocommit_true or autocommit_false:
if sys.version_info < (3, 12):
raise pytest.UsageError(
"--sqlite-autocommit requires Python 3.12 or higher"
"--sqlite-autocommit and --sqlite-autocommit-false require "
"Python 3.12 or higher"
)
real_connect = sqlite3.connect
def autocommit_connect(*args, **kwargs):
kwargs.setdefault("autocommit", True)
kwargs.setdefault("autocommit", autocommit_true)
return real_connect(*args, **kwargs)
sqlite3.connect = autocommit_connect
@ -81,5 +97,6 @@ def db_path(tmpdir):
path = str(tmpdir / "test.db")
db = sqlite3.connect(path)
db.executescript(CREATE_TABLES)
db.commit()
db.close()
return path

View file

@ -135,8 +135,13 @@ def test_analyze_column(db_to_analyze, column, extra_kwargs, expected):
def db_to_analyze_path(db_to_analyze, tmpdir):
path = str(tmpdir / "test.db")
db = sqlite3.connect(path)
if getattr(db, "autocommit", None) is False:
# iterdump scripts contain BEGIN TRANSACTION, which the driver's
# always-open implicit transaction would reject
db.autocommit = True
sql = "\n".join(db_to_analyze.iterdump())
db.executescript(sql)
db.commit()
db.close()
return path

View file

@ -121,7 +121,7 @@ def test_transform_does_not_commit_open_atomic_block(fresh_db):
def test_transform_parent_table_with_foreign_keys_in_atomic(fresh_db):
fresh_db.conn.execute("PRAGMA foreign_keys=ON")
fresh_db.execute("PRAGMA foreign_keys=ON")
fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id")
fresh_db["books"].insert(
{"id": 1, "title": "Book", "author_id": 1},
@ -141,7 +141,7 @@ def test_transform_parent_table_with_foreign_keys_in_atomic(fresh_db):
def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db):
fresh_db.conn.execute("PRAGMA foreign_keys=ON")
fresh_db.execute("PRAGMA foreign_keys=ON")
fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id")
fresh_db["books"].insert(
{"id": 1, "title": "Book", "author_id": 1},
@ -163,7 +163,7 @@ def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db):
def test_transform_detects_foreign_key_check_violations(fresh_db):
fresh_db.conn.execute("PRAGMA foreign_keys=ON")
fresh_db.execute("PRAGMA foreign_keys=ON")
fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id")
fresh_db["books"].insert({"id": 1, "author_id": 2}, pk="id")
@ -180,7 +180,7 @@ def test_atomic_inside_manual_transaction_uses_savepoint(fresh_db):
with fresh_db.atomic():
fresh_db["t"].insert({"id": 2}, pk="id")
# Nothing is committed until the user's own transaction commits
assert fresh_db.conn.in_transaction
assert fresh_db.in_transaction
fresh_db.rollback()
assert [r["id"] for r in fresh_db["t"].rows] == [1]
# And with a commit instead, the atomic block's writes persist
@ -197,9 +197,9 @@ def test_begin_commit_rollback(tmpdir):
db["t"].insert({"id": 1}, pk="id")
db.begin()
db["t"].insert({"id": 2}, pk="id")
assert db.conn.in_transaction
assert db.in_transaction
db.rollback()
assert not db.conn.in_transaction
assert not db.in_transaction
assert [r["id"] for r in db["t"].rows] == [1]
db.begin()
db["t"].insert({"id": 3}, pk="id")
@ -220,7 +220,7 @@ def test_begin_inside_transaction_errors(fresh_db):
def test_commit_and_rollback_without_transaction_are_noops(fresh_db):
fresh_db.commit()
fresh_db.rollback()
assert not fresh_db.conn.in_transaction
assert not fresh_db.in_transaction
def test_execute_write_commits_immediately(tmpdir):
@ -229,7 +229,7 @@ def test_execute_write_commits_immediately(tmpdir):
db["t"].insert({"id": 1}, pk="id")
db.execute("insert into t (id) values (2)")
# No implicit transaction is left open
assert not db.conn.in_transaction
assert not db.in_transaction
# A completely separate connection sees the row straight away
other = sqlite3.connect(path)
assert other.execute("select count(*) from t").fetchone()[0] == 2
@ -242,7 +242,7 @@ def test_execute_write_respects_explicit_transaction(fresh_db):
fresh_db.begin()
fresh_db.execute("insert into t (id) values (2)")
# Still inside the explicit transaction - not committed
assert fresh_db.conn.in_transaction
assert fresh_db.in_transaction
fresh_db.rollback()
assert [r["id"] for r in fresh_db["t"].rows] == [1]
@ -252,7 +252,7 @@ def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db):
# out from under the caller
fresh_db["t"].insert({"id": 1}, pk="id")
fresh_db.execute("-- start a transaction\nbegin")
assert fresh_db.conn.in_transaction
assert fresh_db.in_transaction
fresh_db.execute("insert into t (id) values (2)")
fresh_db.rollback()
assert [r["id"] for r in fresh_db["t"].rows] == [1]
@ -275,7 +275,7 @@ def test_execute_prefixed_begin_leaves_transaction_open(fresh_db, begin_sql):
pytest.skip("This SQLite version rejects a leading byte order mark")
fresh_db["t"].insert({"id": 1}, pk="id")
fresh_db.execute(begin_sql)
assert fresh_db.conn.in_transaction
assert fresh_db.in_transaction
fresh_db.execute("insert into t (id) values (2)")
fresh_db.rollback()
assert [r["id"] for r in fresh_db["t"].rows] == [1]
@ -289,7 +289,7 @@ def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir):
db["t"].insert({"id": 1}, pk="id")
with pytest.raises(sqlite3.IntegrityError):
db.execute("insert into t (id) values (1)")
assert not db.conn.in_transaction
assert not db.in_transaction
# Subsequent writes commit as normal and survive closing the connection
db["other"].insert({"id": 2})
db.close()
@ -306,7 +306,7 @@ def test_execute_failed_write_preserves_explicit_transaction(fresh_db):
fresh_db.execute("insert into t (id) values (2)")
with pytest.raises(sqlite3.IntegrityError):
fresh_db.execute("insert into t (id) values (1)")
assert fresh_db.conn.in_transaction
assert fresh_db.in_transaction
fresh_db.commit()
assert [r["id"] for r in fresh_db["t"].rows] == [1, 2]
@ -332,7 +332,7 @@ def test_query_returning_commits_after_iteration(tmpdir):
db["t"].insert({"id": 1}, pk="id")
rows = list(db.query("insert into t (id) values (2) returning id"))
assert rows == [{"id": 2}]
assert not db.conn.in_transaction
assert not db.in_transaction
other = sqlite3.connect(path)
assert other.execute("select count(*) from t").fetchone()[0] == 2
other.close()
@ -357,7 +357,7 @@ def test_atomic_preserves_error_from_transaction_destroying_trigger(fresh_db):
with pytest.raises(sqlite3.IntegrityError, match="trigger says no"):
with fresh_db.atomic():
fresh_db.execute("insert into t (v) values ('bad')")
assert not fresh_db.conn.in_transaction
assert not fresh_db.in_transaction
def test_nested_atomic_preserves_error_from_transaction_destroying_trigger(
@ -371,7 +371,7 @@ def test_nested_atomic_preserves_error_from_transaction_destroying_trigger(
with fresh_db.atomic():
with fresh_db.atomic():
fresh_db.execute("insert into t (v) values ('bad')")
assert not fresh_db.conn.in_transaction
assert not fresh_db.in_transaction
def test_atomic_preserves_error_from_insert_or_rollback(fresh_db):
@ -379,4 +379,4 @@ def test_atomic_preserves_error_from_insert_or_rollback(fresh_db):
with pytest.raises(sqlite3.IntegrityError):
with fresh_db.atomic():
fresh_db.execute("insert or rollback into t (id) values (1)")
assert not fresh_db.conn.in_transaction
assert not fresh_db.in_transaction

View file

@ -2200,7 +2200,7 @@ def test_search_quote(tmpdir):
def test_indexes(tmpdir):
db_path = str(tmpdir / "test.db")
db = Database(db_path)
db.conn.executescript("""
db.executescript("""
create table Gosh (c1 text, c2 text, c3 text);
create index Gosh_idx on Gosh(c2, c3 desc);
""")
@ -2294,7 +2294,7 @@ def test_triggers(tmpdir, extra_args, expected):
pk="id",
)
db["counter"].insert({"count": 1})
db.conn.execute(textwrap.dedent("""
db.execute(textwrap.dedent("""
CREATE TRIGGER blah AFTER INSERT ON articles
BEGIN
UPDATE counter SET count = count + 1;

View file

@ -1,5 +1,4 @@
from sqlite_utils import Database
from sqlite_utils.db import TransactionError
from sqlite_utils.utils import sqlite3
import pytest
import sys
@ -65,23 +64,10 @@ def test_database_close(tmpdir, memory):
sys.version_info < (3, 12),
reason="sqlite3.connect(autocommit=) requires Python 3.12",
)
def test_autocommit_false_connections_are_rejected(tmpdir):
# autocommit=False keeps an implicit transaction open at all times,
# which breaks the explicit transaction handling used by every write
# method, so the constructor refuses these connections
conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=False)
with pytest.raises(TransactionError):
Database(conn)
conn.close()
@pytest.mark.skipif(
sys.version_info < (3, 12),
reason="sqlite3.connect(autocommit=) requires Python 3.12",
)
def test_autocommit_true_connection_writes_persist(tmpdir):
@pytest.mark.parametrize("autocommit", [True, False])
def test_autocommit_connection_writes_persist(tmpdir, autocommit):
path = str(tmpdir / "test.db")
conn = sqlite3.connect(path, autocommit=True)
conn = sqlite3.connect(path, autocommit=autocommit)
db = Database(conn)
db["t"].insert({"id": 1}, pk="id")
db.execute("insert into t (id) values (2)")
@ -96,8 +82,9 @@ def test_autocommit_true_connection_writes_persist(tmpdir):
sys.version_info < (3, 12),
reason="sqlite3.connect(autocommit=) requires Python 3.12",
)
def test_autocommit_true_connection_transactions(tmpdir):
conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=True)
@pytest.mark.parametrize("autocommit", [True, False])
def test_autocommit_connection_transactions(tmpdir, autocommit):
conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=autocommit)
db = Database(conn)
db["t"].insert({"id": 1}, pk="id")
# atomic() rolls back on error
@ -124,8 +111,9 @@ def test_autocommit_true_connection_transactions(tmpdir):
sys.version_info < (3, 12),
reason="sqlite3.connect(autocommit=) requires Python 3.12",
)
def test_autocommit_true_connection_wal(tmpdir):
conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=True)
@pytest.mark.parametrize("autocommit", [True, False])
def test_autocommit_connection_wal(tmpdir, autocommit):
conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=autocommit)
db = Database(conn)
db.enable_wal()
assert db.journal_mode == "wal"

View file

@ -42,7 +42,7 @@ def test_delete_where_commits(tmpdir):
db["table"].delete_where("id > ?", [2])
# The connection must not be left inside an open transaction,
# otherwise subsequent atomic() blocks never commit either
assert not db.conn.in_transaction
assert not db.in_transaction
db["table"].insert({"id": 100})
db.close()
db2 = sqlite_utils.Database(path)

View file

@ -360,7 +360,7 @@ def test_optimize_and_rebuild_fts_commit(tmpdir, method):
getattr(table, method)()
# The connection must not be left inside an open transaction,
# otherwise this and all subsequent writes are lost on close
assert not db.conn.in_transaction
assert not db.in_transaction
table.insert(search_records[1])
db.close()
db2 = Database(path)

View file

@ -24,14 +24,14 @@ def test_query_rejects_statements_that_return_no_rows(fresh_db):
fresh_db.query("update dogs set name = 'Cleopaws'")
assert "execute()" in str(ex.value)
# The rejected update was rolled back, and no transaction is left open
assert not fresh_db.conn.in_transaction
assert not fresh_db.in_transaction
assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
def test_query_rejected_ddl_is_rolled_back(fresh_db):
with pytest.raises(ValueError):
fresh_db.query("create table dogs (id integer primary key)")
assert not fresh_db.conn.in_transaction
assert not fresh_db.in_transaction
assert fresh_db.table_names() == []
@ -42,7 +42,7 @@ def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db):
with pytest.raises(ValueError):
fresh_db.query("update dogs set name = 'Cleopaws'")
# The transaction is still open and the earlier insert is intact
assert fresh_db.conn.in_transaction
assert fresh_db.in_transaction
fresh_db.commit()
assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo", "Pancakes"]
@ -69,7 +69,7 @@ def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql):
with pytest.raises(ValueError) as ex:
fresh_db.query(sql)
assert "execute()" in str(ex.value)
assert not fresh_db.conn.in_transaction
assert not fresh_db.in_transaction
def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db):
@ -82,7 +82,7 @@ def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db):
with pytest.raises(ValueError):
fresh_db.query("/* comment */ COMMIT")
# The explicit transaction is still open and can still be rolled back
assert fresh_db.conn.in_transaction
assert fresh_db.in_transaction
fresh_db.rollback()
assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
@ -99,7 +99,7 @@ def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql):
with pytest.raises(ValueError):
fresh_db.query(sql)
# The explicit transaction is still open and can still be rolled back
assert fresh_db.conn.in_transaction
assert fresh_db.in_transaction
fresh_db.rollback()
assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
@ -107,7 +107,7 @@ def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql):
def test_query_error_leaves_no_transaction_open(fresh_db):
with pytest.raises(sqlite3.OperationalError):
fresh_db.query("select * from missing_table")
assert not fresh_db.conn.in_transaction
assert not fresh_db.in_transaction
def test_query_pragma(tmpdir):
@ -152,7 +152,7 @@ def test_query_comment_prefixed_pragma_inside_transaction(fresh_db):
assert list(fresh_db.query("-- check version\npragma user_version")) == [
{"user_version": 0}
]
assert fresh_db.conn.in_transaction
assert fresh_db.in_transaction
fresh_db.rollback()
@ -209,7 +209,7 @@ def test_query_insert_returning_commits_without_iteration(tmpdir):
db["dogs"].insert({"name": "Cleo"})
# Never iterate over the results
db.query("insert into dogs (name) values ('Pancakes') returning name")
assert not db.conn.in_transaction
assert not db.in_transaction
# A completely separate connection sees the new row straight away
other = sqlite3.connect(path)
assert other.execute("select count(*) from dogs").fetchone()[0] == 2
@ -233,7 +233,7 @@ def test_query_insert_returning_partial_iteration_still_commits(tmpdir):
)
)
assert row == {"name": "Pancakes"}
assert not db.conn.in_transaction
assert not db.in_transaction
other = sqlite3.connect(path)
assert other.execute("select count(*) from dogs").fetchone()[0] == 3
other.close()
@ -252,7 +252,7 @@ def test_query_insert_returning_respects_explicit_transaction(fresh_db):
)
assert rows == [{"name": "Pancakes"}]
# Still inside the explicit transaction - not committed
assert fresh_db.conn.in_transaction
assert fresh_db.in_transaction
fresh_db.rollback()
assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
@ -299,5 +299,5 @@ def test_query_preserves_error_from_transaction_destroying_trigger(fresh_db):
""")
with pytest.raises(sqlite3.IntegrityError, match="trigger says no"):
fresh_db.query("insert into t (id, v) values (1, 'bad') returning id")
assert not fresh_db.conn.in_transaction
assert not fresh_db.in_transaction
assert fresh_db.execute("select count(*) from t").fetchone()[0] == 0

View file

@ -109,7 +109,7 @@ def test_transform_sql_table_with_primary_key(
dogs = fresh_db["dogs"]
if use_pragma_foreign_keys:
fresh_db.conn.execute("PRAGMA foreign_keys=ON")
fresh_db.execute("PRAGMA foreign_keys=ON")
dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id")
sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}})
assert sql == expected_sql
@ -182,7 +182,7 @@ def test_transform_sql_table_with_no_primary_key(
dogs = fresh_db["dogs"]
if use_pragma_foreign_keys:
fresh_db.conn.execute("PRAGMA foreign_keys=ON")
fresh_db.execute("PRAGMA foreign_keys=ON")
dogs.insert({"id": 1, "name": "Cleo", "age": "5"})
sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}})
assert sql == expected_sql
@ -351,7 +351,7 @@ def test_transform_foreign_keys_survive_renamed_column(
authors_db, use_pragma_foreign_keys
):
if use_pragma_foreign_keys:
authors_db.conn.execute("PRAGMA foreign_keys=ON")
authors_db.execute("PRAGMA foreign_keys=ON")
authors_db["books"].transform(rename={"author_id": "author_id_2"})
assert authors_db["books"].foreign_keys == [
ForeignKey(
@ -381,7 +381,7 @@ _CAVEAU = {
@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True])
def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys):
if use_pragma_foreign_keys:
fresh_db.conn.execute("PRAGMA foreign_keys=ON")
fresh_db.execute("PRAGMA foreign_keys=ON")
# Create table with three foreign keys so we can drop two of them
_add_country_city_continent(fresh_db)
fresh_db["places"].insert(
@ -413,7 +413,7 @@ def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys):
def test_transform_verify_foreign_keys(fresh_db):
fresh_db.conn.execute("PRAGMA foreign_keys=ON")
fresh_db.execute("PRAGMA foreign_keys=ON")
fresh_db["authors"].insert({"id": 3, "name": "Tina"}, pk="id")
fresh_db["books"].insert(
{"id": 1, "title": "Book", "author_id": 3}, pk="id", foreign_keys={"author_id"}

View file

@ -52,12 +52,17 @@ def test_disable_wal_inside_transaction_raises(db_path_tmpdir):
def test_ensure_autocommit_on(db_path_tmpdir):
db, path, tmpdir = db_path_tmpdir
previous_isolation_level = db.conn.isolation_level
assert previous_isolation_level is not None
previous_autocommit = getattr(db.conn, "autocommit", None)
with db.ensure_autocommit_on():
# isolation_level of None means driver-level autocommit mode
assert db.conn.isolation_level is None
# Driver-level autocommit mode: raw writes on the connection do not
# open an implicit transaction, they commit immediately
db.conn.execute("create table t1 (id integer)")
db.conn.execute("insert into t1 values (1)")
assert not db.conn.in_transaction
# Restored afterwards
assert db.conn.isolation_level == previous_isolation_level
assert getattr(db.conn, "autocommit", None) == previous_autocommit
assert db.execute("select count(*) from t1").fetchone()[0] == 1
def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir):
@ -83,6 +88,6 @@ def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir):
with db.ensure_autocommit_on():
pass
# The transaction is still open and can still be rolled back
assert db.conn.in_transaction
assert db.in_transaction
db.rollback()
assert [r["id"] for r in db["test"].rows] == [1]