diff --git a/docs/changelog.rst b/docs/changelog.rst index d2c07da..d2ba16a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Unreleased - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept ``--type column-name type`` to :ref:`override the type automatically chosen when the table is created `. This is useful for CSV or TSV columns such as ZIP codes that look like integers but should be stored as ``TEXT`` to preserve leading zeros. (:issue:`131`) - New ``table.drop_index(name)`` method and ``sqlite-utils drop-index`` command for dropping an index by name. Both accept ``ignore=True``/``--ignore`` to ignore a missing index. (:issue:`626`) +- ``Database()`` now accepts connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` option. The library manages transactions using explicit ``BEGIN``/``COMMIT``/``ROLLBACK`` and savepoint statements, which behave identically in that mode. Connections created with ``autocommit=False`` are still rejected with a ``TransactionError``, because the driver holds an implicit transaction open at all times in that mode, breaking explicit transaction handling. See :ref:`python_api_transactions_modes`. .. _v4_0: diff --git a/docs/python-api.rst b/docs/python-api.rst index fed617e..1e72c1c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -95,7 +95,7 @@ Instead of a file path you can pass in an existing SQLite connection: db = Database(sqlite3.connect("my_database.db")) -The connection must use Python's default transaction handling. Connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options are rejected with a ``sqlite_utils.db.TransactionError`` - see :ref:`python_api_transactions_modes`. +The connection can use Python's default transaction handling or the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` mode. Connections created with ``autocommit=False`` are rejected with a ``sqlite_utils.db.TransactionError`` - see :ref:`python_api_transactions_modes`. If you want to create an in-memory database, you can do so like this: @@ -409,9 +409,9 @@ Two related safeguards to be aware of: Supported connection modes -------------------------- -``db.atomic()`` and the automatic per-method transactions require a connection in Python's default transaction handling mode. Passing a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options to ``Database()`` raises a ``sqlite_utils.db.TransactionError``. +``db.atomic()`` and the automatic per-method transactions work with connections in Python's default transaction handling mode and with connections created using the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` option. The library manages transactions itself using explicit ``BEGIN``, ``COMMIT``, ``ROLLBACK`` and savepoint statements, which behave identically in both modes. -This is because ``commit()`` and ``rollback()`` behave differently on those connections - under ``autocommit=True`` they are documented no-ops - which would cause every write made by this library to be silently discarded when the connection closed, rather than failing loudly. +Passing a connection created with ``autocommit=False`` to ``Database()`` raises a ``sqlite_utils.db.TransactionError``. In that mode the ``sqlite3`` driver holds an implicit transaction open at all times, which breaks the explicit transaction handling used by every write method - for example ``BEGIN`` fails because a transaction is always already open. .. _python_api_table: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 92b582a..da9a3e9 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -109,7 +109,7 @@ Two related behavior changes to ``table.transform()``: compound foreign keys now - Multi-step operations such as ``table.transform()`` no longer commit an existing transaction you have open - they use savepoints inside it instead. - ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, instead of silently committing it. - Using ``Database`` as a context manager (``with Database(path) as db:``) closes the connection on exit *without* committing - a transaction you explicitly opened with ``db.begin()`` and did not commit is rolled back. -- ``Database()`` rejects connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options, raising ``sqlite_utils.db.TransactionError``. On those connections every write the library made was silently discarded when the connection closed. +- ``Database()`` rejects connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=False)`` option, raising ``sqlite_utils.db.TransactionError``. In that mode the driver holds an implicit transaction open at all times, breaking the library's explicit transaction handling. Connections created with ``autocommit=True`` are supported. Packaging changes ----------------- diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 62e3656..25029cc 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -557,17 +557,16 @@ class Database: if recreate: raise ValueError("recreate cannot be used with connections, only paths") self.conn = cast(sqlite3.Connection, filename_or_conn) - # Python 3.12+ autocommit=True/False connections make commit() - # and rollback() behave differently, silently breaking the - # transaction handling used by every write method - autocommit = getattr(self.conn, "autocommit", None) - if autocommit is not None and autocommit != getattr( - sqlite3, "LEGACY_TRANSACTION_CONTROL", -1 - ): + # Python 3.12+ autocommit=False connections hold an implicit + # transaction open at all times, which breaks the explicit + # BEGIN/COMMIT transaction handling used by every write method. + # autocommit=True connections work fine - the library manages + # transactions itself with explicit SQL statements + if getattr(self.conn, "autocommit", None) is False: raise TransactionError( - "sqlite-utils requires a connection that uses the default " - "transaction handling - connections created with " - "autocommit=True or autocommit=False are not supported" + "sqlite-utils does not support connections created with " + "autocommit=False - use autocommit=True or the default " + "transaction handling instead" ) self._tracer: Optional[Tracer] = tracer if recursive_triggers: diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 7428dcf..470d089 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -65,16 +65,75 @@ def test_database_close(tmpdir, memory): sys.version_info < (3, 12), reason="sqlite3.connect(autocommit=) requires Python 3.12", ) -@pytest.mark.parametrize("autocommit", [True, False]) -def test_autocommit_connections_are_rejected(tmpdir, autocommit): - # These connection modes break commit()/rollback() in ways that - # silently lose data, so the constructor refuses them - conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=autocommit) +def test_autocommit_false_connections_are_rejected(tmpdir): + # autocommit=False keeps an implicit transaction open at all times, + # which breaks the explicit transaction handling used by every write + # method, so the constructor refuses these connections + conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=False) with pytest.raises(TransactionError): Database(conn) conn.close() +@pytest.mark.skipif( + sys.version_info < (3, 12), + reason="sqlite3.connect(autocommit=) requires Python 3.12", +) +def test_autocommit_true_connection_writes_persist(tmpdir): + path = str(tmpdir / "test.db") + conn = sqlite3.connect(path, autocommit=True) + db = Database(conn) + db["t"].insert({"id": 1}, pk="id") + db.execute("insert into t (id) values (2)") + db.close() + # The writes survived closing the connection + db2 = Database(path) + assert [r["id"] for r in db2["t"].rows] == [1, 2] + db2.close() + + +@pytest.mark.skipif( + sys.version_info < (3, 12), + reason="sqlite3.connect(autocommit=) requires Python 3.12", +) +def test_autocommit_true_connection_transactions(tmpdir): + conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=True) + db = Database(conn) + db["t"].insert({"id": 1}, pk="id") + # atomic() rolls back on error + with pytest.raises(ZeroDivisionError): + with db.atomic(): + db["t"].insert({"id": 2}, pk="id") + 1 / 0 + assert [r["id"] for r in db["t"].rows] == [1] + # atomic() commits on success, including a nested block + with db.atomic(): + db["t"].insert({"id": 3}, pk="id") + with db.atomic(): + db["t"].insert({"id": 4}, pk="id") + assert [r["id"] for r in db["t"].rows] == [1, 3, 4] + # begin()/rollback() work too + db.begin() + db["t"].insert({"id": 5}, pk="id") + db.rollback() + assert [r["id"] for r in db["t"].rows] == [1, 3, 4] + db.close() + + +@pytest.mark.skipif( + sys.version_info < (3, 12), + reason="sqlite3.connect(autocommit=) requires Python 3.12", +) +def test_autocommit_true_connection_wal(tmpdir): + conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=True) + db = Database(conn) + db.enable_wal() + assert db.journal_mode == "wal" + db.disable_wal() + assert db.journal_mode == "delete" + db.close() + + @pytest.mark.skipif( sys.version_info < (3, 12), reason="sqlite3.LEGACY_TRANSACTION_CONTROL requires Python 3.12",