diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab30b2d..4ecfcc7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,9 @@ jobs: - name: Install SpatiaLite if: matrix.os == 'ubuntu-latest' run: sudo apt-get install libsqlite3-mod-spatialite + - name: On macOS with Python 3.10 test with sqlean.py + if: matrix.os == 'macos-latest' && matrix.python-version == '3.10' + run: pip install sqlean.py sqlite-dump - name: Build extension for --load-extension test if: matrix.os == 'ubuntu-latest' run: |- diff --git a/docs/installation.rst b/docs/installation.rst index cb17ebf..beb6d4a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -39,6 +39,32 @@ Using pipx pipx install sqlite-utils +.. _installation_sqlite3_alternatives: + +Alternatives to sqlite3 +======================= + +By default, ``sqlite-utils`` uses the ``sqlite3`` package bundled with the Python standard library. + +Depending on your operating system, this may come with some limitations. + +On some platforms the ability to load additional extensions (via ``conn.load_extension(...)`` or ``--load-extension=/path/to/extension``) may be disabled. + +You may also see the error ``sqlite3.OperationalError: table sqlite_master may not be modified`` when trying to alter an existing table. + +You can work around these limitations by installing either the `pysqlite3 `__ package or the `sqlean.py `__ package, both of which provide drop-in replacements for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions. + +To install ``sqlean.py`` (which has compiled binary wheels available for all major platforms) run the following: + +.. code-block:: bash + + sqlite-utils install sqlean.py + +``pysqlite3`` and ``sqlean.py`` do not provide implementations of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package: + +.. code-block:: bash + + sqlite-utils install sqlite-dump .. _installation_completion: diff --git a/docs/python-api.rst b/docs/python-api.rst index 4b143c3..bb9ea5e 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1779,6 +1779,25 @@ The ``db.sqlite_version`` property returns a tuple of integers representing the >>> db.sqlite_version (3, 36, 0) +.. _python_api_itedump: + +Dumping the database to SQL +=========================== + +The ``db.iterdump()`` method returns a sequence of SQL strings representing a complete dump of the database. Use it like this: + +.. code-block:: python + + full_sql = "".join(db.iterdump()) + +This uses the `sqlite3.Connection.iterdump() `__ method. + +If you are using ``pysqlite3`` or ``sqlean.py`` the underlying method may be missing. If you install the `sqlite-dump `__ package then the ``db.iterdump()`` method will use that implementation instead: + +.. code-block:: bash + + pip install sqlite-dump + .. _python_api_introspection: Introspecting tables and views diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..768d182 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,4 @@ +[mypy] + +[mypy-pysqlite3,sqlean,sqlite_dump] +ignore_missing_imports = True \ No newline at end of file diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index ab9c524..bbcc888 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -395,7 +395,7 @@ def dump(path, load_extension): """ db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - for line in db.conn.iterdump(): + for line in db.iterdump(): click.echo(line) @@ -1893,7 +1893,7 @@ def memory( return if dump: - for line in db.conn.iterdump(): + for line in db.iterdump(): click.echo(line) return @@ -1903,7 +1903,7 @@ def memory( if save: db2 = sqlite_utils.Database(save) - for line in db.conn.iterdump(): + for line in db.iterdump(): db2.execute(line) return diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 997c2a5..0d2135e 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -38,6 +38,12 @@ from typing import ( ) import uuid +try: + from sqlite_dump import iterdump +except ImportError: + iterdump = None + + SQLITE_MAX_VARS = 999 _quote_fts_re = re.compile(r'\s+|(".*?")') @@ -340,6 +346,25 @@ class Database: "Close the SQLite connection, and the underlying database file" self.conn.close() + @contextlib.contextmanager + def ensure_autocommit_off(self): + """ + Ensure autocommit is off for this database connection. + + Example usage:: + + with db.ensure_autocommit_off(): + # do stuff here + + This will reset to the previous autocommit state at the end of the block. + """ + old_isolation_level = self.conn.isolation_level + try: + self.conn.isolation_level = None + yield + finally: + self.conn.isolation_level = old_isolation_level + @contextlib.contextmanager def tracer(self, tracer: Optional[Callable] = None): """ @@ -656,12 +681,14 @@ class Database: Sets ``journal_mode`` to ``'wal'`` to enable Write-Ahead Log mode. """ if self.journal_mode != "wal": - self.execute("PRAGMA journal_mode=wal;") + with self.ensure_autocommit_off(): + self.execute("PRAGMA journal_mode=wal;") def disable_wal(self): "Sets ``journal_mode`` back to ``'delete'`` to disable Write-Ahead Log mode." if self.journal_mode != "delete": - self.execute("PRAGMA journal_mode=delete;") + with self.ensure_autocommit_off(): + self.execute("PRAGMA journal_mode=delete;") def _ensure_counts_table(self): with self.conn: @@ -1149,6 +1176,18 @@ class Database: sql += " [{}]".format(name) self.execute(sql) + def iterdump(self) -> Generator[str, None, None]: + "A sequence of strings representing a SQL dump of the database" + if iterdump: + yield from iterdump(self.conn) + else: + try: + yield from self.conn.iterdump() + except AttributeError: + raise AttributeError( + "conn.iterdump() not found - try pip install sqlite-dump" + ) + def init_spatialite(self, path: Optional[str] = None) -> bool: """ The ``init_spatialite`` method will load and initialize the SpatiaLite extension. diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 06c1a4c..d302907 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -14,15 +14,22 @@ from typing import Dict, cast, BinaryIO, Iterable, Optional, Tuple, Type import click try: - import pysqlite3 as sqlite3 # type: ignore - import pysqlite3.dbapi2 # type: ignore + import pysqlite3 as sqlite3 # noqa: F401 + from pysqlite3 import dbapi2 # noqa: F401 - OperationalError = pysqlite3.dbapi2.OperationalError + OperationalError = dbapi2.OperationalError except ImportError: - # https://github.com/python/mypy/issues/1153#issuecomment-253842414 - import sqlite3 # type: ignore + try: + import sqlean as sqlite3 # noqa: F401 + from sqlean import dbapi2 # noqa: F401 + + OperationalError = dbapi2.OperationalError + except ImportError: + import sqlite3 # noqa: F401 + from sqlite3 import dbapi2 # noqa: F401 + + OperationalError = dbapi2.OperationalError - OperationalError = sqlite3.OperationalError SPATIALITE_PATHS = ( "/usr/lib/x86_64-linux-gnu/mod_spatialite.so", diff --git a/tests/test_analyze.py b/tests/test_analyze.py index a47c8be..a4cd8a2 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -14,7 +14,9 @@ def db(fresh_db): def test_analyze_whole_database(db): assert set(db.table_names()) == {"one_index", "two_indexes"} db.analyze() - assert set(db.table_names()) == {"one_index", "two_indexes", "sqlite_stat1"} + assert set(db.table_names()).issuperset( + {"one_index", "two_indexes", "sqlite_stat1"} + ) assert list(db["sqlite_stat1"].rows) == [ {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, @@ -24,13 +26,15 @@ def test_analyze_whole_database(db): @pytest.mark.parametrize("method", ("db_method_with_name", "table_method")) def test_analyze_one_table(db, method): - assert set(db.table_names()) == {"one_index", "two_indexes"} + assert set(db.table_names()).issuperset({"one_index", "two_indexes"}) if method == "db_method_with_name": db.analyze("one_index") elif method == "table_method": db["one_index"].analyze() - assert set(db.table_names()) == {"one_index", "two_indexes", "sqlite_stat1"} + assert set(db.table_names()).issuperset( + {"one_index", "two_indexes", "sqlite_stat1"} + ) assert list(db["sqlite_stat1"].rows) == [ {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"} ] @@ -39,7 +43,9 @@ def test_analyze_one_table(db, method): def test_analyze_index_by_name(db): assert set(db.table_names()) == {"one_index", "two_indexes"} db.analyze("idx_two_indexes_species") - assert set(db.table_names()) == {"one_index", "two_indexes", "sqlite_stat1"} + assert set(db.table_names()).issuperset( + {"one_index", "two_indexes", "sqlite_stat1"} + ) assert list(db["sqlite_stat1"].rows) == [ {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, ] diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index dc90325..9634cfc 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -135,7 +135,8 @@ def test_analyze_column(db_to_analyze, column, extra_kwargs, expected): def db_to_analyze_path(db_to_analyze, tmpdir): path = str(tmpdir / "test.db") db = sqlite3.connect(path) - db.executescript("\n".join(db_to_analyze.conn.iterdump())) + sql = "\n".join(db_to_analyze.iterdump()) + db.executescript(sql) return path diff --git a/tests/test_cli.py b/tests/test_cli.py index 5360e56..eb569f0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1429,7 +1429,7 @@ def test_enable_wal(): db = Database(dbname) db["t"].create({"pk": int}, pk="pk") assert db.journal_mode == "delete" - result = runner.invoke(cli.cli, ["enable-wal"] + dbs) + result = runner.invoke(cli.cli, ["enable-wal"] + dbs, catch_exceptions=False) assert 0 == result.exit_code for dbname in dbs: db = Database(dbname) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index 41bb00a..f69de22 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -164,9 +164,9 @@ def test_memory_dump(extra_args): input="id,name\n1,Cleo\n2,Bants", ) assert result.exit_code == 0 - assert result.output.strip() == ( + expected = ( "BEGIN TRANSACTION;\n" - 'CREATE TABLE "stdin" (\n' + 'CREATE TABLE IF NOT EXISTS "stdin" (\n' " [id] INTEGER,\n" " [name] TEXT\n" ");\n" @@ -176,6 +176,9 @@ def test_memory_dump(extra_args): "CREATE VIEW t AS select * from [stdin];\n" "COMMIT;" ) + # Using sqlite-dump it won't have IF NOT EXISTS + expected_alternative = expected.replace("IF NOT EXISTS ", "") + assert result.output.strip() in (expected, expected_alternative) @pytest.mark.parametrize("extra_args", ([], ["select 1"])) diff --git a/tests/test_gis.py b/tests/test_gis.py index 3b1fbf1..ee2e2a3 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -6,6 +6,12 @@ from sqlite_utils.cli import cli from sqlite_utils.db import Database from sqlite_utils.utils import find_spatialite, sqlite3 +try: + import sqlean +except ImportError: + sqlean = None + + pytestmark = [ pytest.mark.skipif( not find_spatialite(), reason="Could not find SpatiaLite extension" @@ -14,6 +20,9 @@ pytestmark = [ not hasattr(sqlite3.Connection, "enable_load_extension"), reason="sqlite3.Connection missing enable_load_extension", ), + pytest.mark.skipif( + sqlean is not None, reason="sqlean.py is not compatible with SpatiaLite" + ), ]