From 89b37248717e2e8c5d45d5be0bc2058694372b16 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 11:45:36 +0000 Subject: [PATCH 01/33] Add test: transform does not cascade-delete referencing records Covers running transform against a table involved in an ON DELETE CASCADE foreign key relationship while PRAGMA foreign_keys is on. Dropping the old table during transform must not trigger the cascade and delete records from the referencing table. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01StxDiBxEbsv8VBmqxamgUV --- tests/test_transform.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/test_transform.py b/tests/test_transform.py index f0f5019..284c4fb 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -432,6 +432,45 @@ def test_transform_verify_foreign_keys(fresh_db): assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] +@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) +def test_transform_on_delete_cascade_does_not_delete_records( + fresh_db, use_pragma_foreign_keys +): + # Transforming a table drops and recreates it - if another table references + # it with ON DELETE CASCADE and PRAGMA foreign_keys is on, that drop must + # not cascade and delete the referencing records + if use_pragma_foreign_keys: + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript( + """ + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """ + ) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + # Transform the table on the other end of the cascading foreign key + fresh_db["authors"].transform(rename={"name": "author_name"}) + assert list(fresh_db["authors"].rows) == [ + {"id": 1, "author_name": "Ursula K. Le Guin"} + ] + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + # Transforming the table with the cascading foreign key should not + # delete its records either + fresh_db["books"].transform(rename={"title": "book_title"}) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "book_title": "The Dispossessed", "author_id": 1} + ] + if use_pragma_foreign_keys: + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + def test_transform_add_foreign_keys_from_scratch(fresh_db): _add_country_city_continent(fresh_db) fresh_db["places"].insert(_CAVEAU) From 8ca780c30d1fe359160c989bd6ff0e7d9d963dc6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 11:55:08 +0000 Subject: [PATCH 02/33] Apply Black Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01StxDiBxEbsv8VBmqxamgUV --- tests/test_transform.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_transform.py b/tests/test_transform.py index 284c4fb..3cc5ba6 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -441,16 +441,14 @@ def test_transform_on_delete_cascade_does_not_delete_records( # not cascade and delete the referencing records if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript( - """ + fresh_db.executescript(""" CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); CREATE TABLE books ( id INTEGER PRIMARY KEY, title TEXT, author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE ); - """ - ) + """) fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) # Transform the table on the other end of the cascading foreign key From d71420065903ff54247b5062b8c6af6165b7e638 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Jul 2026 05:00:32 -0700 Subject: [PATCH 03/33] Add test: transform does not cascade-delete referencing records (#792) > Add a test that covers what happens if you run transform against a table with ON CASCADE DELETE for one of its foreign keys - those records should not be deleted during the transform even though the table is dropped as part of that procedure --- tests/test_transform.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_transform.py b/tests/test_transform.py index f0f5019..3cc5ba6 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -432,6 +432,43 @@ def test_transform_verify_foreign_keys(fresh_db): assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] +@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) +def test_transform_on_delete_cascade_does_not_delete_records( + fresh_db, use_pragma_foreign_keys +): + # Transforming a table drops and recreates it - if another table references + # it with ON DELETE CASCADE and PRAGMA foreign_keys is on, that drop must + # not cascade and delete the referencing records + if use_pragma_foreign_keys: + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + # Transform the table on the other end of the cascading foreign key + fresh_db["authors"].transform(rename={"name": "author_name"}) + assert list(fresh_db["authors"].rows) == [ + {"id": 1, "author_name": "Ursula K. Le Guin"} + ] + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + # Transforming the table with the cascading foreign key should not + # delete its records either + fresh_db["books"].transform(rename={"title": "book_title"}) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "book_title": "The Dispossessed", "author_id": 1} + ] + if use_pragma_foreign_keys: + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + def test_transform_add_foreign_keys_from_scratch(fresh_db): _add_country_city_continent(fresh_db) fresh_db["places"].insert(_CAVEAU) From f66ddcb215e76dcbc1fa1dff4359f2ac3dc702e5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Jul 2026 08:43:51 -0700 Subject: [PATCH 04/33] Transform now refuses to run inside a transaction if destructive foreign keys exist (#795) * Transform now refuses to run inside a transaction if destructive foreign keys exist Closes #794 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014StVTWQJpFhfZJK2CYVBwv --- docs/python-api.rst | 33 ++++++++++- sqlite_utils/db.py | 35 ++++++++++++ tests/test_transform.py | 124 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 190 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 1ed238e..43b734d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -434,9 +434,10 @@ The library will never commit a transaction you opened. If you call write method Prefer ``db.atomic()`` or ``db.begin()``, ``db.commit()`` and ``db.rollback()`` over mixing sqlite-utils transaction methods with calls to ``db.conn.commit()``, ``db.conn.rollback()`` or raw transaction-control SQL. Mixing the two layers makes it much harder to tell which layer owns the current transaction. -Two related safeguards to be aware of: +Some related safeguards to be aware of: - ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, because changing the journal mode would commit it as a side effect. +- ``table.transform()`` raises a ``sqlite_utils.db.TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions, because the pragma cannot be turned off mid-transaction to protect those referencing rows - see :ref:`python_api_transform_foreign_keys_transactions`. - Closing the database - explicitly with ``db.close()``, or by exiting a ``with Database(...) as db:`` block - rolls back any transaction that is still open, see :ref:`python_api_close`. .. _python_api_transactions_modes: @@ -1996,6 +1997,36 @@ If you want to do something more advanced, you can call the ``table.transform_sq This method will return a list of SQL statements that should be executed to implement the change. You can then make modifications to that SQL - or add additional SQL statements - before executing it yourself. +.. _python_api_transform_foreign_keys_transactions: + +Foreign keys and transactions +----------------------------- + +Because ``.transform()`` drops the old table, running it with ``PRAGMA foreign_keys`` enabled could fire ``ON DELETE`` actions on any tables that reference it - an inbound ``ON DELETE CASCADE`` foreign key would silently delete those referencing rows. To prevent this, ``.transform()`` turns ``PRAGMA foreign_keys`` off for the duration of the operation and restores it afterwards, running ``PRAGMA foreign_key_check`` before committing. + +``PRAGMA foreign_keys`` cannot be changed inside a transaction, so this protection is impossible if you call ``.transform()`` while a transaction is already open - for example inside a ``with db.atomic():`` block or after ``db.begin()``. If ``PRAGMA foreign_keys`` is on and another table references the table being transformed with a destructive ``ON DELETE`` action - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT`` - the method will refuse to run and raise a ``sqlite_utils.db.TransactionError``: + +.. code-block:: python + + from sqlite_utils.db import TransactionError + + try: + with db.atomic(): + db["authors"].transform(types={"id": str}) + except TransactionError as ex: + print("Could not transform in transaction:", ex) + +To transform such a table either call ``.transform()`` outside of the transaction, or execute ``PRAGMA foreign_keys = off`` before opening it: + +.. code-block:: python + + db.execute("PRAGMA foreign_keys = off") + with db.atomic(): + db["authors"].transform(types={"id": str}) + db.execute("PRAGMA foreign_keys = on") + +Tables referenced by foreign keys without a destructive action (the default ``NO ACTION``, or ``RESTRICT``) can still be transformed inside a transaction - sqlite-utils uses ``PRAGMA defer_foreign_keys`` to postpone the foreign key checks until the transaction commits. + .. _python_api_extract: Extracting columns into a separate table diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d709fb9..e97b7d9 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2522,6 +2522,11 @@ class Table(Queryable): See :ref:`python_api_transform` for full details. + Raises :py:class:`sqlite_utils.db.TransactionError` if called while a + transaction is open with ``PRAGMA foreign_keys`` enabled and the table + is referenced by foreign keys with destructive ``ON DELETE`` actions - + see :ref:`python_api_transform_foreign_keys_transactions`. + :param types: Columns that should have their type changed, for example ``{"weight": float}`` :param rename: Columns to rename, for example ``{"headline": "title"}`` :param drop: Columns to drop @@ -2566,6 +2571,36 @@ class Table(Queryable): should_defer_foreign_keys = ( pragma_foreign_keys_was_on and already_in_transaction ) + if should_defer_foreign_keys: + # PRAGMA foreign_keys is a no-op inside a transaction, and + # defer_foreign_keys only defers violation checks, not ON DELETE + # actions - so dropping the old table would still fire destructive + # actions on any tables that reference it. Refuse rather than + # silently modify or delete those rows. + destructive_fks = [ + (table.name, fk) + for table in self.db.tables + for fk in table.foreign_keys + if fk.other_table == self.name + and fk.on_delete in ("CASCADE", "SET NULL", "SET DEFAULT") + ] + if destructive_fks: + raise TransactionError( + "Cannot transform table {table} while a transaction is open: " + "PRAGMA foreign_keys cannot be changed inside a transaction, " + "and the table is referenced by foreign keys with ON DELETE " + "actions that would fire when the old table is dropped: " + "{fks}. Call transform() outside of the transaction, or " + 'execute "PRAGMA foreign_keys = off" before opening it.'.format( + table=self.name, + fks=", ".join( + "{}.{} (ON DELETE {})".format( + table_name, ", ".join(fk.columns), fk.on_delete + ) + for table_name, fk in destructive_fks + ), + ) + ) defer_foreign_keys_was_on = False try: if should_disable_foreign_keys: diff --git a/tests/test_transform.py b/tests/test_transform.py index 3cc5ba6..362f1ca 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,6 +1,6 @@ import sqlite3 -from sqlite_utils.db import ForeignKey, TransformError +from sqlite_utils.db import ForeignKey, TransactionError, TransformError from sqlite_utils.utils import OperationalError import pytest @@ -469,6 +469,128 @@ def test_transform_on_delete_cascade_does_not_delete_records( assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] +@pytest.mark.parametrize("on_delete", ["CASCADE", "SET NULL", "SET DEFAULT", "cascade"]) +def test_transform_in_transaction_refuses_destructive_on_delete(fresh_db, on_delete): + # PRAGMA foreign_keys is a no-op inside a transaction, so transforming a + # table referenced by ON DELETE CASCADE / SET NULL / SET DEFAULT foreign + # keys inside an open transaction would fire those actions when the old + # table is dropped - transform() should refuse instead + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE {} + ); + """.format(on_delete)) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + previous_schema = fresh_db["authors"].schema + with fresh_db.atomic(): + with pytest.raises(TransactionError) as excinfo: + fresh_db["authors"].transform(rename={"name": "author_name"}) + message = str(excinfo.value) + assert "books" in message + assert "ON DELETE {}".format(on_delete.upper()) in message + # Nothing should have changed + assert fresh_db["authors"].schema == previous_schema + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + +def test_transform_in_transaction_refuses_self_referential_cascade(fresh_db): + # The copied table carries a foreign key referencing the original table + # name, so a self-referential cascade would wipe the copy too + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE categories ( + id INTEGER PRIMARY KEY, + name TEXT, + parent_id INTEGER REFERENCES categories(id) ON DELETE CASCADE + ); + """) + fresh_db["categories"].insert_all( + [ + {"id": 1, "name": "Fiction", "parent_id": None}, + {"id": 2, "name": "Science Fiction", "parent_id": 1}, + ] + ) + with fresh_db.atomic(): + with pytest.raises(TransactionError) as excinfo: + fresh_db["categories"].transform(rename={"name": "title"}) + assert "categories" in str(excinfo.value) + assert fresh_db["categories"].count == 2 + + +def test_transform_in_transaction_allowed_with_no_action_foreign_key(fresh_db): + # An inbound foreign key without a destructive ON DELETE action is safe + # inside a transaction thanks to PRAGMA defer_foreign_keys + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + with fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "author_name"}) + assert list(fresh_db["authors"].rows) == [ + {"id": 1, "author_name": "Ursula K. Le Guin"} + ] + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + +def test_transform_in_transaction_allowed_for_child_table(fresh_db): + # The table being transformed only has an outbound foreign key - dropping + # it fires no ON DELETE actions, so this is allowed inside a transaction + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + with fresh_db.atomic(): + fresh_db["books"].transform(rename={"title": "book_title"}) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "book_title": "The Dispossessed", "author_id": 1} + ] + + +def test_transform_in_transaction_allowed_with_foreign_keys_off(fresh_db): + # With PRAGMA foreign_keys off (the default) no cascades can fire, so + # transform inside a transaction is safe even with a CASCADE schema + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + with fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "author_name"}) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + + def test_transform_add_foreign_keys_from_scratch(fresh_db): _add_country_city_continent(fresh_db) fresh_db["places"].insert(_CAVEAU) From 458b3ab5b169eff1f8319c44a7c320c68f54d28b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Jul 2026 13:52:14 -0700 Subject: [PATCH 05/33] Release 4.1.1 Refs #791, #792, #794, #795 --- docs/changelog.rst | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5b9355f..a8006c3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _v4_1_1: + +4.1.1 (2026-07-12) +------------------ + +- ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`) +- The CLI and Python API documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) .. _v4_1: 4.1 (2026-07-11) diff --git a/pyproject.toml b/pyproject.toml index 971f5a0..003322c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.1" +version = "4.1.1" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From a947dc673923ff6e95b41d3dfabe1cbd95e6de86 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Jul 2026 17:14:01 -0700 Subject: [PATCH 06/33] Changelog now links to CLI and Python API in most recent entry --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a8006c3..4c868f4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ ------------------ - ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`) -- The CLI and Python API documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) +- The :ref:`CLI ` and :ref:`Python API ` documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) .. _v4_1: 4.1 (2026-07-11) From 69a1c0d960abb20ac03a085142bd59f7fbe002f7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 14:53:12 -0700 Subject: [PATCH 07/33] Fixes for Ruff>=0.16.0 (#814) * Automated upgrades by Ruff uvx --with 'ruff>=0.16.0' ruff check . --fix --unsafe-fixes * Fix remaining Ruff errors with GPT-5.6 Sol high https://gist.github.com/simonw/6da7906a9fea6e90da131c21a9055199 * Fix flake E501 long lines * New Protocol for migrations to make ty happy --- .gitignore | 1 + docs/cli-reference.rst | 4 +- docs/conf.py | 11 +- pyproject.toml | 9 +- sqlite_utils/__init__.py | 7 +- sqlite_utils/cli.py | 199 +++--- sqlite_utils/db.py | 988 +++++++++++++---------------- sqlite_utils/hookspecs.py | 3 +- sqlite_utils/migrations.py | 27 +- sqlite_utils/plugins.py | 10 +- sqlite_utils/recipes.py | 12 +- sqlite_utils/utils.py | 111 ++-- tests/conftest.py | 5 +- tests/test_analyze_tables.py | 10 +- tests/test_atomic.py | 81 ++- tests/test_cli.py | 88 +-- tests/test_cli_bulk.py | 8 +- tests/test_cli_convert.py | 16 +- tests/test_cli_insert.py | 17 +- tests/test_cli_memory.py | 13 +- tests/test_cli_migrate.py | 3 +- tests/test_column_affinity.py | 3 +- tests/test_constructor.py | 6 +- tests/test_convert.py | 3 +- tests/test_create.py | 68 +- tests/test_create_view.py | 1 + tests/test_default_value.py | 2 +- tests/test_delete.py | 2 +- tests/test_docs.py | 12 +- tests/test_duplicate.py | 6 +- tests/test_enable_counts.py | 10 +- tests/test_extract.py | 15 +- tests/test_extracts.py | 19 +- tests/test_foreign_keys.py | 3 +- tests/test_fts.py | 26 +- tests/test_get.py | 1 + tests/test_gis.py | 5 +- tests/test_hypothesis.py | 3 +- tests/test_insert_files.py | 14 +- tests/test_introspect.py | 7 +- tests/test_list_mode.py | 1 + tests/test_lookup.py | 3 +- tests/test_m2m.py | 6 +- tests/test_migrations.py | 8 +- tests/test_plugins.py | 13 +- tests/test_query.py | 3 +- tests/test_recipes.py | 6 +- tests/test_recreate.py | 6 +- tests/test_rows_from_file.py | 6 +- tests/test_sniff.py | 6 +- tests/test_suggest_column_types.py | 4 +- tests/test_tracer.py | 54 +- tests/test_transform.py | 33 +- tests/test_update.py | 2 +- tests/test_upsert.py | 5 +- tests/test_utils.py | 6 +- tests/test_wal.py | 32 +- 57 files changed, 974 insertions(+), 1049 deletions(-) diff --git a/.gitignore b/.gitignore index 6743708..5b5d2c6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ venv .schema .vscode .hypothesis +.claude/ Pipfile Pipfile.lock uv.lock diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 9fafe28..a4ec402 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -662,7 +662,7 @@ See :ref:`cli_convert`. Convert a string like a,b,c into a JSON array ["a", "b", "c"] r.parsedate(value: 'str', dayfirst: 'bool' = False, yearfirst: 'bool' = False, - errors: 'Optional[object]' = None) -> 'Optional[str]' + errors: 'object | None' = None) -> 'str | None' Parse a date and convert it to ISO date format: yyyy-mm-dd - dayfirst=True: treat xx as the day in xx/yy/zz @@ -671,7 +671,7 @@ See :ref:`cli_convert`. - errors=r.SET_NULL to set values that cannot be parsed to null r.parsedatetime(value: 'str', dayfirst: 'bool' = False, yearfirst: 'bool' = - False, errors: 'Optional[object]' = None) -> 'Optional[str]' + False, errors: 'object | None' = None) -> 'str | None' Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS - dayfirst=True: treat xx as the day in xx/yy/zz diff --git a/docs/conf.py b/docs/conf.py index 4f29b39..62d4642 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,10 +1,7 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - import inspect -from pathlib import Path -from subprocess import Popen, PIPE, check_output import sys +from pathlib import Path +from subprocess import PIPE, CalledProcessError, Popen, check_output # This file is execfile()d with the current directory set to its # containing dir. @@ -50,7 +47,7 @@ extlinks = { def _linkcode_git_ref(): try: return check_output(["git", "rev-parse", "HEAD"]).decode("utf8").strip() - except Exception: + except (CalledProcessError, OSError): return "main" @@ -79,7 +76,7 @@ def linkcode_resolve(domain, info): obj = inspect.unwrap(obj) source_file = inspect.getsourcefile(obj) _, line_number = inspect.getsourcelines(obj) - except Exception: + except (OSError, TypeError, ValueError): return None if source_file is None: diff --git a/pyproject.toml b/pyproject.toml index 003322c..6bc0a64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,14 @@ build-backend = "setuptools.build_meta" max-line-length = 160 # Black compatibility, E203 whitespace before ':': extend-ignore = ["E203"] -extend-exclude = [".venv", "build", "dist", "docs", "sqlite_utils.egg-info"] +extend-exclude = [ + ".venv", + ".claude", + "build", + "dist", + "docs", + "sqlite_utils.egg-info", +] [tool.setuptools.package-data] sqlite_utils = ["py.typed"] diff --git a/sqlite_utils/__init__.py b/sqlite_utils/__init__.py index 58ee7ab..0d25716 100644 --- a/sqlite_utils/__init__.py +++ b/sqlite_utils/__init__.py @@ -1,7 +1,6 @@ -from .utils import suggest_column_types -from .hookspecs import hookimpl -from .hookspecs import hookspec from .db import Database +from .hookspecs import hookimpl, hookspec from .migrations import Migrations +from .utils import suggest_column_types -__all__ = ["Database", "Migrations", "suggest_column_types", "hookimpl", "hookspec"] +__all__ = ["Database", "Migrations", "hookimpl", "hookspec", "suggest_column_types"] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index e0b8969..dab4b67 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,17 +1,30 @@ import base64 +import csv as csv_std import difflib -from typing import Any -import click -from click_default_group import DefaultGroup -from datetime import datetime, timezone import hashlib +import inspect +import io +import itertools +import json +import os import pathlib +import pdb # noqa: T100 +import sys +import textwrap +from datetime import datetime, timezone from runpy import run_module +from typing import Any + +import click +import tabulate +from click_default_group import DefaultGroup + import sqlite_utils +from sqlite_utils import recipes from sqlite_utils.db import ( + DEFAULT, AlterError, BadMultiValues, - DEFAULT, DescIndex, InvalidColumns, NoTable, @@ -19,36 +32,28 @@ from sqlite_utils.db import ( PrimaryKeyRequired, quote_identifier, ) -from sqlite_utils.plugins import ensure_plugins_loaded, pm, get_plugins +from sqlite_utils.plugins import ensure_plugins_loaded, get_plugins, pm from sqlite_utils.utils import maximize_csv_field_size_limit -from sqlite_utils import recipes -import textwrap -import inspect -import io -import itertools -import json -import os -import pdb -import sys -import csv as csv_std -import tabulate + from .utils import ( + Format, OperationalError, + TypeTracker, _compile_code, chunks, + decode_base64_values, dedupe_keys, file_progress, find_spatialite, - flatten as _flatten, - sqlite3, - decode_base64_values, progressbar, rows_from_file, - Format, - TypeTracker, + sqlite3, +) +from .utils import ( + flatten as _flatten, ) -CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) +CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} def _register_db_for_cleanup(db): @@ -67,7 +72,7 @@ def _close_databases(ctx): for db in ctx.meta.get("_databases_to_close", []): try: db.close() - except Exception: + except sqlite3.Error: pass @@ -174,7 +179,6 @@ def functions_option(fn): @click.version_option() def cli(): "Commands for interacting with a SQLite database" - pass @cli.command() @@ -891,7 +895,7 @@ def enable_counts(path, tables, load_extension): # Check all tables exist bad_tables = [table for table in tables if not db[table].exists()] if bad_tables: - raise click.ClickException("Invalid tables: {}".format(bad_tables)) + raise click.ClickException(f"Invalid tables: {bad_tables}") for table in tables: db.table(table).enable_counts() @@ -1140,9 +1144,7 @@ def insert_upsert_implementation( ) ): raise click.ClickException( - "{}\n\nTry using --alter to add additional columns".format( - e.args[0] - ) + f"{e.args[0]}\n\nTry using --alter to add additional columns" ) # If we can find sql= and parameters= arguments, show those variables = _find_variables(e.__traceback__, ["sql", "parameters"]) @@ -1240,7 +1242,7 @@ def insert_upsert_implementation( reader = csv_std.reader(decoded, **csv_reader_args) # type: ignore first_row = next(reader) if no_headers: - headers = ["untitled_{}".format(i + 1) for i in range(len(first_row))] + headers = [f"untitled_{i + 1}" for i in range(len(first_row))] reader = itertools.chain([first_row], reader) else: headers = first_row @@ -1269,9 +1271,7 @@ def insert_upsert_implementation( docs = [docs] except json.decoder.JSONDecodeError as ex: raise click.ClickException( - "Invalid JSON - use --csv for CSV or --tsv for TSV files\n\nJSON error: {}".format( - ex - ) + f"Invalid JSON - use --csv for CSV or --tsv for TSV files\n\nJSON error: {ex}" ) if flatten: docs = (_flatten(doc) for doc in docs) @@ -1290,7 +1290,7 @@ def insert_upsert_implementation( docs = (fn(doc["line"]) for doc in docs) elif text: # Special case: this is allowed to be an iterable - text_value = list(docs)[0]["text"] + text_value = next(iter(docs))["text"] fn_return = fn(text_value) if isinstance(fn_return, dict): docs = [fn_return] @@ -1774,17 +1774,14 @@ def create_table( ctype = columns.pop(0) if ctype.upper() not in VALID_COLUMN_TYPES: raise click.ClickException( - "column types must be one of {}".format(VALID_COLUMN_TYPES) + f"column types must be one of {VALID_COLUMN_TYPES}" ) coltypes[name] = ctype.upper() # Does table already exist? - if table in db.table_names(): - if not ignore and not replace and not transform: - raise click.ClickException( - 'Table "{}" already exists. Use --replace to delete and replace it.'.format( - table - ) - ) + if table in db.table_names() and not ignore and not replace and not transform: + raise click.ClickException( + f'Table "{table}" already exists. Use --replace to delete and replace it.' + ) db.table(table).create( coltypes, pk=pks[0] if len(pks) == 1 else pks, @@ -1819,7 +1816,7 @@ def duplicate(path, table, new_table, ignore, load_extension): db.table(table).duplicate(new_table) except NoTable: if not ignore: - raise click.ClickException('Table "{}" does not exist'.format(table)) + raise click.ClickException(f'Table "{table}" does not exist') @cli.command(name="rename-table") @@ -1843,9 +1840,7 @@ def rename_table(path, table, new_name, ignore, load_extension): db.rename_table(table, new_name) except sqlite3.OperationalError as ex: if not ignore: - raise click.ClickException( - 'Table "{}" could not be renamed. {}'.format(table, str(ex)) - ) + raise click.ClickException(f'Table "{table}" could not be renamed. {ex!s}') @cli.command(name="drop-table") @@ -1874,10 +1869,10 @@ def drop_table(path, table, ignore, load_extension): # A view exists with this name if not ignore: raise click.ClickException( - '"{}" is a view, not a table - use drop-view to drop it'.format(table) + f'"{table}" is a view, not a table - use drop-view to drop it' ) except OperationalError: - raise click.ClickException('Table "{}" does not exist'.format(table)) + raise click.ClickException(f'Table "{table}" does not exist') @cli.command(name="create-view") @@ -1919,9 +1914,7 @@ def create_view(path, view, select, ignore, replace, load_extension): db.view(view).drop() else: raise click.ClickException( - 'View "{}" already exists. Use --replace to delete and replace it.'.format( - view - ) + f'View "{view}" already exists. Use --replace to delete and replace it.' ) db.create_view(view, select) @@ -1953,9 +1946,9 @@ def drop_view(path, view, ignore, load_extension): return if view in db.table_names(): raise click.ClickException( - '"{}" is a table, not a view - use drop-table to drop it'.format(view) + f'"{view}" is a table, not a view - use drop-table to drop it' ) - raise click.ClickException('View "{}" does not exist'.format(view)) + raise click.ClickException(f'View "{view}" does not exist') @cli.command() @@ -2177,7 +2170,7 @@ def memory( file_path = pathlib.Path(path) stem = file_path.stem if stem_counts.get(stem): - file_table = "{}_{}".format(stem, stem_counts[stem]) + file_table = f"{stem}_{stem_counts[stem]}" else: file_table = stem stem_counts[stem] = stem_counts.get(stem, 1) + 1 @@ -2196,14 +2189,14 @@ def memory( if tracker is not None and db.table(file_table).exists(): db.table(file_table).transform(types=tracker.types) # Add convenient t / t1 / t2 views - view_names = ["t{}".format(i + 1)] + view_names = [f"t{i + 1}"] if i == 0: view_names.append("t") for view_name in view_names: if not db[view_name].exists(): db.create_view( view_name, - "select * from {}".format(quote_identifier(file_table)), + f"select * from {quote_identifier(file_table)}", ) finally: if should_close_fp and fp: @@ -2373,19 +2366,17 @@ def search( # Check table exists table_obj = db.table(dbtable) if not table_obj.exists(): - raise click.ClickException("Table '{}' does not exist".format(dbtable)) + raise click.ClickException(f"Table '{dbtable}' does not exist") if not table_obj.detect_fts(): raise click.ClickException( - "Table '{}' is not configured for full-text search".format(dbtable) + f"Table '{dbtable}' is not configured for full-text search" ) if column: # Check they all exist table_columns = table_obj.columns_dict for c in column: if c not in table_columns: - raise click.ClickException( - "Table '{}' has no column '{}".format(dbtable, c) - ) + raise click.ClickException(f"Table '{dbtable}' has no column '{c}") sql = table_obj.search_sql(columns=column, order_by=order, limit=limit) if show_sql: click.echo(sql) @@ -2412,7 +2403,7 @@ def search( except click.ClickException as e: if "malformed MATCH expression" in str(e) or "unterminated string" in str(e): raise click.ClickException( - "{}\n\nTry running this again with the --quote option".format(str(e)) + f"{e!s}\n\nTry running this again with the --quote option" ) else: raise @@ -2479,15 +2470,15 @@ def rows( columns = "*" if column: columns = ", ".join(quote_identifier(c) for c in column) - sql = "select {} from {}".format(columns, quote_identifier(dbtable)) + sql = f"select {columns} from {quote_identifier(dbtable)}" if where: sql += " where " + where if order: sql += " order by " + order if limit: - sql += " limit {}".format(limit) + sql += f" limit {limit}" if offset: - sql += " offset {}".format(offset) + sql += f" offset {offset}" ctx.invoke( query, path=path, @@ -2760,7 +2751,7 @@ def transform( for column, ctype in type: if ctype.upper() not in VALID_COLUMN_TYPES: raise click.ClickException( - "column types must be one of {}".format(VALID_COLUMN_TYPES) + f"column types must be one of {VALID_COLUMN_TYPES}" ) types[column] = ctype.upper() @@ -2858,12 +2849,12 @@ def extract( db = sqlite_utils.Database(path) _register_db_for_cleanup(db) _load_extensions(db, load_extension) - kwargs: dict[str, Any] = dict( - columns=columns, - table=other_table, - fk_column=fk_column, - rename=dict(rename), - ) + kwargs: dict[str, Any] = { + "columns": columns, + "table": other_table, + "fk_column": fk_column, + "rename": dict(rename), + } try: db.table(table).extract(**kwargs) except (NoTable, InvalidColumns) as e: @@ -2958,7 +2949,7 @@ def insert_files( with progressbar(paths_and_relative_paths, silent=silent) as bar: def to_insert(): - for path, relative_path in bar: + for file_path, relative_path in bar: row = {} # content_text is special case as it considers 'encoding' @@ -2970,19 +2961,21 @@ def insert_files( raise UnicodeDecodeErrorForPath(e, resolved) lookups = dict(FILE_COLUMNS, content_text=_content_text) - if path == "-": + if file_path == "-": stdin_data = sys.stdin.buffer.read() # We only support a subset of columns for this case lookups = { "name": lambda p: name or "-", "path": lambda p: name or "-", - "content": lambda p: stdin_data, - "content_text": lambda p: stdin_data.decode( + "content": lambda p, data=stdin_data: data, + "content_text": lambda p, data=stdin_data: data.decode( encoding or "utf-8" ), - "sha256": lambda p: hashlib.sha256(stdin_data).hexdigest(), - "md5": lambda p: hashlib.md5(stdin_data).hexdigest(), - "size": lambda p: len(stdin_data), + "sha256": lambda p, data=stdin_data: hashlib.sha256( + data + ).hexdigest(), + "md5": lambda p, data=stdin_data: hashlib.md5(data).hexdigest(), + "size": lambda p, data=stdin_data: len(data), } for coldef in column: if ":" in coldef: @@ -2990,7 +2983,7 @@ def insert_files( else: colname, coltype = coldef, coldef try: - value = lookups[coltype](path) + value = lookups[coltype](file_path) row[colname] = value except KeyError: raise click.ClickException( @@ -3018,7 +3011,7 @@ def insert_files( except UnicodeDecodeErrorForPath as e: raise click.ClickException( UNICODE_ERROR.format( - "Could not read file '{}' as text\n\n{}".format(e.path, e.exception) + f"Could not read file '{e.path}' as text\n\n{e.exception}" ) ) @@ -3196,7 +3189,7 @@ def _generate_convert_help(): for name in recipe_names: fn = getattr(recipes, name) doc = textwrap.dedent(fn.__doc__.rstrip()).replace("\b\n", "") - help += "\n\nr.{}{}\n\n\b{}".format(name, str(inspect.signature(fn)), doc) + help += f"\n\nr.{name}{inspect.signature(fn)!s}\n\n\b{doc}" help += "\n\n" help += textwrap.dedent(""" You can use these recipes like so: @@ -3299,7 +3292,7 @@ def convert( """.format( column=columns[0], table=table, - where=" where {}".format(where) if where is not None else "", + where=f" where {where}" if where is not None else "", ) for row in db.conn.execute(sql, where_args).fetchall(): click.echo(str(row[0])) @@ -3319,7 +3312,7 @@ def convert( def wrapped_fn(value): try: return fn_(value) - except Exception as ex: + except Exception as ex: # noqa: BLE001 print("\nException raised, dropping into pdb...:", ex) pdb.post_mortem(ex.__traceback__) sys.exit(1) @@ -3339,9 +3332,7 @@ def convert( ) except BadMultiValues as e: raise click.ClickException( - "When using --multi code must return a Python dictionary - returned: {}".format( - repr(e.values) - ) + f"When using --multi code must return a Python dictionary - returned: {e.values!r}" ) @@ -3459,7 +3450,7 @@ def create_spatial_index(db_path, table, column_name, load_extension): def _find_migration_files(migrations): if not migrations: - migrations = [pathlib.Path(".").resolve()] + migrations = [pathlib.Path.cwd()] files = set() for path_str in migrations: path = pathlib.Path(path_str) @@ -3484,7 +3475,7 @@ def _load_migration_sets(files): "__file__": str(filepath), "__name__": "__sqlite_utils_migration__", } - exec(code, namespace) + exec(code, namespace) # noqa: S102 migration_sets.extend( obj for obj in namespace.values() if _compatible_migration_set(obj) ) @@ -3493,17 +3484,17 @@ def _load_migration_sets(files): def _display_migration_list(db, migration_sets): for migration_set in migration_sets: - click.echo("Migrations for: {}".format(migration_set.name)) + click.echo(f"Migrations for: {migration_set.name}") click.echo() click.echo(" Applied:") for migration in migration_set.applied(db): - click.echo(" {} - {}".format(migration.name, migration.applied_at)) + click.echo(f" {migration.name} - {migration.applied_at}") click.echo() click.echo(" Pending:") output = False for migration in migration_set.pending(db): output = True - click.echo(" {}".format(migration.name)) + click.echo(f" {migration.name}") if not output: click.echo(" (none)") click.echo() @@ -3583,7 +3574,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose): prev_schema = db.schema if verbose: - click.echo("Migrating {}".format(db_path)) + click.echo(f"Migrating {db_path}") click.echo("\nSchema before:\n") click.echo(textwrap.indent(prev_schema, " ") or " (empty)") click.echo() @@ -3594,9 +3585,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose): names = {m.name for m in migration_set.pending(db)} names.update(m.name for m in migration_set.applied(db)) known_names.update(names) - known_names.update( - "{}:{}".format(migration_set.name, name) for name in names - ) + known_names.update(f"{migration_set.name}:{name}" for name in names) unknown = [value for value in stop_before if value not in known_names] if unknown: raise click.ClickException( @@ -3652,7 +3641,7 @@ def _render_common(title, values): return "" lines = [title] for value, count in values: - lines.append(" {}: {}".format(count, value)) + lines.append(f" {count}: {value}") return "\n".join(lines) @@ -3722,7 +3711,7 @@ def maybe_json(value): if not isinstance(value, str): return value stripped = value.strip() - if not (stripped.startswith("{") or stripped.startswith("[")): + if not (stripped.startswith(("{", "["))): return value try: return json.loads(stripped) @@ -3740,7 +3729,7 @@ def json_binary(value): def verify_is_dict(doc): if not isinstance(doc, dict): raise click.ClickException( - "Rows must all be dictionaries, got: {}".format(repr(doc)[:1000]) + f"Rows must all be dictionaries, got: {repr(doc)[:1000]}" ) return doc @@ -3768,14 +3757,14 @@ def _register_functions(db, functions): try: functions = pathlib.Path(functions).read_text() except FileNotFoundError: - raise click.ClickException("File not found: {}".format(functions)) + raise click.ClickException(f"File not found: {functions}") sqlite3.enable_callback_tracebacks(True) globals = {} try: - exec(functions, globals) + exec(functions, globals) # noqa: S102 except SyntaxError as ex: - raise click.ClickException("Error in functions definition: {}".format(ex)) + raise click.ClickException(f"Error in functions definition: {ex}") # Register all callables in the locals dict: for name, value in globals.items(): if callable(value) and not name.startswith("_"): @@ -3796,12 +3785,12 @@ def _rows_from_code(code): try: code = pathlib.Path(code).read_text() except FileNotFoundError: - raise click.ClickException("File not found: {}".format(code)) + raise click.ClickException(f"File not found: {code}") namespace = {} try: - exec(code, namespace) + exec(code, namespace) # noqa: S102 except SyntaxError as ex: - raise click.ClickException("Error in --code: {}".format(ex)) + raise click.ClickException(f"Error in --code: {ex}") rows = namespace.get("rows") if callable(rows): rows = rows() diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e97b7d9..9a00123 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,19 +1,4 @@ -from .utils import ( - chunks, - dedupe_keys, - hash_record, - sqlite3, - OperationalError, - suggest_column_types, - types_for_column_types, - column_affinity, - progressbar, - find_spatialite, -) import binascii -from collections import namedtuple -from dataclasses import dataclass, field -from collections.abc import Mapping import contextlib import datetime import decimal @@ -25,26 +10,36 @@ import os import pathlib import re import secrets -from sqlite_fts4 import rank_bm25 import textwrap -from typing import ( - cast, - Any, - Callable, - Dict, - Generator, - Iterable, - Sequence, - Set, - Type, - Union, - Optional, - List, - Tuple, -) import uuid +from collections import namedtuple +from collections.abc import Callable, Generator, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from types import TracebackType +from typing import ( + Any, + Union, + cast, +) + +from sqlite_fts4 import rank_bm25 +from typing_extensions import Self + from sqlite_utils.plugins import ensure_plugins_loaded, pm +from .utils import ( + OperationalError, + chunks, + column_affinity, + dedupe_keys, + find_spatialite, + hash_record, + progressbar, + sqlite3, + suggest_column_types, + types_for_column_types, +) + try: iterdump = importlib.import_module("sqlite_dump").iterdump except ImportError: @@ -226,11 +221,11 @@ class ForeignKey: table: str # column/other_column are None for compound keys, which would break # ordering against str values - comparison uses columns/other_columns - column: Optional[str] = field(compare=False) + column: str | None = field(compare=False) other_table: str - other_column: Optional[str] = field(compare=False) - columns: Tuple[str, ...] = () - other_columns: Tuple[str, ...] = () + other_column: str | None = field(compare=False) + columns: tuple[str, ...] = () + other_columns: tuple[str, ...] = () is_compound: bool = False on_delete: str = "NO ACTION" on_update: str = "NO ACTION" @@ -259,9 +254,9 @@ def _fk_actions_sql(fk: ForeignKey) -> str: "ON UPDATE/ON DELETE clauses for a foreign key, or an empty string." actions = "" if fk.on_update and fk.on_update != "NO ACTION": - actions += " ON UPDATE {}".format(fk.on_update) + actions += f" ON UPDATE {fk.on_update}" if fk.on_delete and fk.on_delete != "NO ACTION": - actions += " ON DELETE {}".format(fk.on_delete) + actions += f" ON DELETE {fk.on_delete}" return actions @@ -278,20 +273,20 @@ class TransformError(Exception): # A single column name, or a tuple of columns for a compound foreign key -ForeignKeyColumns = Union[str, Tuple[str, ...], List[str]] +ForeignKeyColumns = str | tuple[str, ...] | list[str] # (table, column(s), other_table, other_column(s)) -ForeignKeyTuple = Tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] +ForeignKeyTuple = tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] -ForeignKeyIndicator = Union[ - str, - ForeignKey, - Tuple[ForeignKeyColumns, str], - Tuple[ForeignKeyColumns, str, ForeignKeyColumns], - ForeignKeyTuple, -] +ForeignKeyIndicator = ( + str + | ForeignKey + | tuple[ForeignKeyColumns, str] + | tuple[ForeignKeyColumns, str, ForeignKeyColumns] + | ForeignKeyTuple +) -ForeignKeysType = Union[Iterable[ForeignKeyIndicator], List[ForeignKeyIndicator]] +ForeignKeysType = Iterable[ForeignKeyIndicator] | list[ForeignKeyIndicator] class Default: @@ -300,7 +295,7 @@ class Default: DEFAULT = Default() -Tracer = Callable[[str, Optional[Union[Sequence[Any], Dict[str, Any]]]], None] +Tracer = Callable[[str, Sequence[Any] | dict[str, Any] | None], None] def _iter_complete_sql_statements(sql: str) -> Generator[str, None, None]: @@ -316,7 +311,7 @@ def _iter_complete_sql_statements(sql: str) -> Generator[str, None, None]: yield statement_sql -COLUMN_TYPE_MAPPING: Dict[Any, str] = { +COLUMN_TYPE_MAPPING: dict[Any, str] = { float: "REAL", int: "INTEGER", bool: "INTEGER", @@ -512,12 +507,12 @@ class Database: def __init__( self, - filename_or_conn: Optional[Union[str, pathlib.Path, sqlite3.Connection]] = None, + filename_or_conn: str | pathlib.Path | sqlite3.Connection | None = None, memory: bool = False, - memory_name: Optional[str] = None, + memory_name: str | None = None, recreate: bool = False, recursive_triggers: bool = True, - tracer: Optional[Tracer] = None, + tracer: Tracer | None = None, use_counts_table: bool = False, execute_plugins: bool = True, use_old_upsert: bool = False, @@ -532,7 +527,7 @@ class Database: ): raise ValueError("Either specify a filename_or_conn or pass memory=True") if memory_name: - uri = "file:{}?mode=memory&cache=shared".format(memory_name) + uri = f"file:{memory_name}?mode=memory&cache=shared" self.conn = sqlite3.connect( uri, uri=True, @@ -569,7 +564,7 @@ class Database: "transaction handling - connections created with " "autocommit=True or autocommit=False are not supported" ) - self._tracer: Optional[Tracer] = tracer + self._tracer: Tracer | None = tracer if recursive_triggers: self.execute("PRAGMA recursive_triggers=on;") self._registered_functions: set = set() @@ -579,14 +574,14 @@ class Database: pm.hook.prepare_connection(conn=self.conn) self.strict = strict - def __enter__(self) -> "Database": + def __enter__(self) -> Self: return self def __exit__( self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[object], + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, ) -> None: self.close() @@ -602,8 +597,8 @@ class Database: Nested blocks use SQLite savepoints. """ if self.conn.in_transaction: - savepoint = "sqlite_utils_{}".format(secrets.token_hex(16)) - self.conn.execute("SAVEPOINT {};".format(savepoint)) + savepoint = f"sqlite_utils_{secrets.token_hex(16)}" + self.conn.execute(f"SAVEPOINT {savepoint};") try: yield self except BaseException: @@ -612,11 +607,11 @@ class Database: # anyway would mask the original exception with # "no such savepoint" if self.conn.in_transaction: - self.conn.execute("ROLLBACK TO SAVEPOINT {};".format(savepoint)) - self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) + self.conn.execute(f"ROLLBACK TO SAVEPOINT {savepoint};") + self.conn.execute(f"RELEASE SAVEPOINT {savepoint};") raise else: - self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) + self.conn.execute(f"RELEASE SAVEPOINT {savepoint};") else: self.conn.execute("BEGIN") try: @@ -695,9 +690,7 @@ class Database: self.conn.isolation_level = old_isolation_level @contextlib.contextmanager - def tracer( - self, tracer: Optional[Tracer] = None - ) -> Generator["Database", None, None]: + def tracer(self, tracer: Tracer | None = None) -> Generator["Database", None, None]: """ Context manager to temporarily set a tracer function - all executed SQL queries will be passed to this. @@ -734,15 +727,15 @@ class Database: return self.table(table_name) def __repr__(self) -> str: - return "".format(self.conn) + return f"" def register_function( self, - fn: Optional[Callable] = None, + fn: Callable | None = None, deterministic: bool = False, replace: bool = False, - name: Optional[str] = None, - ) -> Optional[Callable[[Callable], Callable]]: + name: str | None = None, + ) -> Callable[[Callable], Callable] | None: """ ``fn`` will be made available as a function within SQL, with the same name and number of arguments. Can be used as a decorator:: @@ -770,7 +763,7 @@ class Database: arity = len(inspect.signature(fn).parameters) if not replace and (fn_name, arity) in self._registered_functions: return fn - kwargs: Dict[str, bool] = {} + kwargs: dict[str, bool] = {} registered = False if deterministic: # Try this, but fall back if sqlite3.NotSupportedError @@ -796,7 +789,7 @@ class Database: "Register the ``rank_bm25(match_info)`` function used for calculating relevance with SQLite FTS4." self.register_function(rank_bm25, deterministic=True, replace=True) - def attach(self, alias: str, filepath: Union[str, pathlib.Path]) -> None: + def attach(self, alias: str, filepath: str | pathlib.Path) -> None: """ Attach another SQLite database file to this connection with the specified alias, equivalent to:: @@ -805,15 +798,13 @@ class Database: :param alias: Alias name to use :param filepath: Path to SQLite database file on disk """ - attach_sql = """ - ATTACH DATABASE '{}' AS {}; - """.format( - str(pathlib.Path(filepath).resolve()), quote_identifier(alias) - ).strip() + attach_sql = f""" + ATTACH DATABASE '{pathlib.Path(filepath).resolve()!s}' AS {quote_identifier(alias)}; + """.strip() self.execute(attach_sql) def query( - self, sql: str, params: Optional[Union[Sequence, Dict[str, Any]]] = None + self, sql: str, params: Sequence | dict[str, Any] | None = None ) -> Generator[dict, None, None]: """ Execute ``sql`` and return an iterable of dictionaries representing each row. @@ -891,7 +882,7 @@ class Database: self.conn.execute('RELEASE "sqlite_utils_query"') def execute( - self, sql: str, parameters: Optional[Union[Sequence, Dict[str, Any]]] = None + self, sql: str, parameters: Sequence | dict[str, Any] | None = None ) -> sqlite3.Cursor: """ Execute SQL query and return a ``sqlite3.Cursor``. @@ -960,7 +951,7 @@ class Database: :param table_name: Name of the table """ if table_name in self.view_names(): - raise NoTable("Table {} is actually a view".format(table_name)) + raise NoTable(f"Table {table_name} is actually a view") kwargs.setdefault("strict", self.strict) return Table(self, table_name, **kwargs) @@ -973,11 +964,9 @@ class Database: if view_name not in self.view_names(): if view_name in self.table_names(): raise NoView( - "View {name} does not exist - {name} is a table".format( - name=view_name - ) + f"View {view_name} does not exist - {view_name} is a table" ) - raise NoView("View {} does not exist".format(view_name)) + raise NoView(f"View {view_name} does not exist") return View(self, view_name) def quote(self, value: str) -> str: @@ -1013,9 +1002,7 @@ class Database: query += '"' bits = _quote_fts_re.split(query) bits = [b for b in bits if b and b != '""'] - return " ".join( - '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits - ) + return " ".join(f'"{bit}"' if not bit.startswith('"') else bit for bit in bits) def quote_default_value(self, value: str) -> str: if any( @@ -1036,11 +1023,11 @@ class Database: if str(value).endswith(")"): # Expr - return "({})".format(value) + return f"({value})" return self.quote(value) - def table_names(self, fts4: bool = False, fts5: bool = False) -> List[str]: + def table_names(self, fts4: bool = False, fts5: bool = False) -> list[str]: """ List of string table names in this database. @@ -1055,7 +1042,7 @@ class Database: sql = "select name from sqlite_master where {}".format(" AND ".join(where)) return [r[0] for r in self.execute(sql).fetchall()] - def view_names(self) -> List[str]: + def view_names(self) -> list[str]: "List of string view names in this database." return [ r[0] @@ -1065,17 +1052,17 @@ class Database: ] @property - def tables(self) -> List["Table"]: + def tables(self) -> list["Table"]: "List of Table objects in this database." return [self.table(name) for name in self.table_names()] @property - def views(self) -> List["View"]: + def views(self) -> list["View"]: "List of View objects in this database." return [self.view(name) for name in self.view_names()] @property - def triggers(self) -> List[Trigger]: + def triggers(self) -> list[Trigger]: "List of ``(name, table_name, sql)`` tuples representing triggers in this database." return [ Trigger(*r) @@ -1085,7 +1072,7 @@ class Database: ] @property - def triggers_dict(self) -> Dict[str, str]: + def triggers_dict(self) -> dict[str, str]: "A ``{trigger_name: sql}`` dictionary of triggers in this database." return {trigger.name: trigger.sql for trigger in self.triggers} @@ -1107,14 +1094,12 @@ class Database: "Does this database support STRICT mode?" if not hasattr(self, "_supports_strict"): try: - table_name = "t{}".format(secrets.token_hex(16)) + table_name = f"t{secrets.token_hex(16)}" with self.atomic(): - self.conn.execute( - "create table {} (name text) strict".format(table_name) - ) - self.conn.execute("drop table {}".format(table_name)) + self.conn.execute(f"create table {table_name} (name text) strict") + self.conn.execute(f"drop table {table_name}") self._supports_strict = True - except Exception: + except sqlite3.OperationalError: self._supports_strict = False return self._supports_strict @@ -1122,32 +1107,28 @@ class Database: def supports_on_conflict(self) -> bool: # SQLite's upsert is implemented as INSERT INTO ... ON CONFLICT DO ... if not hasattr(self, "_supports_on_conflict"): - table_name = "t{}".format(secrets.token_hex(16)) + table_name = f"t{secrets.token_hex(16)}" try: with self.atomic(): self.conn.execute( - "create table {} (id integer primary key, name text)".format( - table_name - ) + f"create table {table_name} (id integer primary key, name text)" ) self.conn.execute( - "insert into {} (id, name) values (1, 'one')".format(table_name) + f"insert into {table_name} (id, name) values (1, 'one')" ) self.conn.execute( - ( - "insert into {} (id, name) values (1, 'two') " - "on conflict do update set name = 'two'" - ).format(table_name) + f"insert into {table_name} (id, name) values (1, 'two') " + "on conflict do update set name = 'two'" ) self._supports_on_conflict = True - except Exception: + except sqlite3.OperationalError: self._supports_on_conflict = False finally: - self.conn.execute("drop table if exists {}".format(table_name)) + self.conn.execute(f"drop table if exists {table_name}") return self._supports_on_conflict @property - def sqlite_version(self) -> Tuple[int, ...]: + def sqlite_version(self) -> tuple[int, ...]: "Version of SQLite, as a tuple of integers for example ``(3, 36, 0)``." row = self.execute("select sqlite_version()").fetchall()[0] return tuple(map(int, row[0].split("."))) @@ -1191,7 +1172,7 @@ class Database: # guarantee of atomic() and of user-managed transactions if self.conn.in_transaction: raise TransactionError( - "{} cannot be used while a transaction is open".format(operation) + f"{operation} cannot be used while a transaction is open" ) def _ensure_counts_table(self) -> None: @@ -1212,14 +1193,14 @@ class Database: table.enable_counts() self.use_counts_table = True - def cached_counts(self, tables: Optional[Iterable[str]] = None) -> Dict[str, int]: + def cached_counts(self, tables: Iterable[str] | None = None) -> dict[str, int]: """ Return ``{table_name: count}`` dictionary of cached counts for specified tables, or all tables if ``tables`` not provided. :param tables: Subset list of tables to return counts for. """ - sql = 'select "table", count from {}'.format(self._counts_table_name) + sql = f'select "table", count from {self._counts_table_name}' tables_list = list(tables) if tables else None if tables_list: sql += ' where "table" in ({})'.format(", ".join("?" for _ in tables_list)) @@ -1241,13 +1222,13 @@ class Database: ) def execute_returning_dicts( - self, sql: str, params: Optional[Union[Sequence, Dict[str, Any]]] = None - ) -> List[dict]: + self, sql: str, params: Sequence | dict[str, Any] | None = None + ) -> list[dict]: return list(self.query(sql, params)) def resolve_foreign_keys( self, name: str, foreign_keys: ForeignKeysType - ) -> List[ForeignKey]: + ) -> list[ForeignKey]: """ Given a list of differing foreign_keys definitions, return a list of fully resolved ForeignKey() named tuples. @@ -1274,7 +1255,7 @@ class Database: fks.append(ForeignKey(name, fk, other_table, other_column)) continue if not isinstance(fk, (tuple, list)): - raise ValueError( + raise ValueError( # noqa: TRY004 "foreign_keys= should be a list of tuples, " "ForeignKey objects or column name strings" ) @@ -1282,9 +1263,7 @@ class Database: if len(tuple_or_list) == 4: if tuple_or_list[0] != name: raise ValueError( - "First item in {} should have been {}".format( - tuple_or_list, name - ) + f"First item in {tuple_or_list} should have been {name}" ) tuple_or_list = tuple_or_list[1:] if len(tuple_or_list) not in (2, 3): @@ -1299,8 +1278,8 @@ class Database: if len(tuple_or_list) == 3: if not isinstance(tuple_or_list[2], (list, tuple)): raise ValueError( - "Compound foreign key {} should reference a tuple " - "of other columns".format(tuple(tuple_or_list)) + f"Compound foreign key {tuple(tuple_or_list)} should reference a tuple " + "of other columns" ) other_columns = tuple(tuple_or_list[2]) else: @@ -1308,8 +1287,8 @@ class Database: other_columns = tuple(self.table(other_table).pks) if len(columns) != len(other_columns): raise ValueError( - "Compound foreign key {} should have the same number " - "of columns on both sides".format(tuple(tuple_or_list)) + f"Compound foreign key {tuple(tuple_or_list)} should have the same number " + "of columns on both sides" ) if len(columns) == 1: # Single-column key passed as a one-item list @@ -1389,15 +1368,15 @@ class Database: def create_table_sql( self, name: str, - columns: Dict[str, Any], - pk: Optional[Any] = None, - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, - hash_id: Optional[str] = None, - hash_id_columns: Optional[Iterable[str]] = None, - extracts: Optional[Union[Dict[str, str], List[str]]] = None, + columns: dict[str, Any], + pk: Any | None = None, + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, + hash_id: str | None = None, + hash_id_columns: Iterable[str] | None = None, + extracts: dict[str, str] | list[str] | None = None, if_not_exists: bool = False, strict: bool = False, ) -> str: @@ -1419,7 +1398,7 @@ class Database: """ if hash_id_columns and (hash_id is None): hash_id = "id" - resolved_fks: List[ForeignKey] = [ + resolved_fks: list[ForeignKey] = [ self._resolve_foreign_key_casing(fk, columns) for fk in self.resolve_foreign_keys(name, foreign_keys or []) ] @@ -1449,15 +1428,11 @@ class Database: raise ValueError("Tables must have at least one column") if not all(n in columns for n in not_null): raise ValueError( - "not_null set {} includes items not in columns {}".format( - repr(not_null), repr(set(columns.keys())) - ) + f"not_null set {not_null!r} includes items not in columns {set(columns.keys())!r}" ) if not all(n in columns for n in defaults): raise ValueError( - "defaults set {} includes items not in columns {}".format( - repr(set(defaults)), repr(set(columns.keys())) - ) + f"defaults set {set(defaults)!r} includes items not in columns {set(columns.keys())!r}" ) column_items = list(columns.items()) if column_order is not None: @@ -1477,9 +1452,7 @@ class Database: if other_column != "rowid" and not any( c for c in self[fk.other_table].columns if c.name == other_column ): - raise AlterError( - "No such column: {}.{}".format(fk.other_table, other_column) - ) + raise AlterError(f"No such column: {fk.other_table}.{other_column}") column_defs = [] # ensure pk is a tuple @@ -1500,16 +1473,12 @@ class Database: column_extras.append("NOT NULL") if column_name in defaults and defaults[column_name] is not None: column_extras.append( - "DEFAULT {}".format(self.quote_default_value(defaults[column_name])) + f"DEFAULT {self.quote_default_value(defaults[column_name])}" ) if column_name in foreign_keys_by_column: fk = foreign_keys_by_column[column_name] column_extras.append( - "REFERENCES {}({}){}".format( - quote_identifier(fk.other_table), - quote_identifier(cast(str, fk.other_column)), - _fk_actions_sql(fk), - ) + f"REFERENCES {quote_identifier(fk.other_table)}({quote_identifier(cast(str, fk.other_column))}){_fk_actions_sql(fk)}" ) column_type_str = COLUMN_TYPE_MAPPING[column_type] # Special case for strict tables to map FLOAT to REAL @@ -1566,15 +1535,15 @@ class Database: def create_table( self, name: str, - columns: Dict[str, Any], - pk: Optional[Any] = None, - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, - hash_id: Optional[str] = None, - hash_id_columns: Optional[Iterable[str]] = None, - extracts: Optional[Union[Dict[str, str], List[str]]] = None, + columns: dict[str, Any], + pk: Any | None = None, + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, + hash_id: str | None = None, + hash_id_columns: Iterable[str] | None = None, + extracts: dict[str, str] | list[str] | None = None, if_not_exists: bool = False, replace: bool = False, ignore: bool = False, @@ -1618,11 +1587,11 @@ class Database: resolve_casing(col_name, existing_columns): col_type for col_name, col_type in columns.items() } - missing_columns = dict( - (col_name, col_type) + missing_columns = { + col_name: col_type for col_name, col_type in columns.items() if col_name not in existing_columns - ) + } columns_to_drop = [ column for column in existing_columns if column not in columns ] @@ -1709,9 +1678,7 @@ class Database: :param new_name: Name to rename it to """ self.execute( - "ALTER TABLE {} RENAME TO {}".format( - quote_identifier(name), quote_identifier(new_name) - ) + f"ALTER TABLE {quote_identifier(name)} RENAME TO {quote_identifier(new_name)}" ) def create_view( @@ -1727,23 +1694,20 @@ class Database: """ if ignore and replace: raise ValueError("Use one or the other of ignore/replace, not both") - create_sql = "CREATE VIEW {name} AS {sql}".format( - name=quote_identifier(name), sql=sql - ) - if ignore or replace: - # Does view exist already? - if name in self.view_names(): - if ignore: + create_sql = f"CREATE VIEW {quote_identifier(name)} AS {sql}" + if (ignore or replace) and name in self.view_names(): + # View exists already + if ignore: + return self + elif replace: + # If SQL is the same, do nothing + if create_sql == self[name].schema: return self - elif replace: - # If SQL is the same, do nothing - if create_sql == self[name].schema: - return self - self[name].drop() + self[name].drop() self.execute(create_sql) return self - def m2m_table_candidates(self, table: str, other_table: str) -> List[str]: + def m2m_table_candidates(self, table: str, other_table: str) -> list[str]: """ Given two table names returns the name of tables that could define a many-to-many relationship between those two tables, based on having @@ -1762,7 +1726,7 @@ class Database: return candidates def add_foreign_keys( - self, foreign_keys: Iterable[Union[ForeignKey, ForeignKeyTuple]] + self, foreign_keys: Iterable[ForeignKey | ForeignKeyTuple] ) -> None: """ See :ref:`python_api_add_foreign_keys`. @@ -1782,7 +1746,7 @@ class Database: "(table, column, other_table, other_column)" ) - foreign_keys_to_create: List[ForeignKey] = [] + foreign_keys_to_create: list[ForeignKey] = [] # Verify that all tables and columns exist for fk in foreign_keys: @@ -1823,7 +1787,7 @@ class Database: table = fk_object.table other_table = fk_object.other_table if not self.table(table).exists(): - raise AlterError("No such table: {}".format(table)) + raise AlterError(f"No such table: {table}") table_obj = self.table(table) fk_object = self._resolve_foreign_key_casing( fk_object, table_obj.columns_dict @@ -1832,18 +1796,16 @@ class Database: other_columns = fk_object.other_columns for column in columns: if column not in table_obj.columns_dict: - raise AlterError("No such column: {} in {}".format(column, table)) + raise AlterError(f"No such column: {column} in {table}") if not self[other_table].exists(): - raise AlterError("No such other_table: {}".format(other_table)) + raise AlterError(f"No such other_table: {other_table}") for other_column in other_columns: if ( other_column != "rowid" and other_column not in self[other_table].columns_dict ): raise AlterError( - "No such other_column: {} in {}".format( - other_column, other_table - ) + f"No such other_column: {other_column} in {other_table}" ) # Silently skip foreign keys that exist already - but only if # they match exactly, including ON DELETE/ON UPDATE actions @@ -1874,7 +1836,7 @@ class Database: ) # Group them by table - by_table: Dict[str, List[ForeignKey]] = {} + by_table: dict[str, list[ForeignKey]] = {} for fk_object in foreign_keys_to_create: by_table.setdefault(fk_object.table, []).append(fk_object) @@ -1899,7 +1861,7 @@ class Database: "Run a SQLite ``VACUUM`` against the database." self.execute("VACUUM;") - def analyze(self, name: Optional[str] = None) -> None: + def analyze(self, name: str | None = None) -> None: """ Run ``ANALYZE`` against the entire database or a named table or index. @@ -1907,7 +1869,7 @@ class Database: """ sql = "ANALYZE" if name is not None: - sql += " {}".format(quote_identifier(name)) + sql += f" {quote_identifier(name)}" self.execute(sql) def iterdump(self) -> Generator[str, None, None]: @@ -1922,7 +1884,7 @@ class Database: "conn.iterdump() not found - try pip install sqlite-dump" ) - def init_spatialite(self, path: Optional[str] = None) -> bool: + def init_spatialite(self, path: str | None = None) -> bool: """ The ``init_spatialite`` method will load and initialize the SpatiaLite extension. The ``path`` argument should be an absolute path to the compiled extension, which @@ -1980,8 +1942,8 @@ class Queryable: def count_where( self, - where: Optional[str] = None, - where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + where: str | None = None, + where_args: Sequence | dict[str, Any] | None = None, ) -> int: """ Executes ``SELECT count(*) FROM table WHERE ...`` and returns a count. @@ -1990,7 +1952,7 @@ class Queryable: :param where_args: Parameters to use with that fragment - an iterable for ``id > ?`` parameters, or a dictionary for ``id > :id`` """ - sql = "select count(*) from {}".format(quote_identifier(self.name)) + sql = f"select count(*) from {quote_identifier(self.name)}" if where is not None: sql += " where " + where return self.db.execute(sql, where_args or []).fetchone()[0] @@ -2005,19 +1967,19 @@ class Queryable: return self.count_where() @property - def rows(self) -> Generator[Dict[str, Any], None, None]: + def rows(self) -> Generator[dict[str, Any], None, None]: "Iterate over every dictionaries for each row in this table or view." return self.rows_where() def rows_where( self, - where: Optional[str] = None, - where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, - order_by: Optional[str] = None, + where: str | None = None, + where_args: Sequence | dict[str, Any] | None = None, + order_by: str | None = None, select: str = "*", - limit: Optional[int] = None, - offset: Optional[int] = None, - ) -> Generator[Dict[str, Any], None, None]: + limit: int | None = None, + offset: int | None = None, + ) -> Generator[dict[str, Any], None, None]: """ Iterate over every row in this table or view that matches the specified where clause. @@ -2033,15 +1995,15 @@ class Queryable: """ if not self.exists(): return - sql = "select {} from {}".format(select, quote_identifier(self.name)) + sql = f"select {select} from {quote_identifier(self.name)}" if where is not None: sql += " where " + where if order_by is not None: sql += " order by " + order_by if limit is not None: - sql += " limit {}".format(limit) + sql += f" limit {limit}" if offset is not None: - sql += " offset {}".format(offset) + sql += f" offset {offset}" cursor = self.db.execute(sql, where_args or []) columns = dedupe_keys(c[0] for c in cursor.description) for row in cursor: @@ -2049,12 +2011,12 @@ class Queryable: def pks_and_rows_where( self, - where: Optional[str] = None, - where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, - order_by: Optional[str] = None, - limit: Optional[int] = None, - offset: Optional[int] = None, - ) -> Generator[Tuple[Any, Dict[str, Any]], None, None]: + where: str | None = None, + where_args: Sequence | dict[str, Any] | None = None, + order_by: str | None = None, + limit: int | None = None, + offset: int | None = None, + ) -> Generator[tuple[Any, dict[str, Any]], None, None]: """ Like ``.rows_where()`` but returns ``(pk, row)`` pairs - ``pk`` can be a single value or tuple. @@ -2096,17 +2058,17 @@ class Queryable: yield row_pk, row @property - def columns(self) -> List["Column"]: + def columns(self) -> list["Column"]: "List of :ref:`Columns ` representing the columns in this table or view." if not self.exists(): return [] rows = self.db.execute( - "PRAGMA table_info({})".format(quote_identifier(self.name)) + f"PRAGMA table_info({quote_identifier(self.name)})" ).fetchall() return [Column(*row) for row in rows] @property - def columns_dict(self) -> Dict[str, Any]: + def columns_dict(self) -> dict[str, Any]: "``{column_name: python-type}`` dictionary representing columns in this table or view." return {column.name: column_affinity(column.type) for column in self.columns} @@ -2146,48 +2108,48 @@ class Table(Queryable): """ #: The ``rowid`` of the last inserted, updated or selected row. - last_rowid: Optional[int] = None + last_rowid: int | None = None #: The primary key of the last inserted, updated or selected row. - last_pk: Optional[Any] = None + last_pk: Any | None = None def __init__( self, db: Database, name: str, - pk: Optional[Any] = None, - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, + pk: Any | None = None, + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, batch_size: int = 100, - hash_id: Optional[str] = None, - hash_id_columns: Optional[Iterable[str]] = None, + hash_id: str | None = None, + hash_id_columns: Iterable[str] | None = None, alter: bool = False, ignore: bool = False, replace: bool = False, - extracts: Optional[Union[Dict[str, str], List[str]]] = None, - conversions: Optional[dict] = None, - columns: Optional[Dict[str, Any]] = None, + extracts: dict[str, str] | list[str] | None = None, + conversions: dict | None = None, + columns: dict[str, Any] | None = None, strict: bool = False, ): super().__init__(db, name) - self._defaults = dict( - pk=pk, - foreign_keys=foreign_keys, - column_order=column_order, - not_null=not_null, - defaults=defaults, - batch_size=batch_size, - hash_id=hash_id, - hash_id_columns=hash_id_columns, - alter=alter, - ignore=ignore, - replace=replace, - extracts=extracts, - conversions=conversions or {}, - columns=columns, - strict=strict, - ) + self._defaults = { + "pk": pk, + "foreign_keys": foreign_keys, + "column_order": column_order, + "not_null": not_null, + "defaults": defaults, + "batch_size": batch_size, + "hash_id": hash_id, + "hash_id_columns": hash_id_columns, + "alter": alter, + "ignore": ignore, + "replace": replace, + "extracts": extracts, + "conversions": conversions or {}, + "columns": columns, + "strict": strict, + } def __repr__(self) -> str: return "".format( @@ -2212,7 +2174,7 @@ class Table(Queryable): return self.name in self.db.table_names() @property - def pks(self) -> List[str]: + def pks(self) -> list[str]: """ Primary key columns for this table, in PRIMARY KEY declaration order - ``PRAGMA table_info`` sets ``is_pk`` to the 1-based position of each @@ -2234,7 +2196,7 @@ class Table(Queryable): "Does this table use ``rowid`` for its primary key (no other primary keys are specified)?" return not any(column for column in self.columns if column.is_pk) - def get(self, pk_values: Union[list, tuple, str, int]) -> dict: + def get(self, pk_values: list | tuple | str | int) -> dict: """ Return row (as dictionary) for the specified primary key. @@ -2253,17 +2215,17 @@ class Table(Queryable): ) ) - wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in pks] + wheres = [f"{quote_identifier(pk_name)} = ?" for pk_name in pks] rows = self.rows_where(" and ".join(wheres), pk_values) try: - row = list(rows)[0] + row = next(iter(rows)) self.last_pk = last_pk return row - except IndexError: + except StopIteration: raise NotFoundError @property - def foreign_keys(self) -> List["ForeignKey"]: + def foreign_keys(self) -> list["ForeignKey"]: """ List of foreign keys defined on this table. @@ -2273,12 +2235,12 @@ class Table(Queryable): """ # PRAGMA foreign_key_list returns one row per column, grouped by "id" # with "seq" giving the column order within a compound foreign key. - by_id: Dict[int, list] = {} + by_id: dict[int, list] = {} for row in self.db.execute( - "PRAGMA foreign_key_list({})".format(quote_identifier(self.name)) + f"PRAGMA foreign_key_list({quote_identifier(self.name)})" ).fetchall(): if row is not None: - id, seq, table_name, from_, to_, on_update, on_delete, match = row + id, seq, table_name, from_, to_, on_update, on_delete, _match = row by_id.setdefault(id, []).append( (seq, table_name, from_, to_, on_update, on_delete) ) @@ -2311,7 +2273,7 @@ class Table(Queryable): return fks @property - def virtual_table_using(self) -> Optional[str]: + def virtual_table_using(self) -> str | None: "Type of virtual table, or ``None`` if this is not a virtual table." match = _virtual_table_using_re.match(self.schema) if match is None: @@ -2319,18 +2281,16 @@ class Table(Queryable): return match.groupdict()["using"].upper() @property - def indexes(self) -> List[Index]: + def indexes(self) -> list[Index]: "List of indexes defined on this table." - sql = 'PRAGMA index_list("{}")'.format(self.name) + sql = f'PRAGMA index_list("{self.name}")' indexes = [] for row in self.db.execute_returning_dicts(sql): index_name = row["name"] index_name_quoted = ( - '"{}"'.format(index_name) - if not index_name.startswith('"') - else index_name + f'"{index_name}"' if not index_name.startswith('"') else index_name ) - column_sql = "PRAGMA index_info({})".format(index_name_quoted) + column_sql = f"PRAGMA index_info({index_name_quoted})" columns = [] for seqno, cid, name in self.db.execute(column_sql).fetchall(): columns.append(name) @@ -2343,18 +2303,16 @@ class Table(Queryable): return indexes @property - def xindexes(self) -> List[XIndex]: + def xindexes(self) -> list[XIndex]: "List of indexes defined on this table using the more detailed ``XIndex`` format." - sql = 'PRAGMA index_list("{}")'.format(self.name) + sql = f'PRAGMA index_list("{self.name}")' indexes = [] for row in self.db.execute_returning_dicts(sql): index_name = row["name"] index_name_quoted = ( - '"{}"'.format(index_name) - if not index_name.startswith('"') - else index_name + f'"{index_name}"' if not index_name.startswith('"') else index_name ) - column_sql = "PRAGMA index_xinfo({})".format(index_name_quoted) + column_sql = f"PRAGMA index_xinfo({index_name_quoted})" index_columns = [] for info in self.db.execute(column_sql).fetchall(): index_columns.append(XIndexColumn(*info)) @@ -2362,7 +2320,7 @@ class Table(Queryable): return indexes @property - def triggers(self) -> List[Trigger]: + def triggers(self) -> list[Trigger]: "List of triggers defined on this table." return [ Trigger(*r) @@ -2374,12 +2332,12 @@ class Table(Queryable): ] @property - def triggers_dict(self) -> Dict[str, str]: + def triggers_dict(self) -> dict[str, str]: "``{trigger_name: sql}`` dictionary of triggers defined on this table." return {trigger.name: trigger.sql for trigger in self.triggers} @property - def default_values(self) -> Dict[str, Any]: + def default_values(self) -> dict[str, Any]: "``{column_name: default_value}`` dictionary of default values for columns in this table." return { column.name: _decode_default_value(column.default_value) @@ -2396,20 +2354,20 @@ class Table(Queryable): def create( self, - columns: Dict[str, Any], - pk: Optional[Any] = DEFAULT, - foreign_keys: Union[Optional[ForeignKeysType], Default] = DEFAULT, - column_order: Union[Optional[List[str]], Default] = DEFAULT, - not_null: Union[Optional[Iterable[str]], Default] = DEFAULT, - defaults: Union[Optional[Dict[str, Any]], Default] = DEFAULT, - hash_id: Union[Optional[str], Default] = DEFAULT, - hash_id_columns: Union[Optional[Iterable[str]], Default] = DEFAULT, - extracts: Union[Optional[Union[Dict[str, str], List[str]]], Default] = DEFAULT, + columns: dict[str, Any], + pk: Any | None = DEFAULT, + foreign_keys: ForeignKeysType | None | Default = DEFAULT, + column_order: list[str] | None | Default = DEFAULT, + not_null: Iterable[str] | None | Default = DEFAULT, + defaults: dict[str, Any] | None | Default = DEFAULT, + hash_id: str | None | Default = DEFAULT, + hash_id_columns: Iterable[str] | None | Default = DEFAULT, + extracts: dict[str, str] | list[str] | None | Default = DEFAULT, if_not_exists: bool = False, replace: bool = False, ignore: bool = False, transform: bool = False, - strict: Union[bool, Default] = DEFAULT, + strict: bool | Default = DEFAULT, ) -> "Table": """ Create a table with the specified columns. @@ -2493,28 +2451,25 @@ class Table(Queryable): if not self.exists(): raise NoTable(f"Table {self.name} does not exist") with self.db.atomic(): - sql = "CREATE TABLE {} AS SELECT * FROM {};".format( - quote_identifier(new_name), - quote_identifier(self.name), - ) + sql = f"CREATE TABLE {quote_identifier(new_name)} AS SELECT * FROM {quote_identifier(self.name)};" self.db.execute(sql) return self.db.table(new_name) def transform( self, *, - types: Optional[dict] = None, - rename: Optional[dict] = None, - drop: Optional[Iterable] = None, - pk: Optional[Any] = DEFAULT, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, - drop_foreign_keys: Optional[Iterable[str]] = None, - add_foreign_keys: Optional[ForeignKeysType] = None, - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - keep_table: Optional[str] = None, - strict: Optional[bool] = None, + types: dict | None = None, + rename: dict | None = None, + drop: Iterable | None = None, + pk: Any | None = DEFAULT, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, + drop_foreign_keys: Iterable[str] | None = None, + add_foreign_keys: ForeignKeysType | None = None, + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + keep_table: str | None = None, + strict: bool | None = None, ) -> "Table": """ Apply an advanced alter table, including operations that are not supported by @@ -2633,20 +2588,20 @@ class Table(Queryable): def transform_sql( self, *, - types: Optional[dict] = None, - rename: Optional[dict] = None, - drop: Optional[Iterable] = None, - pk: Optional[Any] = DEFAULT, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, - drop_foreign_keys: Optional[Iterable] = None, - add_foreign_keys: Optional[ForeignKeysType] = None, - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - tmp_suffix: Optional[str] = None, - keep_table: Optional[str] = None, - strict: Optional[bool] = None, - ) -> List[str]: + types: dict | None = None, + rename: dict | None = None, + drop: Iterable | None = None, + pk: Any | None = DEFAULT, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, + drop_foreign_keys: Iterable | None = None, + add_foreign_keys: ForeignKeysType | None = None, + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + tmp_suffix: str | None = None, + keep_table: str | None = None, + strict: bool | None = None, + ) -> list[str]: """ Return a list of SQL statements that should be executed in order to apply this transformation. @@ -2689,7 +2644,7 @@ class Table(Queryable): if isinstance(not_null, dict): not_null = { resolve_casing(c, existing_columns): v - for c, v in cast(Dict[str, Any], not_null).items() + for c, v in cast(dict[str, Any], not_null).items() } elif isinstance(not_null, set): not_null = {resolve_casing(c, existing_columns) for c in not_null} @@ -2700,7 +2655,7 @@ class Table(Queryable): if column_order is not None: column_order = [resolve_casing(c, existing_columns) for c in column_order] - create_table_foreign_keys: List[ForeignKeyIndicator] = [] + create_table_foreign_keys: list[ForeignKeyIndicator] = [] if foreign_keys is not None: if add_foreign_keys is not None: @@ -2777,9 +2732,7 @@ class Table(Queryable): for fk in self.db.resolve_foreign_keys(self.name, add_foreign_keys): create_table_foreign_keys.append(fk_with_renamed_columns(fk)) - new_table_name = "{}_new_{}".format( - self.name, tmp_suffix or os.urandom(6).hex() - ) + new_table_name = f"{self.name}_new_{tmp_suffix or os.urandom(6).hex()}" current_column_pairs = list(self.columns_dict.items()) new_column_pairs = [] copy_from_to = {column: column for column, _ in current_column_pairs} @@ -2824,9 +2777,7 @@ class Table(Queryable): pass else: raise ValueError( - "not_null must be a dict or a set or None, it was {}".format( - repr(not_null) - ) + f"not_null must be a dict or a set or None, it was {not_null!r}" ) # defaults= create_table_defaults = { @@ -2876,17 +2827,13 @@ class Table(Queryable): # Drop (or keep) the old table if keep_table: sqls.append( - "ALTER TABLE {} RENAME TO {};".format( - quote_identifier(self.name), quote_identifier(keep_table) - ) + f"ALTER TABLE {quote_identifier(self.name)} RENAME TO {quote_identifier(keep_table)};" ) else: - sqls.append("DROP TABLE {};".format(quote_identifier(self.name))) + sqls.append(f"DROP TABLE {quote_identifier(self.name)};") # Rename the new one sqls.append( - "ALTER TABLE {} RENAME TO {};".format( - quote_identifier(new_table_name), quote_identifier(self.name) - ) + f"ALTER TABLE {quote_identifier(new_table_name)} RENAME TO {quote_identifier(self.name)};" ) # Re-add existing indexes for index in self.indexes: @@ -2904,7 +2851,7 @@ class Table(Queryable): if keep_table: sqls.append(f"DROP INDEX IF EXISTS {quote_identifier(index.name)};") for col in index.columns: - if col in rename.keys() or col in drop: + if col in rename or col in drop: raise TransformError( f"Index '{index.name}' column '{col}' is not in updated table '{self.name}'. " f"You must manually drop this index prior to running this transformation " @@ -2916,10 +2863,10 @@ class Table(Queryable): def extract( self, - columns: Union[str, Iterable[str]], - table: Optional[str] = None, - fk_column: Optional[str] = None, - rename: Optional[Dict[str, str]] = None, + columns: str | Iterable[str], + table: str | None = None, + fk_column: str | None = None, + rename: dict[str, str] | None = None, ) -> "Table": """ Extract specified columns into a separate table. @@ -2938,15 +2885,13 @@ class Table(Queryable): rename = {resolve_casing(k, self.columns_dict): v for k, v in rename.items()} if not set(columns).issubset(self.columns_dict.keys()): raise InvalidColumns( - "Invalid columns {} for table with columns {}".format( - columns, list(self.columns_dict.keys()) - ) + f"Invalid columns {columns} for table with columns {list(self.columns_dict.keys())}" ) with self.db.atomic(): table = table or "_".join(columns) lookup_table = self.db.table(table) - fk_column = fk_column or "{}_id".format(table) - magic_lookup_column = "{}_{}".format(fk_column, os.urandom(6).hex()) + fk_column = fk_column or f"{table}_id" + magic_lookup_column = f"{fk_column}_{os.urandom(6).hex()}" # Populate the lookup table with all of the extracted unique values lookup_columns_definition = { @@ -2959,16 +2904,12 @@ class Table(Queryable): lookup_table.columns_dict.items() ): raise InvalidColumns( - "Lookup table {} already exists but does not have columns {}".format( - table, lookup_columns_definition - ) + f"Lookup table {table} already exists but does not have columns {lookup_columns_definition}" ) else: lookup_table.create( { - **{ - "id": int, - }, + "id": int, **lookup_columns_definition, }, pk="id", @@ -2978,19 +2919,14 @@ class Table(Queryable): # Rows where every extracted column is null are left alone - they # get a null foreign key and no lookup table record, see #186 all_columns_are_null = " AND ".join( - "{} IS NULL".format(quote_identifier(c)) for c in columns + f"{quote_identifier(c)} IS NULL" for c in columns ) # INSERT OR IGNORE dedupes against the unique index, but unique # indexes treat NULLs as distinct - the NOT EXISTS guard uses IS # comparison so NULL-containing rows match existing lookup rows # instead of being inserted again already_in_lookup = " AND ".join( - "{lookup}.{lookup_col} IS {source}.{source_col}".format( - lookup=quote_identifier(table), - lookup_col=quote_identifier(rename.get(column) or column), - source=quote_identifier(self.name), - source_col=quote_identifier(column), - ) + f"{quote_identifier(table)}.{quote_identifier(rename.get(column) or column)} IS {quote_identifier(self.name)}.{quote_identifier(column)}" for column in columns ) self.db.execute( @@ -3018,12 +2954,10 @@ class Table(Queryable): quote_identifier(magic_lookup_column), quote_identifier(table), where=" AND ".join( - "{}.{} IS {}.{}".format( - quote_identifier(self.name), - quote_identifier(column), - quote_identifier(table), - quote_identifier(rename.get(column) or column), - ) + f"{quote_identifier(self.name)}." + f"{quote_identifier(column)} IS " + f"{quote_identifier(table)}." + f"{quote_identifier(rename.get(column) or column)}" for column in columns ), all_null=all_columns_are_null, @@ -3052,8 +2986,8 @@ class Table(Queryable): def create_index( self, - columns: Iterable[Union[str, DescIndex]], - index_name: Optional[str] = None, + columns: Iterable[str | DescIndex], + index_name: str | None = None, unique: bool = False, if_not_exists: bool = False, find_unique_name: bool = False, @@ -3080,16 +3014,14 @@ class Table(Queryable): columns_sql = [] for column in columns: if isinstance(column, DescIndex): - columns_sql.append("{} desc".format(quote_identifier(column))) + columns_sql.append(f"{quote_identifier(column)} desc") else: columns_sql.append(quote_identifier(column)) suffix = None created_index_name = None while True: - created_index_name = ( - "{}_{}".format(index_name, suffix) if suffix else index_name - ) + created_index_name = f"{index_name}_{suffix}" if suffix else index_name sql = ( textwrap.dedent(""" CREATE {unique}INDEX {if_not_exists}{index_name} @@ -3121,7 +3053,7 @@ class Table(Queryable): suffix += 1 continue else: - raise e + raise if analyze: self.db.analyze(created_index_name) return self @@ -3136,19 +3068,17 @@ class Table(Queryable): if index_name not in {index.name for index in self.indexes}: if ignore: return self - raise OperationalError( - "No index named {} on table {}".format(index_name, self.name) - ) - self.db.execute("DROP INDEX {}".format(quote_identifier(index_name))) + raise OperationalError(f"No index named {index_name} on table {self.name}") + self.db.execute(f"DROP INDEX {quote_identifier(index_name)}") return self def add_column( self, col_name: str, - col_type: Optional[Any] = None, - fk: Optional[str] = None, - fk_col: Optional[str] = None, - not_null_default: Optional[Any] = None, + col_type: Any | None = None, + fk: str | None = None, + fk_col: str | None = None, + not_null_default: Any | None = None, ): """ Add a column to this table. See :ref:`python_api_add_column`. @@ -3163,12 +3093,12 @@ class Table(Queryable): if fk is not None: # fk must be a valid table if fk not in self.db.table_names(): - raise AlterError("table '{}' does not exist".format(fk)) + raise AlterError(f"table '{fk}' does not exist") # if fk_col specified, must be a valid column if fk_col is not None: fk_col = resolve_casing(fk_col, self.db[fk].columns_dict) if fk_col not in self.db[fk].columns_dict: - raise AlterError("table '{}' has no column {}".format(fk, fk_col)) + raise AlterError(f"table '{fk}' has no column {fk_col}") else: # automatically set fk_col to first primary_key of fk table pks = sorted( @@ -3185,8 +3115,8 @@ class Table(Queryable): col_type = str not_null_sql = None if not_null_default is not None: - not_null_sql = "NOT NULL DEFAULT {}".format( - self.db.quote_default_value(not_null_default) + not_null_sql = ( + f"NOT NULL DEFAULT {self.db.quote_default_value(not_null_default)}" ) sql = "ALTER TABLE {} ADD COLUMN {} {col_type}{not_null_default};".format( quote_identifier(self.name), @@ -3206,7 +3136,7 @@ class Table(Queryable): :param ignore: Set to ``True`` to ignore the error if the table does not exist """ try: - self.db.execute("DROP TABLE {}".format(quote_identifier(self.name))) + self.db.execute(f"DROP TABLE {quote_identifier(self.name)}") except sqlite3.OperationalError: if not ignore: raise @@ -3238,16 +3168,14 @@ class Table(Queryable): return existing_tables[table] # If we get here there's no obvious candidate - raise an error raise NoObviousTable( - "No obvious foreign key table for column '{}' - tried {}".format( - column, repr(possibilities) - ) + f"No obvious foreign key table for column '{column}' - tried {possibilities!r}" ) def guess_foreign_column(self, other_table: str) -> str: pks = [c for c in self.db[other_table].columns if c.is_pk] if len(pks) != 1: raise BadPrimaryKey( - "Could not detect single primary key for table '{}'".format(other_table) + f"Could not detect single primary key for table '{other_table}'" ) else: return pks[0].name @@ -3255,8 +3183,8 @@ class Table(Queryable): def add_foreign_key( self, column: ForeignKeyColumns, - other_table: Optional[str] = None, - other_column: Optional[ForeignKeyColumns] = None, + other_table: str | None = None, + other_column: ForeignKeyColumns | None = None, ignore: bool = False, on_delete: str = "NO ACTION", on_update: str = "NO ACTION", @@ -3279,7 +3207,7 @@ class Table(Queryable): # Ensure columns exist for col in columns: if col not in self.columns_dict: - raise AlterError("No such column: {}".format(col)) + raise AlterError(f"No such column: {col}") # If other_table is not specified, attempt to guess it from the column if other_table is None: if len(columns) > 1: @@ -3312,7 +3240,7 @@ class Table(Queryable): not [c for c in self.db[other_table].columns if c.name == other_col] and other_col != "rowid" ): - raise AlterError("No such column: {}.{}".format(other_table, other_col)) + raise AlterError(f"No such column: {other_table}.{other_col}") # Check we do not already have an existing foreign key if any( fk @@ -3413,9 +3341,7 @@ class Table(Queryable): def has_counts_triggers(self) -> bool: "Does this table have triggers setup to update cached counts?" trigger_names = { - "{table}{counts_table}_{suffix}".format( - counts_table=self.db._counts_table_name, table=self.name, suffix=suffix - ) + f"{self.name}{self.db._counts_table_name}_{suffix}" for suffix in ["insert", "delete"] } return trigger_names.issubset(self.triggers_dict.keys()) @@ -3425,7 +3351,7 @@ class Table(Queryable): columns: Iterable[str], fts_version: str = "FTS5", create_triggers: bool = False, - tokenize: Optional[str] = None, + tokenize: str | None = None, replace: bool = False, ): """ @@ -3452,13 +3378,13 @@ class Table(Queryable): table_fts=quote_identifier(self.name + "_fts"), columns=", ".join(quote_identifier(c) for c in columns), fts_version=fts_version, - tokenize="\n tokenize='{}',".format(tokenize) if tokenize else "", + tokenize=f"\n tokenize='{tokenize}'," if tokenize else "", ) ) should_recreate = False - if replace and self.db["{}_fts".format(self.name)].exists(): + if replace and self.db[f"{self.name}_fts"].exists(): # Does the table need to be recreated? - fts_schema = self.db["{}_fts".format(self.name)].schema + fts_schema = self.db[f"{self.name}_fts"].schema if fts_schema != create_fts_sql: should_recreate = True expected_triggers = {self.name + suffix for suffix in ("_ai", "_ad", "_au")} @@ -3477,8 +3403,8 @@ class Table(Queryable): self.populate_fts(columns) if create_triggers: - old_cols = ", ".join("old.{}".format(quote_identifier(c)) for c in columns) - new_cols = ", ".join("new.{}".format(quote_identifier(c)) for c in columns) + old_cols = ", ".join(f"old.{quote_identifier(c)}" for c in columns) + new_cols = ", ".join(f"new.{quote_identifier(c)}" for c in columns) columns_quoted = ", ".join(quote_identifier(c) for c in columns) table = quote_identifier(self.name) table_fts = quote_identifier(self.name + "_fts") @@ -3550,7 +3476,7 @@ class Table(Queryable): with self.db.atomic(): for trigger_name in trigger_names: self.db.execute( - "DROP TRIGGER IF EXISTS {}".format(quote_identifier(trigger_name)) + f"DROP TRIGGER IF EXISTS {quote_identifier(trigger_name)}" ) return self @@ -3568,7 +3494,7 @@ class Table(Queryable): ) return self - def detect_fts(self) -> Optional[str]: + def detect_fts(self) -> str | None: "Detect if table has a corresponding FTS virtual table and return it" sql = textwrap.dedent(""" SELECT name FROM sqlite_master @@ -3583,8 +3509,8 @@ class Table(Queryable): ) """).strip() args = { - "like": "%VIRTUAL TABLE%USING FTS%content=[{}]%".format(self.name), - "like2": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), + "like": f"%VIRTUAL TABLE%USING FTS%content=[{self.name}]%", + "like2": f'%VIRTUAL TABLE%USING FTS%content="{self.name}"%', "table": self.name, } rows = self.db.execute(sql, args).fetchall() @@ -3605,11 +3531,11 @@ class Table(Queryable): def search_sql( self, - columns: Optional[Iterable[str]] = None, - order_by: Optional[str] = None, - limit: Optional[int] = None, - offset: Optional[int] = None, - where: Optional[str] = None, + columns: Iterable[str] | None = None, + order_by: str | None = None, + limit: int | None = None, + offset: int | None = None, + where: str | None = None, include_rank: bool = False, ) -> str: """ " @@ -3626,16 +3552,16 @@ class Table(Queryable): original = "original_" if self.name == "original" else "original" original_quoted = quote_identifier(original) columns_sql = "*" - columns_with_prefix_sql = "{}.*".format(original_quoted) + columns_with_prefix_sql = f"{original_quoted}.*" if columns: columns_sql = ",\n ".join(quote_identifier(c) for c in columns) columns_with_prefix_sql = ",\n ".join( - "{}.{}".format(original_quoted, quote_identifier(c)) for c in columns + f"{original_quoted}.{quote_identifier(c)}" for c in columns ) fts_table = self.detect_fts() if not fts_table: raise ValueError( - "Full-text search is not configured for table '{}'".format(self.name) + f"Full-text search is not configured for table '{self.name}'" ) fts_table_quoted = quote_identifier(fts_table) virtual_table_using = self.db.table(fts_table).virtual_table_using @@ -3658,22 +3584,20 @@ class Table(Queryable): {limit_offset} """).strip() if virtual_table_using == "FTS5": - rank_implementation = "{}.rank".format(fts_table_quoted) + rank_implementation = f"{fts_table_quoted}.rank" else: self.db.register_fts4_bm25() - rank_implementation = "rank_bm25(matchinfo({}, 'pcnalx'))".format( - fts_table_quoted - ) + rank_implementation = f"rank_bm25(matchinfo({fts_table_quoted}, 'pcnalx'))" if include_rank: columns_with_prefix_sql += ",\n " + rank_implementation + " rank" limit_offset = "" if limit is not None: - limit_offset += " limit {}".format(limit) + limit_offset += f" limit {limit}" if offset is not None: - limit_offset += " offset {}".format(offset) + limit_offset += f" offset {offset}" return sql.format( dbtable=quote_identifier(self.name), - where_clause="\n where {}".format(where) if where else "", + where_clause=f"\n where {where}" if where else "", original=original_quoted, columns=columns_sql, columns_with_prefix=columns_with_prefix_sql, @@ -3685,12 +3609,12 @@ class Table(Queryable): def search( self, q: str, - order_by: Optional[str] = None, - columns: Optional[Iterable[str]] = None, - limit: Optional[int] = None, - offset: Optional[int] = None, - where: Optional[str] = None, - where_args: Optional[Union[Iterable, dict]] = None, + order_by: str | None = None, + columns: Iterable[str] | None = None, + limit: int | None = None, + offset: int | None = None, + where: str | None = None, + where_args: Iterable | dict | None = None, include_rank: bool = False, quote: bool = False, ) -> Generator[dict, None, None]: @@ -3736,7 +3660,7 @@ class Table(Queryable): def value_or_default(self, key: str, value: Any) -> Any: return self._defaults[key] if value is DEFAULT else value - def delete(self, pk_values: Union[list, tuple, str, int, float]) -> "Table": + def delete(self, pk_values: list | tuple | str | float) -> "Table": """ Delete row matching the specified primary key. @@ -3745,7 +3669,7 @@ class Table(Queryable): if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] self.get(pk_values) - wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in self.pks] + wheres = [f"{quote_identifier(pk_name)} = ?" for pk_name in self.pks] sql = "delete from {} where {wheres}".format( quote_identifier(self.name), wheres=" and ".join(wheres) ) @@ -3755,8 +3679,8 @@ class Table(Queryable): def delete_where( self, - where: Optional[str] = None, - where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + where: str | None = None, + where_args: Sequence | dict[str, Any] | None = None, analyze: bool = False, ) -> "Table": """ @@ -3771,7 +3695,7 @@ class Table(Queryable): """ if not self.exists(): return self - sql = "delete from {}".format(quote_identifier(self.name)) + sql = f"delete from {quote_identifier(self.name)}" if where is not None: sql += " where " + where with self.db.atomic(): @@ -3782,10 +3706,10 @@ class Table(Queryable): def update( self, - pk_values: Union[list, tuple, str, int, float], - updates: Optional[dict] = None, + pk_values: list | tuple | str | float, + updates: dict | None = None, alter: bool = False, - conversions: Optional[dict] = None, + conversions: dict | None = None, ) -> "Table": """ Execute a SQL ``UPDATE`` against the specified row. @@ -3816,7 +3740,7 @@ class Table(Queryable): "{} = {}".format(quote_identifier(key), conversions.get(key, "?")) ) args.append(jsonify_if_needed(value)) - wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in pks] + wheres = [f"{quote_identifier(pk_name)} = ?" for pk_name in pks] args.extend(pk_values) sql = "update {} set {sets} where {wheres}".format( quote_identifier(self.name), @@ -3841,14 +3765,14 @@ class Table(Queryable): def convert( self, - columns: Union[str, List[str]], + columns: str | list[str], fn: Callable, - output: Optional[str] = None, - output_type: Optional[Any] = None, + output: str | None = None, + output_type: Any | None = None, drop: bool = False, multi: bool = False, - where: Optional[str] = None, - where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + where: str | None = None, + where_args: Sequence | dict[str, Any] | None = None, show_progress: bool = False, ) -> "Table": """ @@ -3905,15 +3829,11 @@ class Table(Queryable): quote_identifier(self.name), sets=", ".join( [ - "{} = {}({})".format( - quote_identifier(output or column), - fn_name, - quote_identifier(column), - ) + f"{quote_identifier(output or column)} = {fn_name}({quote_identifier(column)})" for column in columns ] ), - where=" where {}".format(where) if where is not None else "", + where=f" where {where}" if where is not None else "", ) with self.db.atomic(): self.db.execute(sql, where_args or []) @@ -3926,7 +3846,7 @@ class Table(Queryable): ): # First we execute the function pk_to_values = {} - new_column_types: Dict[str, Set[type]] = {} + new_column_types: dict[str, set[type]] = {} pks = self.pks with progressbar( @@ -3958,15 +3878,17 @@ class Table(Queryable): self.add_column(column_name, column_type) # Run the updates - with progressbar( - length=self.count, silent=not show_progress, label="2: Updating" - ) as bar: - with self.db.atomic(): - for pk, updates in pk_to_values.items(): - self.update(pk, updates) - bar.update(1) - if drop: - self.transform(drop=(column,)) + with ( + progressbar( + length=self.count, silent=not show_progress, label="2: Updating" + ) as bar, + self.db.atomic(), + ): + for pk, updates in pk_to_values.items(): + self.update(pk, updates) + bar.update(1) + if drop: + self.transform(drop=(column,)) def build_insert_queries_and_params( self, @@ -4166,9 +4088,7 @@ class Table(Queryable): ) for col in set_cols ), - wheres=" AND ".join( - "{} = ?".format(quote_identifier(pk)) for pk in pks - ), + wheres=" AND ".join(f"{quote_identifier(pk)} = ?" for pk in pks), ) queries_and_params.append( ( @@ -4201,7 +4121,7 @@ class Table(Queryable): replace, ignore, list_mode=False, - ) -> Optional[sqlite3.Cursor]: + ) -> sqlite3.Cursor | None: queries_and_params = self.build_insert_queries_and_params( extracts, chunk, @@ -4271,21 +4191,21 @@ class Table(Queryable): def insert( self, - record: Dict[str, Any], + record: dict[str, Any], pk=DEFAULT, foreign_keys=DEFAULT, - column_order: Optional[Union[List[str], Default]] = DEFAULT, - not_null: Optional[Union[Iterable[str], Default]] = DEFAULT, - defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT, - hash_id: Optional[Union[str, Default]] = DEFAULT, - hash_id_columns: Optional[Union[Iterable[str], Default]] = DEFAULT, - alter: Optional[Union[bool, Default]] = DEFAULT, - ignore: Optional[Union[bool, Default]] = DEFAULT, - replace: Optional[Union[bool, Default]] = DEFAULT, - extracts: Optional[Union[Dict[str, str], List[str], Default]] = DEFAULT, - conversions: Optional[Union[Dict[str, str], Default]] = DEFAULT, - columns: Optional[Union[Dict[str, Any], Default]] = DEFAULT, - strict: Optional[Union[bool, Default]] = DEFAULT, + column_order: list[str] | Default | None = DEFAULT, + not_null: Iterable[str] | Default | None = DEFAULT, + defaults: dict[str, Any] | Default | None = DEFAULT, + hash_id: str | Default | None = DEFAULT, + hash_id_columns: Iterable[str] | Default | None = DEFAULT, + alter: bool | Default | None = DEFAULT, + ignore: bool | Default | None = DEFAULT, + replace: bool | Default | None = DEFAULT, + extracts: dict[str, str] | list[str] | Default | None = DEFAULT, + conversions: dict[str, str] | Default | None = DEFAULT, + columns: dict[str, Any] | Default | None = DEFAULT, + strict: bool | Default | None = DEFAULT, ) -> "Table": """ Insert a single record into the table. The table will be created with a schema that matches @@ -4340,10 +4260,7 @@ class Table(Queryable): def insert_all( self, - records: Union[ - Iterable[Dict[str, Any]], - Iterable[Sequence[Any]], - ], + records: Iterable[dict[str, Any]] | Iterable[Sequence[Any]], pk=DEFAULT, foreign_keys=DEFAULT, column_order=DEFAULT, @@ -4440,7 +4357,7 @@ class Table(Queryable): # Detect if we're using list-based iteration or dict-based iteration list_mode = False - column_names: List[str] = [] + column_names: list[str] = [] # Fix up any records with square braces in the column names (only for dict mode) # We'll handle this differently for list mode @@ -4460,7 +4377,7 @@ class Table(Queryable): raise ValueError( "When using list-based iteration, the first yielded value must be a list of column name strings" ) - column_names = cast(List[str], list(first_record)) + column_names = cast(list[str], list(first_record)) all_columns = column_names num_columns = len(column_names) # Get the actual first data record @@ -4469,7 +4386,7 @@ class Table(Queryable): except StopIteration: return self # Only headers, no data if not isinstance(first_record, (list, tuple)): - raise ValueError( + raise ValueError( # noqa: TRY004 "After column names list, all subsequent records must also be lists" ) else: @@ -4479,13 +4396,11 @@ class Table(Queryable): first_record = next(records_iter) except StopIteration: return self - first_record = cast(Dict[str, Any], first_record) + first_record = cast(dict[str, Any], first_record) num_columns = len(first_record.keys()) if num_columns > SQLITE_MAX_VARS: - raise ValueError( - "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) - ) + raise ValueError(f"Rows can have a maximum of {SQLITE_MAX_VARS} columns") batch_size = ( 1 if num_columns == 0 @@ -4495,7 +4410,7 @@ class Table(Queryable): self.last_pk = None if truncate and self.exists(): with self.db.atomic(): - self.db.execute("DELETE FROM {};".format(quote_identifier(self.name))) + self.db.execute(f"DELETE FROM {quote_identifier(self.name)};") result = None for chunk in chunks(itertools.chain([first_record], records_iter), batch_size): chunk = list(chunk) @@ -4508,7 +4423,7 @@ class Table(Queryable): chunk_as_dicts = [dict(zip(column_names, row)) for row in chunk] column_types = suggest_column_types(chunk_as_dicts) else: - dict_chunk = cast(List[Dict[str, Any]], chunk) + dict_chunk = cast(list[dict[str, Any]], chunk) column_types = suggest_column_types(dict_chunk) if extracts: for col in extracts: @@ -4535,10 +4450,10 @@ class Table(Queryable): if hash_id: all_columns.insert(0, hash_id) else: - all_columns_set: Set[str] = set() - for record in cast(List[Dict[str, Any]], chunk): + all_columns_set: set[str] = set() + for record in cast(list[dict[str, Any]], chunk): all_columns_set.update(record.keys()) - all_columns = list(sorted(all_columns_set)) + all_columns = sorted(all_columns_set) if hash_id: all_columns.insert(0, hash_id) if deferred_invalid_pk_check is not None: @@ -4553,7 +4468,7 @@ class Table(Queryable): raise invalid_pk_error else: if not list_mode: - for record in cast(List[Dict[str, Any]], chunk): + for record in cast(list[dict[str, Any]], chunk): all_columns += [ column for column in record if column not in all_columns ] @@ -4592,7 +4507,7 @@ class Table(Queryable): zip(column_names, cast(Sequence[Any], first_record)) ) else: - first_record_dict = cast(Dict[str, Any], first_record) + first_record_dict = cast(dict[str, Any], first_record) if hash_id: self.last_pk = hash_record(first_record_dict, hash_id_columns) elif isinstance(pk, str): @@ -4608,7 +4523,7 @@ class Table(Queryable): # columns so we can report its rowid (and pk if not already # known). Falls back to leaving them unset if the conflict # cannot be resolved to a pk lookup (e.g. a UNIQUE column). - key_cols: Optional[List[str]] = None + key_cols: list[str] | None = None if isinstance(pk, str): key_cols = [pk] elif pk: @@ -4625,12 +4540,10 @@ class Table(Queryable): key_values = None if key_values is not None: where = " and ".join( - "{} = ?".format(quote_identifier(c)) for c in key_cols + f"{quote_identifier(c)} = ?" for c in key_cols ) existing = self.db.execute( - "select rowid from {} where {} limit 1".format( - quote_identifier(self.name), where - ), + f"select rowid from {quote_identifier(self.name)} where {where} limit 1", key_values, ).fetchone() if existing is not None: @@ -4650,7 +4563,9 @@ class Table(Queryable): rowid_pk = isinstance(pk, str) and pk.lower() in ROWID_ALIASES if (hash_id or (pk and not rowid_pk)) and self.last_rowid: # Set self.last_pk to the pk(s) for that rowid - row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] + row = next( + iter(self.rows_where("rowid = ?", [self.last_rowid])) + ) if hash_id: self.last_pk = row[hash_id] elif isinstance(pk, str): @@ -4680,7 +4595,7 @@ class Table(Queryable): for p in pk ) else: - first_record_dict = cast(Dict[str, Any], first_record) + first_record_dict = cast(dict[str, Any], first_record) if hash_id: self.last_pk = hash_record(first_record_dict, hash_id_columns) else: @@ -4738,10 +4653,7 @@ class Table(Queryable): def upsert_all( self, - records: Union[ - Iterable[Dict[str, Any]], - Iterable[Sequence[Any]], - ], + records: Iterable[dict[str, Any]] | Iterable[Sequence[Any]], pk=DEFAULT, foreign_keys=DEFAULT, column_order=DEFAULT, @@ -4779,7 +4691,7 @@ class Table(Queryable): strict=strict, ) - def add_missing_columns(self, records: Iterable[Dict[str, Any]]) -> "Table": + def add_missing_columns(self, records: Iterable[dict[str, Any]]) -> "Table": needed_columns = suggest_column_types(records) current_columns = {c.lower() for c in self.columns_dict} for col_name, col_type in needed_columns.items(): @@ -4789,17 +4701,17 @@ class Table(Queryable): def lookup( self, - lookup_values: Dict[str, Any], - extra_values: Optional[Dict[str, Any]] = None, - pk: Optional[str] = "id", - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, - extracts: Optional[Union[Dict[str, str], List[str]]] = None, - conversions: Optional[Dict[str, str]] = None, - columns: Optional[Dict[str, Any]] = None, - strict: Optional[bool] = False, + lookup_values: dict[str, Any], + extra_values: dict[str, Any] | None = None, + pk: str | None = "id", + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, + extracts: dict[str, str] | list[str] | None = None, + conversions: dict[str, str] | None = None, + columns: dict[str, Any] | None = None, + strict: bool | None = False, ): """ Create or populate a lookup table with the specified values. @@ -4825,7 +4737,7 @@ class Table(Queryable): :param strict: Boolean, apply STRICT mode if creating the table. """ if not isinstance(lookup_values, dict): - raise ValueError("lookup_values must be a dictionary") + raise ValueError("lookup_values must be a dictionary") # noqa: TRY004 if pk is None: raise ValueError("pk cannot be None") if extra_values is not None and not isinstance(extra_values, dict): @@ -4843,9 +4755,7 @@ class Table(Queryable): } not in unique_column_sets: self.create_index(lookup_values.keys(), unique=True) # IS rather than = so that null values are matched correctly - wheres = [ - "{} IS ?".format(quote_identifier(column)) for column in lookup_values - ] + wheres = [f"{quote_identifier(column)} IS ?" for column in lookup_values] rows = list( self.rows_where( " and ".join(wheres), [value for _, value in lookup_values.items()] @@ -4885,12 +4795,10 @@ class Table(Queryable): def m2m( self, other_table: Union[str, "Table"], - record_or_iterable: Optional[ - Union[Iterable[Dict[str, Any]], Dict[str, Any]] - ] = None, - pk: Optional[Union[Any, Default]] = DEFAULT, - lookup: Optional[Dict[str, Any]] = None, - m2m_table: Optional[str] = None, + record_or_iterable: Iterable[dict[str, Any]] | dict[str, Any] | None = None, + pk: Any | Default | None = DEFAULT, + lookup: dict[str, Any] | None = None, + m2m_table: str | None = None, alter: bool = False, ): """ @@ -4923,8 +4831,8 @@ class Table(Queryable): raise ValueError("Provide lookup= or record, not both") elif record_or_iterable is None: raise ValueError("Provide lookup= or record, not both") - tables = list(sorted([self.name, other_table.name])) - columns = ["{}_id".format(t) for t in tables] + tables = sorted([self.name, other_table.name]) + columns = [f"{t}_id" for t in tables] if m2m_table is not None: m2m_table_name = m2m_table else: @@ -4934,9 +4842,7 @@ class Table(Queryable): m2m_table_name = candidates[0] elif len(candidates) > 1: raise NoObviousTable( - "No single obvious m2m table for {}, {} - use m2m_table= parameter".format( - self.name, other_table.name - ) + f"No single obvious m2m table for {self.name}, {other_table.name} - use m2m_table= parameter" ) else: # If not, create a new table @@ -4947,7 +4853,7 @@ class Table(Queryable): if isinstance(record_or_iterable, Mapping): records = [record_or_iterable] else: - records = cast(List, record_or_iterable) + records = cast(list, record_or_iterable) # Ensure each record exists in other table for record in records: id = other_table.insert( @@ -4955,8 +4861,8 @@ class Table(Queryable): ).last_pk m2m_table_obj.insert( { - "{}_id".format(other_table.name): id, - "{}_id".format(self.name): our_id, + f"{other_table.name}_id": id, + f"{self.name}_id": our_id, }, replace=True, ) @@ -4964,8 +4870,8 @@ class Table(Queryable): id = other_table.lookup(lookup) m2m_table_obj.insert( { - "{}_id".format(other_table.name): id, - "{}_id".format(self.name): our_id, + f"{other_table.name}_id": id, + f"{self.name}_id": our_id, }, replace=True, ) @@ -5012,21 +4918,19 @@ class Table(Queryable): table_quoted = quote_identifier(table) column_quoted = quote_identifier(column) num_null = db.execute( - "select count(*) from {} where {} is null".format( - table_quoted, column_quoted - ) + f"select count(*) from {table_quoted} where {column_quoted} is null" ).fetchone()[0] num_blank = db.execute( - "select count(*) from {} where {} = ''".format(table_quoted, column_quoted) + f"select count(*) from {table_quoted} where {column_quoted} = ''" ).fetchone()[0] num_distinct = db.execute( - "select count(distinct {}) from {}".format(column_quoted, table_quoted) + f"select count(distinct {column_quoted}) from {table_quoted}" ).fetchone()[0] most_common_results = None least_common_results = None if num_distinct == 1: value = db.execute( - "select {} from {} limit 1".format(column_quoted, table_quoted) + f"select {column_quoted} from {table_quoted} limit 1" ).fetchone()[0] most_common_results = [(truncate(value), total_rows)] elif num_distinct != total_rows: @@ -5038,13 +4942,10 @@ class Table(Queryable): most_common_results = [ (truncate(r[0]), r[1]) for r in db.execute( - "select {}, count(*) from {} group by {} order by count(*) desc, {} limit {}".format( - column_quoted, - table_quoted, - column_quoted, - column_quoted, - common_limit, - ) + f"select {column_quoted}, count(*) " + f"from {table_quoted} group by {column_quoted} " + f"order by count(*) desc, {column_quoted} " + f"limit {common_limit}" ).fetchall() ] most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True) @@ -5056,13 +4957,10 @@ class Table(Queryable): least_common_results = [ (truncate(r[0]), r[1]) for r in db.execute( - "select {}, count(*) from {} group by {} order by count(*), {} desc limit {}".format( - column_quoted, - table_quoted, - column_quoted, - column_quoted, - common_limit, - ) + f"select {column_quoted}, count(*) " + f"from {table_quoted} group by {column_quoted} " + f"order by count(*), {column_quoted} desc " + f"limit {common_limit}" ).fetchall() ] least_common_results.sort(key=lambda p: (p[1], p[0])) @@ -5179,7 +5077,7 @@ class View(Queryable): """ try: - self.db.execute("DROP VIEW {}".format(quote_identifier(self.name))) + self.db.execute(f"DROP VIEW {quote_identifier(self.name)}") except sqlite3.OperationalError: if not ignore: raise @@ -5192,16 +5090,14 @@ def jsonify_if_needed(value: object) -> object: return json.dumps(value, default=repr, ensure_ascii=False) elif isinstance(value, (datetime.time, datetime.date, datetime.datetime)): return value.isoformat() - elif isinstance(value, datetime.timedelta): - return str(value) - elif isinstance(value, uuid.UUID): + elif isinstance(value, (datetime.timedelta, uuid.UUID)): return str(value) else: return value def resolve_extracts( - extracts: Optional[Union[Dict[str, str], List[str], Tuple[str]]], + extracts: dict[str, str] | list[str] | tuple[str] | None, ) -> dict: if extracts is None: extracts = {} diff --git a/sqlite_utils/hookspecs.py b/sqlite_utils/hookspecs.py index a746619..73d1acc 100644 --- a/sqlite_utils/hookspecs.py +++ b/sqlite_utils/hookspecs.py @@ -1,8 +1,7 @@ import sqlite3 import click -from pluggy import HookimplMarker -from pluggy import HookspecMarker +from pluggy import HookimplMarker, HookspecMarker hookspec = HookspecMarker("sqlite_utils") hookimpl = HookimplMarker("sqlite_utils") diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py index 00d0fa5..69397ba 100644 --- a/sqlite_utils/migrations.py +++ b/sqlite_utils/migrations.py @@ -1,19 +1,28 @@ -from collections.abc import Iterable -from dataclasses import dataclass import datetime -from typing import Callable, cast, TYPE_CHECKING +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol, TypeVar, cast if TYPE_CHECKING: from sqlite_utils.db import Database, Table +class _MigrationFunction(Protocol): + __name__: str + + def __call__(self, db: "Database", /) -> None: ... + + +_MigrationFunctionT = TypeVar("_MigrationFunctionT", bound=_MigrationFunction) + + class Migrations: migrations_table = "_sqlite_migrations" @dataclass class _Migration: name: str - fn: Callable + fn: _MigrationFunction transactional: bool = True @dataclass @@ -32,7 +41,7 @@ class Migrations: def __call__( self, *, name: str | None = None, transactional: bool = True - ) -> Callable: + ) -> Callable[[_MigrationFunctionT], _MigrationFunctionT]: """ :param name: The name to use for this migration - if not provided, the name of the function will be used. @@ -43,13 +52,11 @@ class Migrations: example those that execute ``VACUUM``. """ - def inner(func: Callable) -> Callable: - migration_name = name or getattr(func, "__name__") + def inner(func: _MigrationFunctionT) -> _MigrationFunctionT: + migration_name = name or func.__name__ if any(m.name == migration_name for m in self._migrations): raise ValueError( - "Migration '{}' is already registered in set '{}'".format( - migration_name, self.name - ) + f"Migration '{migration_name}' is already registered in set '{self.name}'" ) self._migrations.append( self._Migration(migration_name, func, transactional) diff --git a/sqlite_utils/plugins.py b/sqlite_utils/plugins.py index 0aff7ff..10815b4 100644 --- a/sqlite_utils/plugins.py +++ b/sqlite_utils/plugins.py @@ -1,7 +1,7 @@ -from typing import Dict, List, Union +import sys import pluggy -import sys + from . import hookspecs pm: pluggy.PluginManager = pluggy.PluginManager("sqlite_utils") @@ -17,13 +17,13 @@ def ensure_plugins_loaded() -> None: _plugins_loaded = True -def get_plugins() -> List[Dict[str, Union[str, List[str]]]]: +def get_plugins() -> list[dict[str, str | list[str]]]: ensure_plugins_loaded() - plugins: List[Dict[str, Union[str, List[str]]]] = [] + plugins: list[dict[str, str | list[str]]] = [] plugin_to_distinfo = dict(pm.list_plugin_distinfo()) for plugin in pm.get_plugins(): hookcallers = pm.get_hookcallers(plugin) or [] - plugin_info: Dict[str, Union[str, List[str]]] = { + plugin_info: dict[str, str | list[str]] = { "name": plugin.__name__, "hooks": [h.name for h in hookcallers], } diff --git a/sqlite_utils/recipes.py b/sqlite_utils/recipes.py index 55b55a4..d28a099 100644 --- a/sqlite_utils/recipes.py +++ b/sqlite_utils/recipes.py @@ -1,9 +1,9 @@ from __future__ import annotations -from typing import Callable, Optional +import json +from collections.abc import Callable from dateutil import parser -import json IGNORE: object = object() SET_NULL: object = object() @@ -13,8 +13,8 @@ def parsedate( value: str, dayfirst: bool = False, yearfirst: bool = False, - errors: Optional[object] = None, -) -> Optional[str]: + errors: object | None = None, +) -> str | None: """ Parse a date and convert it to ISO date format: yyyy-mm-dd \b @@ -44,8 +44,8 @@ def parsedatetime( value: str, dayfirst: bool = False, yearfirst: bool = False, - errors: Optional[object] = None, -) -> Optional[str]: + errors: object | None = None, +) -> str | None: """ Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS \b diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index b39b117..ed5a558 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -9,20 +9,11 @@ import itertools import json import os import sys +from collections.abc import Callable, Generator, Iterable, Iterator from typing import ( + TYPE_CHECKING, Any, BinaryIO, - Callable, - Dict, - Generator, - Iterable, - Iterator, - List, - Optional, - Set, - Tuple, - Type, - TYPE_CHECKING, TypeVar, Union, cast, @@ -33,8 +24,8 @@ import click from . import recipes if TYPE_CHECKING: - import sqlite3 # noqa: F401 - from sqlite3 import dbapi2 # noqa: F401 + import sqlite3 + from sqlite3 import dbapi2 OperationalError = dbapi2.OperationalError else: @@ -44,7 +35,7 @@ else: OperationalError = dbapi2.OperationalError except ImportError: import sqlite3 # noqa: F401 - from sqlite3 import dbapi2 # noqa: F401 + from sqlite3 import dbapi2 OperationalError = dbapi2.OperationalError @@ -61,8 +52,8 @@ SPATIALITE_PATHS = ( ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit() # Type alias for row dictionaries - values can be various SQLite-compatible types -RowValue = Union[None, int, float, str, bytes, bool, List[str]] -Row = Dict[str, RowValue] +RowValue = None | int | float | str | bytes | bool | list[str] +Row = dict[str, RowValue] T = TypeVar("T") @@ -103,7 +94,7 @@ def maximize_csv_field_size_limit() -> None: field_size_limit = int(field_size_limit / 10) -def find_spatialite() -> Optional[str]: +def find_spatialite() -> str | None: """ The ``find_spatialite()`` function searches for the `SpatiaLite `__ SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found. @@ -132,9 +123,9 @@ def find_spatialite() -> Optional[str]: def suggest_column_types( - records: Iterable[Dict[str, Any]], -) -> Dict[str, type]: - all_column_types: Dict[str, Set[type]] = {} + records: Iterable[dict[str, Any]], +) -> dict[str, type]: + all_column_types: dict[str, set[type]] = {} for record in records: for key, value in record.items(): all_column_types.setdefault(key, set()).add(type(value)) @@ -142,9 +133,9 @@ def suggest_column_types( def types_for_column_types( - all_column_types: Dict[str, Set[type]], -) -> Dict[str, type]: - column_types: Dict[str, type] = {} + all_column_types: dict[str, set[type]], +) -> dict[str, type]: + column_types: dict[str, type] = {} for key, types in all_column_types.items(): # Ignore null values if at least one other type present: if len(types) > 1: @@ -153,7 +144,7 @@ def types_for_column_types( if {None.__class__} == types: t = str elif len(types) == 1: - t = list(types)[0] + t = next(iter(types)) # But if it's a subclass of list / tuple / dict, use str # instead as we will be storing it as JSON in the table for superclass in (list, tuple, dict): @@ -190,7 +181,7 @@ def column_affinity(column_type: str) -> type: return float -def decode_base64_values(doc: Dict[str, Any]) -> Dict[str, Any]: +def decode_base64_values(doc: dict[str, Any]) -> dict[str, Any]: # Looks for '{"$base64": true..., "encoded": ...}' values and decodes them to_fix = [ k @@ -263,9 +254,9 @@ class RowError(Exception): def _extra_key_strategy( - reader: Iterable[Dict[Optional[str], object]], - ignore_extras: Optional[bool] = False, - extras_key: Optional[str] = None, + reader: Iterable[dict[str | None, object]], + ignore_extras: bool | None = False, + extras_key: str | None = None, ) -> Iterable[Row]: # Logic for handling CSV rows with more values than there are headings for row in reader: @@ -279,9 +270,7 @@ def _extra_key_strategy( yield cast(Row, row) elif not extras_key: extras = row.pop(None) - raise RowError( - "Row {} contained these extra values: {}".format(row, extras) - ) + raise RowError(f"Row {row} contained these extra values: {extras}") else: extras_value = row.pop(None) row_out = cast(Row, row) @@ -291,12 +280,12 @@ def _extra_key_strategy( def rows_from_file( fp: BinaryIO, - format: Optional[Format] = None, - dialect: Optional[Type[csv.Dialect]] = None, - encoding: Optional[str] = None, - ignore_extras: Optional[bool] = False, - extras_key: Optional[str] = None, -) -> Tuple[Iterable[Row], Format]: + format: Format | None = None, + dialect: type[csv.Dialect] | None = None, + encoding: str | None = None, + ignore_extras: bool | None = False, + extras_key: str | None = None, +) -> tuple[Iterable[Row], Format]: """ Load a sequence of dictionaries from a file-like object containing one of four different formats. @@ -363,7 +352,7 @@ def rows_from_file( ) return ( _extra_key_strategy( - cast(Iterable[Dict[Optional[str], object]], rows), + cast(Iterable[dict[str | None, object]], rows), ignore_extras, extras_key, ), @@ -379,7 +368,7 @@ def rows_from_file( raise TypeError( "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO" ) - if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): + if first_bytes.startswith((b"[", b"{")): # TODO: Detect newline-JSON return rows_from_file(buffered, format=Format.JSON) else: @@ -393,7 +382,7 @@ def rows_from_file( detected_format = Format.TSV if dialect.delimiter == "\t" else Format.CSV return ( _extra_key_strategy( - cast(Iterable[Dict[Optional[str], object]], rows), + cast(Iterable[dict[str | None, object]], rows), ignore_extras, extras_key, ), @@ -425,9 +414,9 @@ class TypeTracker: """ def __init__(self) -> None: - self.trackers: Dict[str, "ValueTracker"] = {} + self.trackers: dict[str, ValueTracker] = {} - def wrap(self, iterator: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]: + def wrap(self, iterator: Iterable[dict[str, Any]]) -> Iterable[dict[str, Any]]: """ Use this to loop through an existing iterator, tracking the column types as part of the iteration. @@ -441,7 +430,7 @@ class TypeTracker: yield row @property - def types(self) -> Dict[str, str]: + def types(self) -> dict[str, str]: """ A dictionary mapping column names to their detected types. This can be passed to the ``db[table_name].transform(types=tracker.types)`` method. @@ -450,17 +439,15 @@ class TypeTracker: class ValueTracker: - couldbe: Dict[str, Callable[[object], bool]] + couldbe: dict[str, Callable[[object], bool]] def __init__(self) -> None: self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()} @classmethod - def get_tests(cls) -> List[str]: + def get_tests(cls) -> list[str]: return [ - key.split("test_")[-1] - for key in cls.__dict__.keys() - if key.startswith("test_") + key.split("test_")[-1] for key in cls.__dict__ if key.startswith("test_") ] def test_integer(self, value: object) -> bool: @@ -492,7 +479,7 @@ class ValueTracker: def evaluate(self, value: object) -> None: if not value or not self.couldbe: return - not_these: List[str] = [] + not_these: list[str] = [] for name, test in self.couldbe.items(): if not test(value): not_these.append(name) @@ -524,14 +511,14 @@ def progressbar(*args: Iterable[T], **kwargs: Any) -> Generator[Any, None, None] def _compile_code( code: str, imports: Iterable[str], variable: str = "value" ) -> Callable[..., Any]: - globals_dict: Dict[str, Any] = {"r": recipes, "recipes": recipes} + globals_dict: dict[str, Any] = {"r": recipes, "recipes": recipes} # Handle imports first so they're available for all approaches for import_ in imports: globals_dict[import_.split(".")[0]] = __import__(import_) # If user defined a convert() function, return that try: - exec(code, globals_dict) + exec(code, globals_dict) # noqa: S102 return cast(Callable[..., object], globals_dict["convert"]) except (AttributeError, SyntaxError, NameError, KeyError, TypeError): pass @@ -542,20 +529,20 @@ def _compile_code( fn = eval(code, globals_dict) if callable(fn): return cast(Callable[..., object], fn) - except Exception: + except Exception: # noqa: BLE001, S110 pass # Try compiling their code as a function instead body_variants = [code] # If single line and no 'return', try adding the return if "\n" not in code and not code.strip().startswith("return "): - body_variants.insert(0, "return {}".format(code)) + body_variants.insert(0, f"return {code}") code_o = None for variant in body_variants: - new_code = ["def fn({}):".format(variable)] + new_code = [f"def fn({variable}):"] for line in variant.split("\n"): - new_code.append(" {}".format(line)) + new_code.append(f" {line}") try: code_o = compile("\n".join(new_code), "", "exec") break @@ -566,7 +553,7 @@ def _compile_code( if code_o is None: raise SyntaxError("Could not compile code") - exec(code_o, globals_dict) + exec(code_o, globals_dict) # noqa: S102 return cast(Callable[..., object], globals_dict["fn"]) @@ -582,7 +569,7 @@ def chunks(sequence: Iterable[T], size: int) -> Iterable[Iterable[T]]: yield itertools.chain([item], itertools.islice(iterator, size - 1)) -def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> str: +def hash_record(record: dict[str, Any], keys: Iterable[str] | None = None) -> str: """ ``record`` should be a Python dictionary. Returns a sha1 hash of the keys and values in that record. @@ -603,7 +590,7 @@ def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> :param record: Record to generate a hash for :param keys: Subset of keys to use for that hash """ - to_hash: Dict[str, Any] = record + to_hash: dict[str, Any] = record if keys is not None: to_hash = {key: record[key] for key in keys} return hashlib.sha1( @@ -613,7 +600,7 @@ def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> ).hexdigest() -def dedupe_keys(keys: Iterable[str]) -> List[str]: +def dedupe_keys(keys: Iterable[str]) -> list[str]: """ Rename duplicates in a list of column names so every name is unique, by appending ``_2``, ``_3``... to later occurrences - skipping any @@ -636,7 +623,7 @@ def dedupe_keys(keys: Iterable[str]) -> List[str]: new_key = key suffix = 2 while new_key in seen or new_key in taken: - new_key = "{}_{}".format(key, suffix) + new_key = f"{key}_{suffix}" suffix += 1 key = new_key seen.add(key) @@ -644,7 +631,7 @@ def dedupe_keys(keys: Iterable[str]) -> List[str]: return result -def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]: +def _flatten(d: dict[str, Any]) -> Generator[tuple[str, Any], None, None]: for key, value in d.items(): if isinstance(value, dict): for key2, value2 in _flatten(value): @@ -653,7 +640,7 @@ def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]: yield key, value -def flatten(row: Dict[str, Any]) -> Dict[str, Any]: +def flatten(row: dict[str, Any]) -> dict[str, Any]: """ Turn a nested dict e.g. ``{"a": {"b": 1}}`` into a flat dict: ``{"a_b": 1}`` diff --git a/tests/conftest.py b/tests/conftest.py index 728db7b..a4eb860 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ +import pytest + from sqlite_utils import Database from sqlite_utils.utils import sqlite3 -import pytest CREATE_TABLES = """ create table Gosh (c1 text, c2 text, c3 text); @@ -55,7 +56,7 @@ def close_all_databases(): for db in databases: try: db.close() - except Exception: + except sqlite3.Error: pass diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index a2ce585..a51bba6 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -1,9 +1,11 @@ -from sqlite_utils.db import Database, ColumnDetails -from sqlite_utils import cli -from click.testing import CliRunner -import pytest import sqlite3 +import pytest +from click.testing import CliRunner + +from sqlite_utils import cli +from sqlite_utils.db import ColumnDetails, Database + @pytest.fixture def db_to_analyze(fresh_db): diff --git a/tests/test_atomic.py b/tests/test_atomic.py index c3fd02f..ba16ca5 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -28,11 +28,13 @@ from sqlite_utils.utils import sqlite3 END; """, [ - "CREATE TRIGGER t_ai AFTER INSERT ON t\n" - " BEGIN\n" - " UPDATE t SET value = 'a;b' WHERE id = new.id;\n" - " INSERT INTO log VALUES ('x;y');\n" - " END;" + ( + "CREATE TRIGGER t_ai AFTER INSERT ON t\n" + " BEGIN\n" + " UPDATE t SET value = 'a;b' WHERE id = new.id;\n" + " INSERT INTO log VALUES ('x;y');\n" + " END;" + ) ], ), ), @@ -49,10 +51,9 @@ def test_atomic_commits(fresh_db): def test_atomic_rolls_back(fresh_db): - with pytest.raises(RuntimeError): - with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") - raise RuntimeError("boom") + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + raise RuntimeError("boom") assert not fresh_db["dogs"].exists() @@ -62,10 +63,9 @@ def test_nested_atomic_rolls_back_to_savepoint(fresh_db): with fresh_db.atomic(): fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}) - with pytest.raises(RuntimeError): - with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) - raise RuntimeError("boom") + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + raise RuntimeError("boom") fresh_db["dogs"].insert({"id": 3, "name": "Marnie"}) assert list(fresh_db["dogs"].rows) == [ @@ -75,20 +75,18 @@ def test_nested_atomic_rolls_back_to_savepoint(fresh_db): def test_outer_atomic_rolls_back_released_savepoint(fresh_db): - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") - with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) - raise RuntimeError("boom") + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + raise RuntimeError("boom") assert not fresh_db["dogs"].exists() def test_executescript_does_not_commit_open_atomic_block(fresh_db): - with pytest.raises(RuntimeError): - with fresh_db.atomic(): - fresh_db.executescript(""" + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db.executescript(""" CREATE TABLE dogs(id INTEGER PRIMARY KEY, name TEXT); CREATE TRIGGER dogs_ai AFTER INSERT ON dogs BEGIN @@ -97,7 +95,7 @@ def test_executescript_does_not_commit_open_atomic_block(fresh_db): -- This comment has a semicolon; INSERT INTO dogs VALUES (1, 'Cleo; the first'); """) - raise RuntimeError("boom") + raise RuntimeError("boom") assert not fresh_db["dogs"].exists() @@ -105,11 +103,10 @@ def test_executescript_does_not_commit_open_atomic_block(fresh_db): def test_transform_does_not_commit_open_atomic_block(fresh_db): fresh_db["dogs"].insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") - with pytest.raises(RuntimeError): - with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"}) - fresh_db["dogs"].transform(rename={"age": "dog_age"}) - raise RuntimeError("boom") + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"}) + fresh_db["dogs"].transform(rename={"age": "dog_age"}) + raise RuntimeError("boom") assert ( fresh_db["dogs"].schema @@ -149,10 +146,9 @@ def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db): foreign_keys={"author_id"}, ) - with pytest.raises(RuntimeError): - with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "full_name"}) - raise RuntimeError("boom") + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "full_name"}) + raise RuntimeError("boom") assert ( fresh_db["authors"].schema @@ -354,9 +350,11 @@ def test_atomic_preserves_error_from_transaction_destroying_trigger(fresh_db): # with "cannot rollback - no transaction is active" fresh_db.execute("create table t (id integer primary key, v text)") fresh_db.execute(TRIGGER_SQL) - with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): - with fresh_db.atomic(): - fresh_db.execute("insert into t (v) values ('bad')") + with ( + pytest.raises(sqlite3.IntegrityError, match="trigger says no"), + fresh_db.atomic(), + ): + fresh_db.execute("insert into t (v) values ('bad')") assert not fresh_db.conn.in_transaction @@ -367,16 +365,17 @@ def test_nested_atomic_preserves_error_from_transaction_destroying_trigger( # "no such savepoint" from ROLLBACK TO SAVEPOINT fresh_db.execute("create table t (id integer primary key, v text)") fresh_db.execute(TRIGGER_SQL) - with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): - with fresh_db.atomic(): - with fresh_db.atomic(): - fresh_db.execute("insert into t (v) values ('bad')") + with ( + pytest.raises(sqlite3.IntegrityError, match="trigger says no"), + fresh_db.atomic(), + fresh_db.atomic(), + ): + fresh_db.execute("insert into t (v) values ('bad')") assert not fresh_db.conn.in_transaction def test_atomic_preserves_error_from_insert_or_rollback(fresh_db): fresh_db["t"].insert({"id": 1}, pk="id") - with pytest.raises(sqlite3.IntegrityError): - with fresh_db.atomic(): - fresh_db.execute("insert or rollback into t (id) values (1)") + with pytest.raises(sqlite3.IntegrityError), fresh_db.atomic(): + fresh_db.execute("insert or rollback into t (id) values (1)") assert not fresh_db.conn.in_transaction diff --git a/tests/test_cli.py b/tests/test_cli.py index a2135b0..a1e072f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,14 +1,16 @@ -from sqlite_utils import cli, Database -from sqlite_utils.db import Index, ForeignKey -from click.testing import CliRunner -from pathlib import Path -import subprocess -import sqlite3 -import sys import json import os -import pytest +import sqlite3 +import subprocess +import sys import textwrap +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli +from sqlite_utils.db import ForeignKey, Index def write_json(file_path, data): @@ -21,7 +23,7 @@ def _supports_pragma_function_list(): try: db.execute("select * from pragma_function_list()") return True - except Exception: + except sqlite3.DatabaseError: return False finally: db.close() @@ -184,9 +186,9 @@ def test_output_table(db_path, options, expected): db["rows"].insert_all( [ { - "c1": "verb{}".format(i), - "c2": "noun{}".format(i), - "c3": "adjective{}".format(i), + "c1": f"verb{i}", + "c2": f"noun{i}", + "c3": f"adjective{i}", } for i in range(4) ] @@ -678,9 +680,9 @@ def test_optimize(db_path, tables): db[table].insert_all( [ { - "c1": "verb{}".format(i), - "c2": "noun{}".format(i), - "c3": "adjective{}".format(i), + "c1": f"verb{i}", + "c2": f"noun{i}", + "c3": f"adjective{i}", } for i in range(10000) ] @@ -704,9 +706,9 @@ def test_rebuild_fts_fixes_docsize_error(db_path): db = Database(db_path, recursive_triggers=False) records = [ { - "c1": "verb{}".format(i), - "c2": "noun{}".format(i), - "c3": "adjective{}".format(i), + "c1": f"verb{i}", + "c2": f"noun{i}", + "c3": f"adjective{i}", } for i in range(10000) ] @@ -1019,16 +1021,14 @@ def test_query_json_binary(db_path): "data": { "$base64": True, "encoded": ( - ( - "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH" - "8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+" - "DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I" - "/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uI" - "jnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3f" - "iCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8" - "IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984A" - "Mf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9" - ) + "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH" + "8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+" + "DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I" + "/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uI" + "jnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3f" + "iCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8" + "IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984A" + "Mf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9" ), }, } @@ -2114,11 +2114,13 @@ _common_other_schema = ( ), ( ["--rename", "name", "name2"], - 'CREATE TABLE "trees" (\n' - ' "id" INTEGER PRIMARY KEY,\n' - ' "address" TEXT,\n' - ' "species_id" INTEGER REFERENCES "species"("id")\n' - ")", + ( + 'CREATE TABLE "trees" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "address" TEXT,\n' + ' "species_id" INTEGER REFERENCES "species"("id")\n' + ")" + ), 'CREATE TABLE "species" (\n "id" INTEGER PRIMARY KEY,\n "species" TEXT\n)', ), ], @@ -2137,9 +2139,9 @@ def test_extract(db_path, args, expected_table_schema, expected_other_schema): assert result.exit_code == 0 schema = db["trees"].schema assert schema == expected_table_schema - other_schema = [t for t in db.tables if t.name not in ("trees", "Gosh", "Gosh2")][ - 0 - ].schema + other_schema = next( + t for t in db.tables if t.name not in ("trees", "Gosh", "Gosh2") + ).schema assert other_schema == expected_other_schema @@ -2431,7 +2433,7 @@ def test_long_csv_column_value(tmpdir): with open(csv_path, "w") as csv_file: long_string = "a" * 131073 csv_file.write("id,text\n") - csv_file.write("1,{}\n".format(long_string)) + csv_file.write(f"1,{long_string}\n") result = CliRunner().invoke( cli.cli, ["insert", db_path, "bigtable", csv_path, "--csv"], @@ -2457,8 +2459,8 @@ def test_import_no_headers(tmpdir, args, tsv): csv_path = str(tmpdir / "test.csv") with open(csv_path, "w") as csv_file: sep = "\t" if tsv else "," - csv_file.write("Cleo{sep}Dog{sep}5\n".format(sep=sep)) - csv_file.write("Tracy{sep}Spider{sep}7\n".format(sep=sep)) + csv_file.write(f"Cleo{sep}Dog{sep}5\n") + csv_file.write(f"Tracy{sep}Spider{sep}7\n") result = CliRunner().invoke( cli.cli, ["insert", db_path, "creatures", csv_path] + args + ["--no-detect-types"], @@ -2690,7 +2692,9 @@ def test_integer_overflow_error(tmpdir): def test_python_dash_m(): "Tool can be run using python -m sqlite_utils" result = subprocess.run( - [sys.executable, "-m", "sqlite_utils", "--help"], stdout=subprocess.PIPE + [sys.executable, "-m", "sqlite_utils", "--help"], + stdout=subprocess.PIPE, + check=False, ) assert result.returncode == 0 assert b"Commands for interacting with a SQLite database" in result.stdout @@ -2830,14 +2834,14 @@ def test_load_extension(entrypoint, should_pass, should_fail): for func in should_pass: result = CliRunner().invoke( cli.cli, - ["memory", "select {}()".format(func), "--load-extension", ext], + ["memory", f"select {func}()", "--load-extension", ext], catch_exceptions=False, ) assert result.exit_code == 0 for func in should_fail: result = CliRunner().invoke( cli.cli, - ["memory", "select {}()".format(func), "--load-extension", ext], + ["memory", f"select {func}()", "--load-extension", ext], catch_exceptions=False, ) assert result.exit_code == 1 diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py index 514f4ac..932269b 100644 --- a/tests/test_cli_bulk.py +++ b/tests/test_cli_bulk.py @@ -1,11 +1,13 @@ -from click.testing import CliRunner -from sqlite_utils import cli, Database import pathlib -import pytest import subprocess import sys import time +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli + @pytest.fixture def test_db_and_path(tmpdir): diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 6c3f5c5..65543b1 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -1,10 +1,12 @@ -from click.testing import CliRunner -from sqlite_utils import cli -import sqlite_utils import json -import textwrap import pathlib +import textwrap + import pytest +from click.testing import CliRunner + +import sqlite_utils +from sqlite_utils import cli @pytest.fixture @@ -50,7 +52,7 @@ def test_convert_code(fresh_db_and_path, code): cli.cli, ["convert", db_path, "t", "text", code], catch_exceptions=False ) assert result.exit_code == 0, result.output - value = list(db["t"].rows)[0]["text"] + value = next(iter(db["t"].rows))["text"] assert value == "Spooktober" @@ -442,7 +444,7 @@ def test_recipe_jsonsplit(tmpdir, delimiter): ) code = "r.jsonsplit(value)" if delimiter: - code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter) + code = f'recipes.jsonsplit(value, delimiter="{delimiter}")' args = ["convert", db_path, "example", "tags", code] result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0, result.output @@ -470,7 +472,7 @@ def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array): ) code = "r.jsonsplit(value)" if type: - code = "recipes.jsonsplit(value, type={})".format(type) + code = f"recipes.jsonsplit(value, type={type})" args = ["convert", db_path, "example", "records", code] result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0, result.output diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index df6f80c..eefb3fa 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -1,11 +1,13 @@ -from sqlite_utils import cli, Database -from click.testing import CliRunner import json -import pytest import subprocess import sys import time +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli + def test_insert_simple(tmpdir): json_path = str(tmpdir / "dog.json") @@ -99,7 +101,7 @@ def test_insert_with_primary_keys(db_path, tmpdir, args, expected_pks): def test_insert_multiple_with_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") - dogs = [{"id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)] + dogs = [{"id": i, "name": f"Cleo {i}", "age": i + 3} for i in range(1, 21)] with open(json_path, "w") as fp: fp.write(json.dumps(dogs)) result = CliRunner().invoke( @@ -114,7 +116,7 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir): def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") dogs = [ - {"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3} + {"breed": "mixed", "id": i, "name": f"Cleo {i}", "age": i + 3} for i in range(1, 21) ] with open(json_path, "w") as fp: @@ -140,8 +142,7 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): def test_insert_not_null_default(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") dogs = [ - {"id": i, "name": "Cleo {}".format(i), "age": i + 3, "score": 10} - for i in range(1, 21) + {"id": i, "name": f"Cleo {i}", "age": i + 3, "score": 10} for i in range(1, 21) ] with open(json_path, "w") as fp: fp.write(json.dumps(dogs)) @@ -587,7 +588,7 @@ def test_insert_streaming_batch_size_1(db_path): return tries += 1 if tries > 10: - assert False, "Expected {}, got {}".format(expected, rows) + assert False, f"Expected {expected}, got {rows}" time.sleep(tries * 0.1) try_until([{"name": "Azi"}]) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index 2ed4aaa..4fb4fb3 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -1,5 +1,6 @@ -import click import json + +import click import pytest from click.testing import CliRunner @@ -28,7 +29,7 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): fp.write(content) result = CliRunner().invoke( cli.cli, - ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + ["memory", csv_path, f"select * from {sql_from}", "--nl"], input=input, ) assert result.exit_code == 0 @@ -53,7 +54,7 @@ def test_memory_tsv(tmpdir, use_stdin): sql_from = "chickens" result = CliRunner().invoke( cli.cli, - ["memory", path, "select * from {}".format(sql_from)], + ["memory", path, f"select * from {sql_from}"], input=input, ) assert result.exit_code == 0, result.output @@ -79,7 +80,7 @@ def test_memory_json(tmpdir, use_stdin): sql_from = "chickens" result = CliRunner().invoke( cli.cli, - ["memory", path, "select * from {}".format(sql_from)], + ["memory", path, f"select * from {sql_from}"], input=input, ) assert result.exit_code == 0, result.output @@ -105,7 +106,7 @@ def test_memory_json_nl(tmpdir, use_stdin): sql_from = "chickens" result = CliRunner().invoke( cli.cli, - ["memory", path, "select * from {}".format(sql_from)], + ["memory", path, f"select * from {sql_from}"], input=input, ) assert result.exit_code == 0, result.output @@ -135,7 +136,7 @@ def test_memory_csv_encoding(tmpdir, use_stdin): CliRunner() .invoke( cli.cli, - ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + ["memory", csv_path, f"select * from {sql_from}", "--nl"], input=input, ) .exit_code diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index 0f29e36..f49ef10 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -1,7 +1,8 @@ import pathlib -from click.testing import CliRunner import pytest +from click.testing import CliRunner + import sqlite_utils import sqlite_utils.cli diff --git a/tests/test_column_affinity.py b/tests/test_column_affinity.py index fb8f340..fa23345 100644 --- a/tests/test_column_affinity.py +++ b/tests/test_column_affinity.py @@ -1,4 +1,5 @@ import pytest + from sqlite_utils.utils import column_affinity EXAMPLES = [ @@ -41,5 +42,5 @@ def test_column_affinity(column_def, expected_type): @pytest.mark.parametrize("column_def,expected_type", EXAMPLES) def test_columns_dict(fresh_db, column_def, expected_type): - fresh_db.execute("create table foo (col {})".format(column_def)) + fresh_db.execute(f"create table foo (col {column_def})") assert {"col": expected_type} == fresh_db["foo"].columns_dict diff --git a/tests/test_constructor.py b/tests/test_constructor.py index a619fba..4282969 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -1,8 +1,10 @@ +import sys + +import pytest + from sqlite_utils import Database from sqlite_utils.db import TransactionError from sqlite_utils.utils import sqlite3 -import pytest -import sys def test_recursive_triggers(): diff --git a/tests/test_convert.py b/tests/test_convert.py index ea3fd96..879267a 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -1,6 +1,7 @@ -from sqlite_utils.db import BadMultiValues import pytest +from sqlite_utils.db import BadMultiValues + @pytest.mark.parametrize( "columns,fn,expected", diff --git a/tests/test_create.py b/tests/test_create.py index d281eb4..40746bf 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,26 +1,28 @@ -from sqlite_utils.db import ( - Index, - Database, - DescIndex, - AlterError, - InvalidColumns, - NoObviousTable, - OperationalError, - ForeignKey, - Table, - View, - NoTable, - NoView, -) -from sqlite_utils.utils import hash_record, sqlite3 import collections import datetime import decimal import json import pathlib -import pytest import uuid +import pytest + +from sqlite_utils.db import ( + AlterError, + Database, + DescIndex, + ForeignKey, + Index, + InvalidColumns, + NoObviousTable, + NoTable, + NoView, + OperationalError, + Table, + View, +) +from sqlite_utils.utils import hash_record, sqlite3 + try: import pandas as pd # type: ignore except ImportError: @@ -699,7 +701,7 @@ def test_bulk_insert_more_than_999_values(fresh_db): "num_columns,should_error", ((900, False), (999, False), (1000, True)) ) def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): - record = dict([("c{}".format(i), i) for i in range(num_columns)]) + record = {f"c{i}": i for i in range(num_columns)} if should_error: with pytest.raises(ValueError): fresh_db["big"].insert(record) @@ -718,17 +720,9 @@ def test_columns_not_in_first_record_should_not_cause_batch_to_be_too_large(fres records = [ {"c0": "first record"}, # one column in first record -> batch size = 999 # fill out the batch with 99 records with enough columns to exceed THRESHOLD - *[ - dict([("c{}".format(i), j) for i in range(extra_columns)]) - for j in range(batch_size - 1) - ], + *[{f"c{i}": j for i in range(extra_columns)} for j in range(batch_size - 1)], ] - try: - fresh_db["too_many_columns"].insert_all( - records, alter=True, batch_size=batch_size - ) - except sqlite3.OperationalError: - raise + fresh_db["too_many_columns"].insert_all(records, alter=True, batch_size=batch_size) @pytest.mark.parametrize( @@ -910,7 +904,7 @@ def test_insert_list_nested_unicode(fresh_db): def test_insert_uuid(fresh_db): uuid4 = uuid.uuid4() fresh_db["test"].insert({"uuid": uuid4}) - row = list(fresh_db["test"].rows)[0] + row = next(iter(fresh_db["test"].rows)) assert {"uuid"} == row.keys() assert isinstance(row["uuid"], str) assert row["uuid"] == str(uuid4) @@ -918,16 +912,14 @@ def test_insert_uuid(fresh_db): def test_insert_memoryview(fresh_db): fresh_db["test"].insert({"data": memoryview(b"hello")}) - row = list(fresh_db["test"].rows)[0] + row = next(iter(fresh_db["test"].rows)) assert {"data"} == row.keys() assert isinstance(row["data"], bytes) assert row["data"] == b"hello" def test_insert_thousands_using_generator(fresh_db): - fresh_db["test"].insert_all( - {"i": i, "word": "word_{}".format(i)} for i in range(10000) - ) + fresh_db["test"].insert_all({"i": i, "word": f"word_{i}"} for i in range(10000)) assert [{"name": "i", "type": "INTEGER"}, {"name": "word", "type": "TEXT"}] == [ {"name": col.name, "type": col.type} for col in fresh_db["test"].columns ] @@ -938,7 +930,7 @@ def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fr # https://github.com/simonw/sqlite-utils/issues/139 with pytest.raises(Exception, match="table test has no column named extra"): fresh_db["test"].insert_all( - [{"i": i, "word": "word_{}".format(i)} for i in range(100)] + [{"i": i, "word": f"word_{i}"} for i in range(100)] + [{"i": 101, "extra": "This extra column should cause an exception"}], ) @@ -946,7 +938,7 @@ def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fr def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db): # https://github.com/simonw/sqlite-utils/issues/139 fresh_db["test"].insert_all( - [{"i": i, "word": "word_{}".format(i)} for i in range(100)] + [{"i": i, "word": f"word_{i}"} for i in range(100)] + [{"i": 101, "extra": "Should trigger ALTER"}], alter=True, ) @@ -958,7 +950,7 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): # https://github.com/simonw/sqlite-utils/issues/732 fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") - rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + rows = [{"a": f"x{i}", "b": i} for i in range(num_rows)] with pytest.raises(InvalidColumns) as ex: fresh_db["t"].insert_all(rows, pk="not_a_column") @@ -975,7 +967,7 @@ def test_insert_all_pk_not_in_records_alter_raises(fresh_db, num_rows): # known - a pk column that is in neither the table nor the records # still raises fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") - rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + rows = [{"a": f"x{i}", "b": i} for i in range(num_rows)] with pytest.raises(InvalidColumns) as ex: fresh_db["t"].insert_all(rows, pk="not_a_column", alter=True) @@ -1146,7 +1138,7 @@ def test_insert_hash_id_columns(fresh_db, use_table_factory): insert_kwargs = {} else: dogs = fresh_db["dogs"] - insert_kwargs = dict(hash_id_columns=("name", "twitter")) + insert_kwargs = {"hash_id_columns": ("name", "twitter")} id = dogs.insert( {"name": "Cleo", "twitter": "cleopaws", "age": 5}, @@ -1654,7 +1646,7 @@ def test_upsert_uses_pk_from_prior_insert_655(fresh_db): # Upsert should work without specifying pk again table.upsert({"id": 1, "name": "Alice Updated"}) assert table.count == 1 - assert list(table.rows)[0]["name"] == "Alice Updated" + assert next(iter(table.rows))["name"] == "Alice Updated" def test_upsert_all_uses_pk_from_prior_insert_655(fresh_db): diff --git a/tests/test_create_view.py b/tests/test_create_view.py index 056e246..2b70099 100644 --- a/tests/test_create_view.py +++ b/tests/test_create_view.py @@ -1,4 +1,5 @@ import pytest + from sqlite_utils.utils import OperationalError diff --git a/tests/test_default_value.py b/tests/test_default_value.py index 3724d99..2815180 100644 --- a/tests/test_default_value.py +++ b/tests/test_default_value.py @@ -31,7 +31,7 @@ EXAMPLES = [ @pytest.mark.parametrize("column_def,initial_value,expected_value", EXAMPLES) def test_quote_default_value(fresh_db, column_def, initial_value, expected_value): - fresh_db.execute("create table foo (col {})".format(column_def)) + fresh_db.execute(f"create table foo (col {column_def})") assert initial_value == fresh_db["foo"].columns[0].default_value assert expected_value == fresh_db.quote_default_value( fresh_db["foo"].columns[0].default_value diff --git a/tests/test_delete.py b/tests/test_delete.py index a2d93aa..dffb6bb 100644 --- a/tests/test_delete.py +++ b/tests/test_delete.py @@ -3,7 +3,7 @@ import sqlite_utils def test_delete_rowid_table(fresh_db): table = fresh_db["table"] - table.insert({"foo": 1}).last_pk + table.insert({"foo": 1}) rowid = table.insert({"foo": 2}).last_pk table.delete(rowid) assert [{"foo": 1}] == list(table.rows) diff --git a/tests/test_docs.py b/tests/test_docs.py index f657416..6bc06c8 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,8 +1,10 @@ -from click.testing import CliRunner -from sqlite_utils import cli, recipes -from pathlib import Path -import pytest import re +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from sqlite_utils import cli, recipes docs_path = Path(__file__).parent.parent / "docs" commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+)") @@ -34,7 +36,7 @@ def test_commands_are_documented(documented_commands, command): @pytest.mark.parametrize("command", cli.cli.commands.values()) def test_commands_have_help(command): - assert command.help, "{} is missing its help".format(command) + assert command.help, f"{command} is missing its help" def test_convert_help(): diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py index 28961d2..ad853a5 100644 --- a/tests/test_duplicate.py +++ b/tests/test_duplicate.py @@ -1,7 +1,9 @@ -from sqlite_utils.db import NoTable import datetime + import pytest +from sqlite_utils.db import NoTable + def test_duplicate(fresh_db): # Create table using native Sqlite statement: @@ -12,7 +14,7 @@ def test_duplicate(fresh_db): "bool_col" INTEGER, "datetime_col" TEXT)""") # Insert one row of mock data: - dt = datetime.datetime.now() + dt = datetime.datetime.now(datetime.timezone.utc) data = { "text_col": "Cleo", "real_col": 3.14, diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index 2f6b0db..71a8936 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -1,14 +1,14 @@ -from sqlite_utils import Database -from sqlite_utils import cli -from click.testing import CliRunner import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli def test_enable_counts_specific_table(fresh_db): foo = fresh_db["foo"] assert fresh_db.table_names() == [] for i in range(10): - foo.insert({"name": "item {}".format(i)}) + foo.insert({"name": f"item {i}"}) assert fresh_db.table_names() == ["foo"] assert foo.count == 10 # Now enable counts @@ -44,7 +44,7 @@ def test_enable_counts_specific_table(fresh_db): assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}] # Add some items to test the triggers for i in range(5): - foo.insert({"name": "item {}".format(10 + i)}) + foo.insert({"name": f"item {10 + i}"}) assert foo.count == 15 assert list(fresh_db["_counts"].rows) == [{"count": 15, "table": "foo"}] # Delete some items diff --git a/tests/test_extract.py b/tests/test_extract.py index c73ee7a..915e6e1 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1,19 +1,21 @@ -from sqlite_utils.db import InvalidColumns import itertools + import pytest +from sqlite_utils.db import InvalidColumns + @pytest.mark.parametrize("table", [None, "Species"]) @pytest.mark.parametrize("fk_column", [None, "species"]) def test_extract_single_column(fresh_db, table, fk_column): expected_table = table or "species" - expected_fk = fk_column or "{}_id".format(expected_table) + expected_fk = fk_column or f"{expected_table}_id" iter_species = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) fresh_db["tree"].insert_all( ( { "id": i, - "name": "Tree {}".format(i), + "name": f"Tree {i}", "species": next(iter_species), "end": 1, } @@ -26,13 +28,12 @@ def test_extract_single_column(fresh_db, table, fk_column): 'CREATE TABLE "tree" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT,\n' - ' "{}" INTEGER REFERENCES "{}"("id"),\n'.format(expected_fk, expected_table) + f' "{expected_fk}" INTEGER REFERENCES "{expected_table}"("id"),\n' + ' "end" INTEGER\n' + ")" ) assert fresh_db[expected_table].schema == ( - 'CREATE TABLE "{}" (\n'.format(expected_table) - + ' "id" INTEGER PRIMARY KEY,\n' + f'CREATE TABLE "{expected_table}" (\n' + ' "id" INTEGER PRIMARY KEY,\n' ' "species" TEXT\n' ")" ) @@ -57,7 +58,7 @@ def test_extract_multiple_columns_with_rename(fresh_db): ( { "id": i, - "name": "Tree {}".format(i), + "name": f"Tree {i}", "common_name": next(iter_common), "latin_name": next(iter_latin), } diff --git a/tests/test_extracts.py b/tests/test_extracts.py index 7add79a..9519b91 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -1,13 +1,14 @@ -from sqlite_utils.db import Index import pytest +from sqlite_utils.db import Index + @pytest.mark.parametrize( "kwargs,expected_table", [ - (dict(extracts={"species_id": "Species"}), "Species"), - (dict(extracts=["species_id"]), "species_id"), - (dict(extracts=("species_id",)), "species_id"), + ({"extracts": {"species_id": "Species"}}, "Species"), + ({"extracts": ["species_id"]}, "species_id"), + ({"extracts": ("species_id",)}, "species_id"), ], ) @pytest.mark.parametrize("use_table_factory", [True, False]) @@ -30,15 +31,11 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): # Should now have two tables: Trees and Species assert {expected_table, "Trees"} == set(fresh_db.table_names()) assert ( - 'CREATE TABLE "{}" (\n "id" INTEGER PRIMARY KEY,\n "value" TEXT\n)'.format( - expected_table - ) + f'CREATE TABLE "{expected_table}" (\n "id" INTEGER PRIMARY KEY,\n "value" TEXT\n)' == fresh_db[expected_table].schema ) assert ( - 'CREATE TABLE "Trees" (\n "id" INTEGER,\n "species_id" INTEGER REFERENCES "{}"("id")\n)'.format( - expected_table - ) + f'CREATE TABLE "Trees" (\n "id" INTEGER,\n "species_id" INTEGER REFERENCES "{expected_table}"("id")\n)' == fresh_db["Trees"].schema ) # Should have a foreign key reference @@ -51,7 +48,7 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): assert [ Index( seq=0, - name="idx_{}_value".format(expected_table), + name=f"idx_{expected_table}_value", unique=1, origin="c", partial=0, diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index b37d374..45f4f35 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -1,6 +1,7 @@ """Tests for compound (multi-column) foreign keys - issue #594.""" import pytest + from sqlite_utils import Database from sqlite_utils.db import AlterError, ForeignKey from sqlite_utils.utils import sqlite3 @@ -64,7 +65,7 @@ def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): fresh_db["books"].add_foreign_key("author_id", "authors", "id") fk = fresh_db["books"].foreign_keys[0] with pytest.raises(TypeError): - table, column, other_table, other_column = fk + _table, _column, _other_table, _other_column = fk with pytest.raises(TypeError): fk[0] diff --git a/tests/test_fts.py b/tests/test_fts.py index 64ec645..50c1770 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -1,7 +1,9 @@ +from unittest.mock import ANY + import pytest + from sqlite_utils import Database from sqlite_utils.utils import sqlite3 -from unittest.mock import ANY search_records = [ { @@ -103,9 +105,10 @@ def test_search_limit_offset(fresh_db): table.enable_fts(["text", "country"], fts_version="FTS4") assert len(list(table.search("are"))) == 2 assert len(list(table.search("are", limit=1))) == 1 - assert list(table.search("are", limit=1, order_by="rowid"))[0]["rowid"] == 1 + assert next(iter(table.search("are", limit=1, order_by="rowid")))["rowid"] == 1 assert ( - list(table.search("are", limit=1, offset=1, order_by="rowid"))[0]["rowid"] == 2 + next(iter(table.search("are", limit=1, offset=1, order_by="rowid")))["rowid"] + == 2 ) @@ -223,20 +226,20 @@ def test_populate_fts_escape_table_names(fresh_db): @pytest.mark.parametrize("fts_version", ("4", "5")) def test_fts_tokenize(fresh_db, fts_version): - table_name = "searchable_{}".format(fts_version) + table_name = f"searchable_{fts_version}" table = fresh_db[table_name] table.insert_all(search_records) # Test without porter stemming table.enable_fts( ["text", "country"], - fts_version="FTS{}".format(fts_version), + fts_version=f"FTS{fts_version}", ) assert [] == list(table.search("bite")) # Test WITH stemming table.disable_fts() table.enable_fts( ["text", "country"], - fts_version="FTS{}".format(fts_version), + fts_version=f"FTS{fts_version}", tokenize="porter", ) rows = list(table.search("bite", order_by="rowid")) @@ -251,10 +254,10 @@ def test_fts_tokenize(fresh_db, fts_version): def test_optimize_fts(fresh_db): for fts_version in ("4", "5"): - table_name = "searchable_{}".format(fts_version) + table_name = f"searchable_{fts_version}" table = fresh_db[table_name] table.insert_all(search_records) - table.enable_fts(["text", "country"], fts_version="FTS{}".format(fts_version)) + table.enable_fts(["text", "country"], fts_version=f"FTS{fts_version}") # You can call optimize successfully against the tables OR their _fts equivalents: for table_name in ( "searchable_4", @@ -310,12 +313,12 @@ def test_disable_fts(fresh_db, create_triggers): expected_triggers = {"searchable_ai", "searchable_ad", "searchable_au"} else: expected_triggers = set() - assert expected_triggers == set( + assert expected_triggers == { r[0] for r in fresh_db.execute( "select name from sqlite_master where type = 'trigger'" ).fetchall() - ) + } # Now run .disable_fts() and confirm it worked table.disable_fts() assert ( @@ -424,7 +427,7 @@ def test_enable_fts_replace(kwargs): db["books"].enable_fts(**kwargs, replace=True) # Check that the new configuration is correct if should_have_changed_columns: - assert db["books_fts"].columns_dict.keys() == set(["title"]) + assert db["books_fts"].columns_dict.keys() == {"title"} if "create_triggers" in kwargs: assert db["books"].triggers if "fts_version" in kwargs: @@ -741,6 +744,7 @@ def test_enable_fts_cli_on_view_errors(tmpdir): db.create_view("v", "select * from t") db.close() from click.testing import CliRunner + from sqlite_utils import cli as cli_module result = CliRunner().invoke(cli_module.cli, ["enable-fts", db_path, "v", "text"]) diff --git a/tests/test_get.py b/tests/test_get.py index 63c4a2e..3cdaed8 100644 --- a/tests/test_get.py +++ b/tests/test_get.py @@ -1,4 +1,5 @@ import pytest + from sqlite_utils.db import NotFoundError diff --git a/tests/test_gis.py b/tests/test_gis.py index f39554e..8b41d22 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -1,7 +1,8 @@ import json -import pytest +import pytest from click.testing import CliRunner + from sqlite_utils.cli import cli from sqlite_utils.db import Database from sqlite_utils.utils import find_spatialite, sqlite3 @@ -104,7 +105,7 @@ def test_query_load_extension(use_spatialite_shortcut): [ ":memory:", "select spatialite_version()", - "--load-extension={}".format(load_extension), + f"--load-extension={load_extension}", ], ) assert result.exit_code == 0, result.stdout diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index f12f865..ab652c7 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -1,5 +1,6 @@ -from hypothesis import given import hypothesis.strategies as st +from hypothesis import given + import sqlite_utils diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 88e49a8..1724d2d 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -1,10 +1,12 @@ -from sqlite_utils import cli, Database -from click.testing import CliRunner import os import pathlib -import pytest import sys +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli + @pytest.mark.parametrize("silent", (False, True)) @pytest.mark.parametrize( @@ -44,7 +46,7 @@ def test_insert_files(silent, pk_args, expected_pks): ) cols = [] for coltype in coltypes: - cols += ["-c", "{}:{}".format(coltype, coltype)] + cols += ["-c", f"{coltype}:{coltype}"] result = runner.invoke( cli.cli, ["insert-files", db_path, "files", str(tmpdir)] @@ -142,7 +144,7 @@ def test_insert_files_stdin(use_text, encoding, input, expected): ) assert result.exit_code == 0, result.stdout db = Database(db_path) - row = list(db["files"].rows)[0] + row = next(iter(db["files"].rows)) key = "content" if use_text: key = "content_text" @@ -167,5 +169,5 @@ def test_insert_files_bad_text_encoding_error(): ) assert result.exit_code == 1, result.output assert result.output.strip().startswith( - "Error: Could not read file '{}' as text".format(str(latin.resolve())) + f"Error: Could not read file '{latin.resolve()!s}' as text" ) diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 8b6765d..385c052 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,6 +1,7 @@ -from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn import pytest +from sqlite_utils.db import Database, Index, View, XIndex, XIndexColumn + def _check_supports_strict(): """Check if SQLite supports strict tables without leaking the database.""" @@ -57,8 +58,8 @@ def test_detect_fts_similar_tables(fresh_db, reverse_order): fresh_db[table2].insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" ) - assert fresh_db[table1].detect_fts() == "{}_fts".format(table1) - assert fresh_db[table2].detect_fts() == "{}_fts".format(table2) + assert fresh_db[table1].detect_fts() == f"{table1}_fts" + assert fresh_db[table2].detect_fts() == f"{table2}_fts" def test_tables(existing_db): diff --git a/tests/test_list_mode.py b/tests/test_list_mode.py index 746c9c1..646098e 100644 --- a/tests/test_list_mode.py +++ b/tests/test_list_mode.py @@ -3,6 +3,7 @@ Tests for list-based iteration in insert_all and upsert_all """ import pytest + from sqlite_utils import Database diff --git a/tests/test_lookup.py b/tests/test_lookup.py index da4f18b..c93d1ed 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -1,6 +1,7 @@ -from sqlite_utils.db import Index import pytest +from sqlite_utils.db import Index + def test_lookup_new_table(fresh_db): species = fresh_db["species"] diff --git a/tests/test_m2m.py b/tests/test_m2m.py index d613bb9..4fca918 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -1,6 +1,7 @@ -from sqlite_utils.db import ForeignKey, NoObviousTable import pytest +from sqlite_utils.db import ForeignKey, NoObviousTable + def test_insert_m2m_single(fresh_db): dogs = fresh_db["dogs"] @@ -65,8 +66,7 @@ def test_insert_m2m_iterable(fresh_db): iterable_records = ({"id": 1, "name": "Phineas"}, {"id": 2, "name": "Ferb"}) def iterable(): - for record in iterable_records: - yield record + yield from iterable_records platypuses = fresh_db["platypuses"] platypuses.insert({"id": 1, "name": "Perry"}, pk="id").m2m( diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 04185fc..3f3dfea 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -1,4 +1,5 @@ import pytest + import sqlite_utils from sqlite_utils import Migrations @@ -154,10 +155,9 @@ def test_non_transactional_migration_allows_vacuum(tmpdir): def test_apply_composes_inside_outer_transaction(migrations): db = sqlite_utils.Database(memory=True) - with pytest.raises(ZeroDivisionError): - with db.atomic(): - migrations.apply(db) - raise ZeroDivisionError + with pytest.raises(ZeroDivisionError), db.atomic(): + migrations.apply(db) + raise ZeroDivisionError # The outer transaction rolled back, taking the migrations with it assert db.table_names() == [] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index c793e32..ef202be 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,9 +1,12 @@ -from click.testing import CliRunner -import click import importlib -import pytest +import sqlite3 import sys -from sqlite_utils import cli, Database, hookimpl, plugins + +import click +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli, hookimpl, plugins def _supports_pragma_function_list(): @@ -11,7 +14,7 @@ def _supports_pragma_function_list(): try: db.execute("select * from pragma_function_list()") return True - except Exception: + except sqlite3.DatabaseError: return False finally: db.close() diff --git a/tests/test_query.py b/tests/test_query.py index 06847da..9d79755 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1,6 +1,7 @@ -import pytest import types +import pytest + from sqlite_utils.utils import sqlite3 diff --git a/tests/test_recipes.py b/tests/test_recipes.py index a7c7ef7..c6222a3 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -1,7 +1,9 @@ +import json + +import pytest + from sqlite_utils import recipes from sqlite_utils.utils import sqlite3 -import json -import pytest @pytest.fixture diff --git a/tests/test_recreate.py b/tests/test_recreate.py index bce53d5..09e237e 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -1,8 +1,10 @@ -from sqlite_utils import Database -import sqlite3 import pathlib +import sqlite3 + import pytest +from sqlite_utils import Database + def test_recreate_ignored_for_in_memory(): # None of these should raise an exception: diff --git a/tests/test_rows_from_file.py b/tests/test_rows_from_file.py index a19fed6..8c080d6 100644 --- a/tests/test_rows_from_file.py +++ b/tests/test_rows_from_file.py @@ -1,7 +1,9 @@ -from sqlite_utils.utils import rows_from_file, Format, RowError from io import BytesIO, StringIO + import pytest +from sqlite_utils.utils import Format, RowError, rows_from_file + @pytest.mark.parametrize( "input,expected_format", @@ -29,7 +31,7 @@ def test_rows_from_file_detect_format(input, expected_format): ) def test_rows_from_file_extra_fields_strategies(ignore_extras, extras_key, expected): try: - rows, format = rows_from_file( + rows, _format = rows_from_file( BytesIO(b"id,name\r\n1,Cleo,oops"), format=Format.CSV, ignore_extras=ignore_extras, diff --git a/tests/test_sniff.py b/tests/test_sniff.py index 4bbdb66..7149978 100644 --- a/tests/test_sniff.py +++ b/tests/test_sniff.py @@ -1,7 +1,9 @@ -from sqlite_utils import cli, Database -from click.testing import CliRunner import pathlib + import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli sniff_dir = pathlib.Path(__file__).parent / "sniff" diff --git a/tests/test_suggest_column_types.py b/tests/test_suggest_column_types.py index e36c58f..d4f28d3 100644 --- a/tests/test_suggest_column_types.py +++ b/tests/test_suggest_column_types.py @@ -1,5 +1,7 @@ -import pytest from collections import OrderedDict + +import pytest + from sqlite_utils.utils import suggest_column_types diff --git a/tests/test_tracer.py b/tests/test_tracer.py index d14697d..ec81f2f 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -53,16 +53,18 @@ def test_with_tracer(): assert len(collected) == 4 assert collected == [ ( - "SELECT name FROM sqlite_master\n" - " WHERE rootpage = 0\n" - " AND (\n" - " sql LIKE :like\n" - " OR sql LIKE :like2\n" - " OR (\n" - " tbl_name = :table\n" - " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" - " )\n" - " )", + ( + "SELECT name FROM sqlite_master\n" + " WHERE rootpage = 0\n" + " AND (\n" + " sql LIKE :like\n" + " OR sql LIKE :like2\n" + " OR (\n" + " tbl_name = :table\n" + " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" + " )\n" + " )" + ), { "like": "%VIRTUAL TABLE%USING FTS%content=[dogs]%", "like2": '%VIRTUAL TABLE%USING FTS%content="dogs"%', @@ -72,21 +74,23 @@ def test_with_tracer(): ("select name from sqlite_master where type = 'view'", None), ("select sql from sqlite_master where name = ?", ("dogs_fts",)), ( - 'with "original" as (\n' - " select\n" - " rowid,\n" - " *\n" - ' from "dogs"\n' - ")\n" - "select\n" - ' "original".*\n' - "from\n" - ' "original"\n' - ' join "dogs_fts" on "original".rowid = "dogs_fts".rowid\n' - "where\n" - ' "dogs_fts" match :query\n' - "order by\n" - ' "dogs_fts".rank', + ( + 'with "original" as (\n' + " select\n" + " rowid,\n" + " *\n" + ' from "dogs"\n' + ")\n" + "select\n" + ' "original".*\n' + "from\n" + ' "original"\n' + ' join "dogs_fts" on "original".rowid = "dogs_fts".rowid\n' + "where\n" + ' "dogs_fts" match :query\n' + "order by\n" + ' "dogs_fts".rank' + ), {"query": "Cleopaws"}, ), ] diff --git a/tests/test_transform.py b/tests/test_transform.py index 362f1ca..b9ee126 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,8 +1,9 @@ import sqlite3 +import pytest + from sqlite_utils.db import ForeignKey, TransactionError, TransformError from sqlite_utils.utils import OperationalError -import pytest @pytest.mark.parametrize( @@ -113,7 +114,7 @@ def test_transform_sql_table_with_primary_key( if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") - sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) + sql = dogs.transform_sql(**{**params, "tmp_suffix": "suffix"}) assert sql == expected_sql # Check that .transform() runs without exceptions: with fresh_db.tracer(tracer): @@ -186,7 +187,7 @@ def test_transform_sql_table_with_no_primary_key( if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) - sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) + sql = dogs.transform_sql(**{**params, "tmp_suffix": "suffix"}) assert sql == expected_sql # Check that .transform() runs without exceptions: with fresh_db.tracer(tracer): @@ -476,23 +477,22 @@ def test_transform_in_transaction_refuses_destructive_on_delete(fresh_db, on_del # keys inside an open transaction would fire those actions when the old # table is dropped - transform() should refuse instead fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(""" + fresh_db.executescript(f""" CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); CREATE TABLE books ( id INTEGER PRIMARY KEY, title TEXT, - author_id INTEGER REFERENCES authors(id) ON DELETE {} + author_id INTEGER REFERENCES authors(id) ON DELETE {on_delete} ); - """.format(on_delete)) + """) fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) previous_schema = fresh_db["authors"].schema - with fresh_db.atomic(): - with pytest.raises(TransactionError) as excinfo: - fresh_db["authors"].transform(rename={"name": "author_name"}) + with fresh_db.atomic(), pytest.raises(TransactionError) as excinfo: + fresh_db["authors"].transform(rename={"name": "author_name"}) message = str(excinfo.value) assert "books" in message - assert "ON DELETE {}".format(on_delete.upper()) in message + assert f"ON DELETE {on_delete.upper()}" in message # Nothing should have changed assert fresh_db["authors"].schema == previous_schema assert list(fresh_db["books"].rows) == [ @@ -518,9 +518,8 @@ def test_transform_in_transaction_refuses_self_referential_cascade(fresh_db): {"id": 2, "name": "Science Fiction", "parent_id": 1}, ] ) - with fresh_db.atomic(): - with pytest.raises(TransactionError) as excinfo: - fresh_db["categories"].transform(rename={"name": "title"}) + with fresh_db.atomic(), pytest.raises(TransactionError) as excinfo: + fresh_db["categories"].transform(rename={"name": "title"}) assert "categories" in str(excinfo.value) assert fresh_db["categories"].count == 2 @@ -715,15 +714,15 @@ def test_transform_preserves_rowids(fresh_db, table_type): # Now delete and insert a row to mix up the `rowid` sequence fresh_db["places"].delete_where("id = ?", ["2"]) fresh_db["places"].insert({"id": "4", "name": "London", "country": "UK"}) - previous_rows = list( + previous_rows = [ tuple(row) for row in fresh_db.execute("select rowid, id, name from places") - ) + ] # Transform it fresh_db["places"].transform(column_order=("country", "name")) # Should be the same - next_rows = list( + next_rows = [ tuple(row) for row in fresh_db.execute("select rowid, id, name from places") - ) + ] assert previous_rows == next_rows diff --git a/tests/test_update.py b/tests/test_update.py index 03bec11..e6ae7d8 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -43,7 +43,7 @@ def test_update_compound_pk_table(fresh_db): ) def test_update_invalid_pk(fresh_db, pk, update_pk): table = fresh_db["table"] - table.insert({"id1": 5, "id2": 3, "v": 1}, pk=pk).last_pk + table.insert({"id1": 5, "id2": 3, "v": 1}, pk=pk) with pytest.raises(NotFoundError): table.update(update_pk, {"v": 2}) diff --git a/tests/test_upsert.py b/tests/test_upsert.py index a782b26..0eaae9b 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -1,7 +1,8 @@ -from sqlite_utils.db import PrimaryKeyRequired -from sqlite_utils import Database import pytest +from sqlite_utils import Database +from sqlite_utils.db import PrimaryKeyRequired + @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_upsert(use_old_upsert): diff --git a/tests/test_utils.py b/tests/test_utils.py index 3de5e94..360a443 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,8 +1,10 @@ -from sqlite_utils import utils import csv import io + import pytest +from sqlite_utils import utils + @pytest.mark.parametrize( "input,expected,should_be_is", @@ -57,7 +59,7 @@ def test_maximize_csv_field_size_limit(): # Reset to default in case other tests have changed it csv.field_size_limit(utils.ORIGINAL_CSV_FIELD_SIZE_LIMIT) long_value = "a" * 131073 - long_csv = "id,text\n1,{}".format(long_value) + long_csv = f"id,text\n1,{long_value}" fp = io.BytesIO(long_csv.encode("utf-8")) # Using rows_from_file should error with pytest.raises(csv.Error): diff --git a/tests/test_wal.py b/tests/test_wal.py index 2ddcf54..35318f8 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -1,4 +1,5 @@ import pytest + from sqlite_utils import Database from sqlite_utils.db import TransactionError @@ -11,7 +12,7 @@ def db_path_tmpdir(tmpdir): def test_enable_disable_wal(db_path_tmpdir): - db, path, tmpdir = db_path_tmpdir + db, _path, tmpdir = db_path_tmpdir assert len(tmpdir.listdir()) == 1 assert "delete" == db.journal_mode assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()] @@ -25,12 +26,11 @@ def test_enable_disable_wal(db_path_tmpdir): def test_enable_wal_inside_transaction_raises(db_path_tmpdir): - db, path, tmpdir = db_path_tmpdir + db, _path, _tmpdir = db_path_tmpdir db["test"].insert({"id": 1}, pk="id") - with pytest.raises(TransactionError): - with db.atomic(): - db["test"].insert({"id": 2}, pk="id") - db.enable_wal() + with pytest.raises(TransactionError), db.atomic(): + db["test"].insert({"id": 2}, pk="id") + db.enable_wal() # The atomic() block must have rolled back cleanly and the # journal mode must be unchanged assert db.journal_mode == "delete" @@ -38,19 +38,18 @@ def test_enable_wal_inside_transaction_raises(db_path_tmpdir): def test_disable_wal_inside_transaction_raises(db_path_tmpdir): - db, path, tmpdir = db_path_tmpdir + db, _path, _tmpdir = db_path_tmpdir db.enable_wal() db["test"].insert({"id": 1}, pk="id") - with pytest.raises(TransactionError): - with db.atomic(): - db["test"].insert({"id": 2}, pk="id") - db.disable_wal() + with pytest.raises(TransactionError), db.atomic(): + db["test"].insert({"id": 2}, pk="id") + db.disable_wal() assert db.journal_mode == "wal" assert [r["id"] for r in db["test"].rows] == [1] def test_ensure_autocommit_on(db_path_tmpdir): - db, path, tmpdir = db_path_tmpdir + db, _path, _tmpdir = db_path_tmpdir previous_isolation_level = db.conn.isolation_level assert previous_isolation_level is not None with db.ensure_autocommit_on(): @@ -63,7 +62,7 @@ def test_ensure_autocommit_on(db_path_tmpdir): def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): # Calling enable_wal() when WAL is already enabled is a no-op, # so it is fine inside a transaction - db, path, tmpdir = db_path_tmpdir + db, _path, _tmpdir = db_path_tmpdir db.enable_wal() with db.atomic(): db["test"].insert({"id": 1}, pk="id") @@ -75,13 +74,12 @@ def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): # Setting isolation_level commits any pending transaction as a side # effect, silently breaking the caller's rollback guarantee - so # entering autocommit mode with a transaction open is an error - db, path, tmpdir = db_path_tmpdir + db, _path, _tmpdir = db_path_tmpdir db["test"].insert({"id": 1}, pk="id") db.begin() db.execute("insert into test (id) values (2)") - with pytest.raises(TransactionError): - with db.ensure_autocommit_on(): - pass + with pytest.raises(TransactionError), db.ensure_autocommit_on(): + pass # The transaction is still open and can still be rolled back assert db.conn.in_transaction db.rollback() From c621499ed1e3572989087c19a2f9d13bfff45021 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 14:53:46 -0700 Subject: [PATCH 08/33] codespell should check sqlite_utils as well It did in CI but did not in the Justfile --- Justfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Justfile b/Justfile index 5caa120..be41523 100644 --- a/Justfile +++ b/Justfile @@ -16,6 +16,7 @@ uv run ty check sqlite_utils uv run cog --check README.md docs/*.rst uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + uv run --group docs codespell sqlite_utils --ignore-words docs/codespell-ignore-words.txt # Rebuild docs with cog @cog: From a7b734946f95341f52d866321c183e3532ac0aad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 21:50:59 -0700 Subject: [PATCH 09/33] Changelog entry for 3.39.1 Refs #815 Copied from e1d55de8f84a486f5f1178f377e80d23a08a404b --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c868f4..a853aa2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _v3_39_1: + +3.39.1 (2026-07-25) +------------------- + +- Fixed a bug where ``table.delete_where()`` left the connection in an open transaction, causing deleted rows to be silently restored when the connection was closed. (:issue:`815`) + .. _v4_1_1: 4.1.1 (2026-07-12) From 6a456830ca33eb5edaa634a9b0febe5d71bea2be Mon Sep 17 00:00:00 2001 From: ikatyal2110 <134458944+ikatyal2110@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:52:04 -0500 Subject: [PATCH 10/33] Fix _decode_default_value to unescape doubled single quotes in string defaults (#811) * Fix _decode_default_value to unescape doubled single quotes in string defaults SQLite stores string defaults with single quotes doubled (e.g. DEFAULT 'O''Brien' is stored as the literal "'O''Brien'" in sqlite_master). The previous code stripped the outer quotes with value[1:-1] but never converted '' back to ', so default_values returned the raw escaped form instead of the true string value. * Test for doubled single quotes in string defaults --- sqlite_utils/db.py | 4 ++-- tests/test_introspect.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9a00123..713b110 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -5108,8 +5108,8 @@ def resolve_extracts( def _decode_default_value(value: str) -> object: if value.startswith("'") and value.endswith("'"): - # It's a string - return value[1:-1] + # It's a string; unescape doubled single quotes + return value[1:-1].replace("''", "'") if value.isdigit(): # It's an integer return int(value) diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 385c052..b7e8fc2 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -312,6 +312,7 @@ def test_table_strict(fresh_db, create_table, expected_strict): 1, 1.3, "foo", + "O'Brien", True, b"binary", ), @@ -324,6 +325,16 @@ def test_table_default_values(fresh_db, value): assert default_values == {"value": value} +def test_table_default_values_escaped_quotes(fresh_db): + # SQLite stores string defaults with single quotes doubled, so + # introspection needs to unescape them again + fresh_db.execute( + "create table t (id integer primary key, name text default 'O''Brien')" + ) + assert "default 'O''Brien'" in fresh_db["t"].schema + assert fresh_db["t"].default_values == {"name": "O'Brien"} + + def test_pks_use_primary_key_declaration_order(fresh_db): # PRIMARY KEY (a, b) declared against columns stored in order (b, a) - # pks must follow the declaration order, which is what SQLite uses to From f726ea4a65c3ce9eaff67057908ee8f2fe7f81e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Aug 2026 20:48:11 -0700 Subject: [PATCH 11/33] transform() now works for tables referenced by views (#832) Closes #831 --- docs/changelog.rst | 8 ++ docs/cli.rst | 4 + docs/python-api.rst | 11 +++ sqlite_utils/db.py | 21 +++++- tests/test_transform.py | 162 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 204 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a853aa2..4ae53a6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _unreleased: + +Unreleased +---------- + +- ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) + .. _v3_39_1: 3.39.1 (2026-07-25) @@ -18,6 +25,7 @@ - ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`) - The :ref:`CLI ` and :ref:`Python API ` documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) + .. _v4_1: 4.1 (2026-07-11) diff --git a/docs/cli.rst b/docs/cli.rst index 2e506dd..cf241aa 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2288,7 +2288,11 @@ If you want to see the SQL that will be executed to make the change without actu INSERT INTO "roadside_attractions_new_4033a60276b9" ("longitude", "latitude", "id", "name") SELECT "longitude", "latitude", "pk", "name" FROM "roadside_attractions"; DROP TABLE "roadside_attractions"; + PRAGMA legacy_alter_table=ON; ALTER TABLE "roadside_attractions_new_4033a60276b9" RENAME TO "roadside_attractions"; + PRAGMA legacy_alter_table=OFF; + +Tables that are referenced by views can be transformed - the view definitions are left unchanged, see :ref:`python_api_transform_views` for details. .. note:: In Python: :ref:`table.transform() ` CLI reference: :ref:`sqlite-utils transform ` diff --git a/docs/python-api.rst b/docs/python-api.rst index 43b734d..53a47dd 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1986,6 +1986,17 @@ A bare column name drops any foreign key that column participates in, including Renaming a column with ``rename=`` updates any foreign keys that use it, and dropping a column with ``drop=`` also drops any foreign keys it participates in - for a compound foreign key this removes the whole constraint. +.. _python_api_transform_views: + +Tables referenced by views +-------------------------- + +Tables that are referenced by views can be safely transformed - the view definitions are left byte-for-byte unchanged, and views continue to read from the live table even when ``keep_table=`` is used to keep a copy of the original around. + +A view that references a column which the transform renamed or dropped will remain defined but will raise a ``no such column`` error when it is next queried. This is inherent to SQLite views, whose SQL is stored as text - if you rename or drop columns that a view depends on you should update that view definition yourself. + +To achieve this, the SQL produced by ``transform_sql()`` turns on ``PRAGMA legacy_alter_table`` for its ``ALTER TABLE ... RENAME TO`` statements, then restores the pragma to the value it had when the SQL was generated - without this, SQLite would attempt to rewrite references to the renamed table in every view definition, which fails when a view references the table that was just dropped. + .. _python_api_transform_sql: Custom transformations with .transform_sql() diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 713b110..5307931 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2824,17 +2824,34 @@ class Table(Queryable): new_cols=", ".join(quote_identifier(col) for col in new_cols), ) sqls.append(copy_sql) - # Drop (or keep) the old table + # Drop (or keep) the old table, then rename the new one into place. + # Since SQLite 3.25 ALTER TABLE ... RENAME TO rewrites references to + # the renamed table in every view definition, which fails if a view + # references the table that was just dropped - and with keep_table= + # would silently repoint views at the backup table. These renames are + # an implementation detail of transform(), so use legacy_alter_table + # to leave view definitions untouched, restoring the connection's + # current value afterwards. + legacy_alter_table_row = self.db.execute("PRAGMA legacy_alter_table").fetchone() + legacy_alter_table_was_on = bool( + legacy_alter_table_row and legacy_alter_table_row[0] + ) if keep_table: + sqls.append("PRAGMA legacy_alter_table=ON;") sqls.append( f"ALTER TABLE {quote_identifier(self.name)} RENAME TO {quote_identifier(keep_table)};" ) else: sqls.append(f"DROP TABLE {quote_identifier(self.name)};") - # Rename the new one + sqls.append("PRAGMA legacy_alter_table=ON;") sqls.append( f"ALTER TABLE {quote_identifier(new_table_name)} RENAME TO {quote_identifier(self.name)};" ) + sqls.append( + "PRAGMA legacy_alter_table={};".format( + "ON" if legacy_alter_table_was_on else "OFF" + ) + ) # Re-add existing indexes for index in self.indexes: if index.origin != "pk": diff --git a/tests/test_transform.py b/tests/test_transform.py index b9ee126..7874421 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -16,7 +16,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change column type @@ -26,7 +28,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Rename a column @@ -36,7 +40,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Drop a column @@ -46,7 +52,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n SELECT "rowid", "id", "name" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Convert type AND rename column @@ -56,7 +64,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" INTEGER\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change primary key @@ -66,7 +76,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT PRIMARY KEY\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change primary key to a compound pk @@ -76,7 +88,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT,\n PRIMARY KEY ("age", "name")\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Remove primary key, creating a rowid table @@ -86,7 +100,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Keeping the table @@ -95,8 +111,10 @@ from sqlite_utils.utils import OperationalError [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n SELECT "rowid", "id", "name" FROM "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs" RENAME TO "kept_table";', 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), ], @@ -139,7 +157,9 @@ def test_transform_sql_table_with_primary_key( 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change column type @@ -149,7 +169,9 @@ def test_transform_sql_table_with_primary_key( 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" INTEGER\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Rename a column @@ -159,7 +181,9 @@ def test_transform_sql_table_with_primary_key( 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "dog_age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Make ID a primary key @@ -169,7 +193,9 @@ def test_transform_sql_table_with_primary_key( 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), ], @@ -903,3 +929,139 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db): "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation." in str(excinfo.value) ) + + +def test_transform_preserves_view(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/831 + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + view_sql_before = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + dogs.transform(rename={"name": "title"}) + view_sql_after = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert view_sql_before == view_sql_after + + +@pytest.mark.parametrize( + "transform_params", + [ + {"types": {"name": int}}, + {"pk": "name"}, + {"add_foreign_keys": [("other_id", "other", "id")]}, + {"drop_foreign_keys": ["other_id"]}, + ], +) +def test_transform_variants_preserve_view(fresh_db, transform_params): + # Covers retyping, changing primary key and foreign key modifications, + # with a view whose columns are untouched by the transform + fresh_db["other"].insert({"id": 1}, pk="id") + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "other_id": 1}, pk="id") + if "drop_foreign_keys" in transform_params: + dogs.transform(add_foreign_keys=[("other_id", "other", "id")]) + fresh_db.execute("create view dogs_view as select id, name from dogs") + view_sql_before = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + dogs.transform(**transform_params) + view_sql_after = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert view_sql_before == view_sql_after + assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}] + + +def test_transform_view_referencing_renamed_column(fresh_db): + # The view survives but querying it raises "no such column" - inherent + # to SQLite views, whose SQL is stored as text + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + dogs.transform(rename={"name": "title"}) + with pytest.raises(OperationalError, match="no such column"): + fresh_db.execute("select * from dogs_view") + + +def test_transform_view_on_view(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view v1 as select id, name from dogs") + fresh_db.execute("create view v2 as select name from v1") + sqls_before = fresh_db.execute( + "select sql from sqlite_master where type = 'view' order by name" + ).fetchall() + dogs.transform(types={"id": str}) + sqls_after = fresh_db.execute( + "select sql from sqlite_master where type = 'view' order by name" + ).fetchall() + assert sqls_before == sqls_after + assert list(fresh_db["v2"].rows) == [{"name": "Cleo"}] + + +def test_transform_keep_table_does_not_repoint_view(fresh_db): + # Without legacy_alter_table the ALTER TABLE dogs RENAME TO dogs_backup + # step would rewrite the view to select from "dogs_backup" + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + dogs.transform(types={"name": str}, keep_table="dogs_backup") + view_sql = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert "dogs_backup" not in view_sql + # View reads from the live table, not the frozen backup + dogs.insert({"id": 2, "name": "Pancakes"}) + assert list(fresh_db["dogs_view"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Pancakes"}, + ] + + +def test_transform_sql_standalone_statements_work_with_view(fresh_db): + # The documented "run these statements yourself" workflow should be + # standalone-correct, so the pragmas must come from transform_sql() + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + sqls = dogs.transform_sql(types={"name": str}, tmp_suffix="suffix") + assert sqls[-3] == "PRAGMA legacy_alter_table=ON;" + assert sqls[-2] == 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";' + assert sqls[-1] == "PRAGMA legacy_alter_table=OFF;" + for sql in sqls: + fresh_db.execute(sql) + assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}] + + +def test_transform_with_view_in_open_transaction(fresh_db): + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + with fresh_db.conn: + fresh_db.execute("insert into dogs (id, name) values (2, 'Pancakes')") + dogs.transform(rename={"name": "title"}) + assert dogs.columns_dict == {"id": int, "title": str} + view_sql = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert view_sql == "CREATE VIEW dogs_view as select id, name from dogs" + + +def test_transform_restores_legacy_alter_table_setting(fresh_db): + if sqlite3.sqlite_version_info < (3, 25, 0): + pytest.skip("legacy_alter_table pragma requires SQLite 3.25 or higher") + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + # Default is OFF, reset to OFF afterwards + dogs.transform(types={"name": str}) + assert fresh_db.execute("PRAGMA legacy_alter_table").fetchone()[0] == 0 + # If the connection has it ON, it should be restored to ON + fresh_db.execute("PRAGMA legacy_alter_table=ON") + sqls = dogs.transform_sql(types={"name": str}, tmp_suffix="suffix") + assert sqls[-1] == "PRAGMA legacy_alter_table=ON;" + dogs.transform(types={"name": str}) + assert fresh_db.execute("PRAGMA legacy_alter_table").fetchone()[0] == 1 From 3db0c57a3bc9d8468db430ebe0ffd0da213fdda3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Aug 2026 21:58:00 -0700 Subject: [PATCH 12/33] table.checks, table.column_checks, table.table_checks, closes #834 Refs #762 --- docs/changelog.rst | 1 + docs/python-api.rst | 37 ++ sqlite_utils/create_table_parser.py | 551 ++++++++++++++++++++++++++++ sqlite_utils/db.py | 22 ++ tests/test_create_table_parser.py | 138 +++++++ tests/test_introspect.py | 27 +- 6 files changed, 775 insertions(+), 1 deletion(-) create mode 100644 sqlite_utils/create_table_parser.py create mode 100644 tests/test_create_table_parser.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 4ae53a6..77f644c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Unreleased ---------- +- New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) - ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) .. _v3_39_1: diff --git a/docs/python-api.rst b/docs/python-api.rst index 53a47dd..93f8a3d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2480,6 +2480,43 @@ Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly False +.. _python_api_introspection_checks: + +.checks +------- + +The ``.checks`` property returns the column-level and table-level ``CHECK`` constraints defined on a table, as a list of ``Check`` objects. Each object has ``check`` (the expression inside ``CHECK (...)``), ``name``, ``column`` and ``options`` attributes. ``column`` is an empty string for a table-level check. ``options`` contains a list of values only when a column check consists entirely of ``column IN (literal, ...)``. The original constraint fragment is available as ``sql``; ``start`` and ``end`` are its offsets within ``table.schema``. + +.. code-block:: python + + >>> db["scores"].checks + [Check(check='score > 0', name='positive', column='score', options=None), + Check(check='score <= maximum', name='within_maximum', column='', options=None)] + +.. _python_api_introspection_column_checks: + +.column_checks +-------------- + +The ``.column_checks`` property returns the column-level checks grouped by column name: + +.. code-block:: python + + >>> db["scores"].column_checks + {'score': [Check(check='score > 0', name='positive', column='score', options=None)]} + +.. _python_api_introspection_table_checks: + +.table_checks +------------- + +The ``.table_checks`` property returns only the table-level checks: + +.. code-block:: python + + >>> db["scores"].table_checks + [Check(check='score <= maximum', name='within_maximum', column='', options=None)] + .. _python_api_introspection_foreign_keys: .foreign_keys diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py new file mode 100644 index 0000000..9c0a9aa --- /dev/null +++ b/sqlite_utils/create_table_parser.py @@ -0,0 +1,551 @@ +"""Helpers for parsing CHECK constraints from SQLite CREATE TABLE SQL. + +SQLite does not expose CHECK constraints through a pragma, so preserving them +across a table rebuild requires reading ``sqlite_schema.sql``. This module is +deliberately small, but it uses a real lexer: strings, quoted identifiers and +comments are opaque, every token retains its source span and malformed input is +reported instead of being silently under-parsed. +""" + +import re +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class Check: + check: str + name: str = "" + column: str = "" + options: list[Any] | None = None + # Source details are excluded from equality and repr so callers can compare + # semantic constraints while still having the original SQL available for + # diagnostics or future lossless edits. + sql: str = field(default="", compare=False, repr=False) + start: int = field(default=-1, compare=False, repr=False) + end: int = field(default=-1, compare=False, repr=False) + + +class ParseError(ValueError): + pass + + +@dataclass(frozen=True) +class _Token: + kind: str + text: str + start: int + end: int + + def is_keyword(self, keyword: str) -> bool: + return self.kind == "word" and self.text.upper() == keyword + + +_PUNCTUATION = frozenset("(),.;+-*/%<>=!~|&?:") +_TRIVIA = frozenset(("whitespace", "comment")) +_TABLE_CONSTRAINT_KEYWORDS = frozenset(("PRIMARY", "UNIQUE", "CHECK", "FOREIGN")) +_OTHER_COLUMN_CONSTRAINT_KEYWORDS = frozenset( + ("PRIMARY", "UNIQUE", "REFERENCES", "DEFAULT", "NOT", "COLLATE", "GENERATED") +) +_SQLITE_KEYWORDS = frozenset( + ( + "ABORT", + "ACTION", + "ADD", + "AFTER", + "ALL", + "ALTER", + "ANALYZE", + "AND", + "AS", + "ASC", + "ATTACH", + "AUTOINCREMENT", + "BEFORE", + "BEGIN", + "BETWEEN", + "BY", + "CASCADE", + "CASE", + "CAST", + "CHECK", + "COLLATE", + "COLUMN", + "COMMIT", + "CONFLICT", + "CONSTRAINT", + "CREATE", + "CROSS", + "CURRENT_DATE", + "CURRENT_TIME", + "CURRENT_TIMESTAMP", + "DATABASE", + "DEFAULT", + "DEFERRABLE", + "DEFERRED", + "DELETE", + "DESC", + "DETACH", + "DISTINCT", + "DO", + "DROP", + "EACH", + "ELSE", + "END", + "ESCAPE", + "EXCEPT", + "EXCLUDE", + "EXCLUSIVE", + "EXISTS", + "EXPLAIN", + "FAIL", + "FALSE", + "FILTER", + "FIRST", + "FOLLOWING", + "FOR", + "FOREIGN", + "FROM", + "FULL", + "GENERATED", + "GLOB", + "GROUP", + "GROUPS", + "HAVING", + "IF", + "IGNORE", + "IMMEDIATE", + "IN", + "INDEX", + "INDEXED", + "INITIALLY", + "INNER", + "INSERT", + "INSTEAD", + "INTERSECT", + "INTO", + "IS", + "ISNULL", + "JOIN", + "KEY", + "LAST", + "LEFT", + "LIKE", + "LIMIT", + "MATCH", + "MATERIALIZED", + "NATURAL", + "NO", + "NOT", + "NOTHING", + "NOTNULL", + "NULL", + "NULLS", + "OF", + "OFFSET", + "ON", + "OR", + "ORDER", + "OTHERS", + "OUTER", + "OVER", + "PARTITION", + "PLAN", + "PRAGMA", + "PRECEDING", + "PRIMARY", + "QUERY", + "RAISE", + "RANGE", + "RECURSIVE", + "REFERENCES", + "REGEXP", + "REINDEX", + "RELEASE", + "RENAME", + "REPLACE", + "RESTRICT", + "RETURNING", + "RIGHT", + "ROLLBACK", + "ROW", + "ROWS", + "SAVEPOINT", + "SELECT", + "SET", + "STRICT", + "TABLE", + "TEMP", + "TEMPORARY", + "THEN", + "TIES", + "TO", + "TRANSACTION", + "TRIGGER", + "TRUE", + "UNBOUNDED", + "UNION", + "UNIQUE", + "UPDATE", + "USING", + "VACUUM", + "VALUES", + "VIEW", + "VIRTUAL", + "WHEN", + "WHERE", + "WINDOW", + "WITH", + "WITHOUT", + ) +) +_INTEGER_RE = re.compile(r"[+-]?(?:0[xX][0-9a-fA-F]+|[0-9]+)\Z") +_FLOAT_RE = re.compile( + r"[+-]?(?:(?:[0-9]+\.[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?|" + r"[0-9]+[eE][+-]?[0-9]+)\Z" +) + + +def _lex(sql: str) -> list[_Token]: + tokens: list[_Token] = [] + i = 0 + while i < len(sql): + start = i + char = sql[i] + if char.isspace(): + i += 1 + while i < len(sql) and sql[i].isspace(): + i += 1 + tokens.append(_Token("whitespace", sql[start:i], start, i)) + continue + if sql.startswith("--", i): + newline = sql.find("\n", i + 2) + i = len(sql) if newline == -1 else newline + 1 + tokens.append(_Token("comment", sql[start:i], start, i)) + continue + if sql.startswith("/*", i): + end = sql.find("*/", i + 2) + if end == -1: + raise ParseError("Unterminated SQL comment") + i = end + 2 + tokens.append(_Token("comment", sql[start:i], start, i)) + continue + if char in ("'", '"', "`"): + quote = char + i += 1 + while i < len(sql): + if sql[i] == quote: + if i + 1 < len(sql) and sql[i + 1] == quote: + i += 2 + continue + i += 1 + break + i += 1 + else: + raise ParseError(f"Unterminated {quote} quoted token") + kind = "string" if quote == "'" else "identifier" + tokens.append(_Token(kind, sql[start:i], start, i)) + continue + if char == "[": + end = sql.find("]", i + 1) + if end == -1: + raise ParseError("Unterminated [ quoted identifier") + i = end + 1 + tokens.append(_Token("identifier", sql[start:i], start, i)) + continue + if char in _PUNCTUATION: + i += 1 + tokens.append(_Token("punct", char, start, i)) + continue + # SQLite accepts any character >= U+0080 in a bare identifier. More + # generally, consume until a lexical delimiter rather than relying on + # Python's narrower definition of an alphanumeric character. + i += 1 + while i < len(sql): + if sql[i].isspace() or sql[i] in _PUNCTUATION or sql[i] in "'\"`[": + break + i += 1 + tokens.append(_Token("word", sql[start:i], start, i)) + return tokens + + +def _meaningful(tokens: list[_Token]) -> list[_Token]: + return [token for token in tokens if token.kind not in _TRIVIA] + + +def _unquote(token: str) -> str: + if len(token) >= 2 and token[0] in ("'", '"', "`") and token[-1] == token[0]: + return token[1:-1].replace(token[0] * 2, token[0]) + if len(token) >= 2 and token[0] == "[" and token[-1] == "]": + return token[1:-1] + return token + + +def _matching_paren(tokens: list[_Token], open_index: int) -> int: + if tokens[open_index].text != "(": + raise ParseError("Expected an opening parenthesis") + depth = 0 + for index in range(open_index, len(tokens)): + if tokens[index].text == "(": + depth += 1 + elif tokens[index].text == ")": + depth -= 1 + if depth == 0: + return index + raise ParseError("Unbalanced parentheses") + + +def _split_spans(sql: str, tokens: list[_Token]) -> list[tuple[str, int, int]]: + if not tokens: + return [] + items: list[tuple[str, int, int]] = [] + depth = 0 + start = tokens[0].start + for token in tokens: + if token.text == "(": + depth += 1 + elif token.text == ")": + depth -= 1 + if depth < 0: + raise ParseError("Unbalanced parentheses") + elif token.text == "," and depth == 0: + raw = sql[start : token.start] + item = raw.strip() + if item: + item_start = start + len(raw) - len(raw.lstrip()) + items.append((item, item_start, item_start + len(item))) + start = token.end + if depth: + raise ParseError("Unbalanced parentheses") + raw = sql[start : tokens[-1].end] + item = raw.strip() + if item: + item_start = start + len(raw) - len(raw.lstrip()) + items.append((item, item_start, item_start + len(item))) + return items + + +def _split_ranges(sql: str, tokens: list[_Token]) -> list[str]: + return [item for item, _, _ in _split_spans(sql, tokens)] + + +def _strip_outer_parens(tokens: list[_Token]) -> list[_Token]: + while tokens and tokens[0].text == "(": + close = _matching_paren(tokens, 0) + if close != len(tokens) - 1: + break + tokens = tokens[1:-1] + return tokens + + +_NO_LITERAL = object() + + +def _literal_value(text: str) -> Any: + tokens = _meaningful(_lex(text)) + if len(tokens) == 1 and tokens[0].kind == "string": + return _unquote(tokens[0].text) + raw = "".join(token.text for token in tokens) + if raw.upper() == "NULL": + return None + if raw.upper() == "TRUE": + return True + if raw.upper() == "FALSE": + return False + if _INTEGER_RE.fullmatch(raw): + try: + return ( + int(raw, 16) if raw.lower().lstrip("+-").startswith("0x") else int(raw) + ) + except ValueError: + return _NO_LITERAL + if _FLOAT_RE.fullmatch(raw): + try: + return float(raw) + except ValueError: + return _NO_LITERAL + return _NO_LITERAL + + +def _ascii_fold(identifier: str) -> str: + return identifier.translate( + str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") + ) + + +def _parse_options(expression: str, column: str) -> list[Any] | None: + tokens = _strip_outer_parens(_meaningful(_lex(expression))) + if len(tokens) < 4: + return None + lhs = tokens[0] + if lhs.kind not in ("word", "identifier"): + return None + if column and _ascii_fold(_unquote(lhs.text)) != _ascii_fold(column): + return None + if not tokens[1].is_keyword("IN") or tokens[2].text != "(": + return None + close = _matching_paren(tokens, 2) + if close != len(tokens) - 1: + return None + inner = expression[tokens[2].end : tokens[close].start] + inner_tokens = _lex(inner) + if not _meaningful(inner_tokens): + return [] + values = [] + for item in _split_ranges(inner, inner_tokens): + value = _literal_value(item) + if value is _NO_LITERAL: + return None + values.append(value) + return values + + +def _check_after( + item: str, + tokens: list[_Token], + check_index: int, + name: str, + column: str, + constraint_start: int, + base_offset: int, +) -> tuple[Check, int]: + if check_index + 1 >= len(tokens) or tokens[check_index + 1].text != "(": + raise ParseError("CHECK must be followed by a parenthesized expression") + close = _matching_paren(tokens, check_index + 1) + expression = item[tokens[check_index + 1].end : tokens[close].start].strip() + source_start = tokens[constraint_start].start + source_end = tokens[close].end + return ( + Check( + expression, + name=name, + column=column, + options=_parse_options(expression, column), + sql=item[source_start:source_end], + start=base_offset + source_start, + end=base_offset + source_end, + ), + close + 1, + ) + + +def _column_checks( + item: str, tokens: list[_Token], column: str, base_offset: int +) -> list[Check]: + checks: list[Check] = [] + pending_name = "" + pending_start: int | None = None + index = 1 + while index < len(tokens): + token = tokens[index] + if token.text == "(": + index = _matching_paren(tokens, index) + 1 + continue + if token.is_keyword("CONSTRAINT"): + if index + 1 >= len(tokens): + raise ParseError("CONSTRAINT is missing its name") + pending_name = _unquote(tokens[index + 1].text) + pending_start = index + index += 2 + continue + if token.is_keyword("CHECK"): + check, index = _check_after( + item, + tokens, + index, + pending_name, + column, + pending_start if pending_start is not None else index, + base_offset, + ) + checks.append(check) + pending_name = "" + pending_start = None + continue + if ( + token.kind == "word" + and token.text.upper() in _OTHER_COLUMN_CONSTRAINT_KEYWORDS + ): + pending_name = "" + pending_start = None + index += 1 + return checks + + +def parse_checks(create_sql: str) -> list[Check]: + """Return CHECK constraints from a valid SQLite CREATE TABLE statement.""" + all_tokens = _lex(create_sql) + tokens = _meaningful(all_tokens) + if not tokens or not tokens[0].is_keyword("CREATE"): + raise ParseError("Expected CREATE TABLE") + index = 1 + if index < len(tokens) and ( + tokens[index].is_keyword("TEMP") or tokens[index].is_keyword("TEMPORARY") + ): + index += 1 + if index < len(tokens) and tokens[index].is_keyword("VIRTUAL"): + return [] + if index >= len(tokens) or not tokens[index].is_keyword("TABLE"): + raise ParseError("Expected CREATE TABLE") + index += 1 + if ( + index + 2 < len(tokens) + and tokens[index].is_keyword("IF") + and tokens[index + 1].is_keyword("NOT") + and tokens[index + 2].is_keyword("EXISTS") + ): + index += 3 + if index >= len(tokens): + raise ParseError("CREATE TABLE is missing its table name") + index += 1 + if index + 1 < len(tokens) and tokens[index].text == ".": + index += 2 + if index < len(tokens) and tokens[index].is_keyword("AS"): + return [] + if index >= len(tokens) or tokens[index].text != "(": + raise ParseError("CREATE TABLE is missing its column list") + close = _matching_paren(tokens, index) + trailing = tokens[close + 1 :] + allowed_trailing = {"STRICT", "WITHOUT", "ROWID", ",", ";"} + if any(token.text.upper() not in allowed_trailing for token in trailing): + raise ParseError("Unexpected SQL after CREATE TABLE column list") + + body_start = tokens[index].end + body_end = tokens[close].start + body = create_sql[body_start:body_end] + body_tokens = _lex(body) + checks: list[Check] = [] + for item, item_start, _ in _split_spans(body, body_tokens): + item_tokens = _meaningful(_lex(item)) + if not item_tokens: + continue + item_index = 0 + constraint_name = "" + if item_tokens[item_index].is_keyword("CONSTRAINT"): + if len(item_tokens) < 2: + raise ParseError("CONSTRAINT is missing its name") + constraint_name = _unquote(item_tokens[1].text) + item_index = 2 + head = item_tokens[item_index] if item_index < len(item_tokens) else None + if ( + head + and head.kind == "word" + and head.text.upper() in _TABLE_CONSTRAINT_KEYWORDS + ): + if head.is_keyword("CHECK"): + check, _ = _check_after( + item, + item_tokens, + item_index, + constraint_name, + "", + 0, + body_start + item_start, + ) + checks.append(check) + continue + column = _unquote(item_tokens[0].text) + checks.extend( + _column_checks(item, item_tokens, column, body_start + item_start) + ) + return checks diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 5307931..d85ca41 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -27,6 +27,7 @@ from typing_extensions import Self from sqlite_utils.plugins import ensure_plugins_loaded, pm +from .create_table_parser import Check, parse_checks from .utils import ( OperationalError, chunks, @@ -2196,6 +2197,27 @@ class Table(Queryable): "Does this table use ``rowid`` for its primary key (no other primary keys are specified)?" return not any(column for column in self.columns if column.is_pk) + @property + def checks(self) -> list[Check]: + "List of column-level and table-level CHECK constraints on this table." + if not self.exists() or self.virtual_table_using is not None: + return [] + return parse_checks(self.schema) + + @property + def column_checks(self) -> dict[str, list[Check]]: + "CHECK constraints grouped by the column on which they are defined." + checks: dict[str, list[Check]] = {} + for check in self.checks: + if check.column: + checks.setdefault(check.column, []).append(check) + return checks + + @property + def table_checks(self) -> list[Check]: + "Table-level CHECK constraints on this table." + return [check for check in self.checks if not check.column] + def get(self, pk_values: list | tuple | str | int) -> dict: """ Return row (as dictionary) for the specified primary key. diff --git a/tests/test_create_table_parser.py b/tests/test_create_table_parser.py new file mode 100644 index 0000000..e7ab3e8 --- /dev/null +++ b/tests/test_create_table_parser.py @@ -0,0 +1,138 @@ +import sqlite3 + +import hypothesis.strategies as st +import pytest +from hypothesis import given + +from sqlite_utils.create_table_parser import Check, ParseError, parse_checks + + +def test_parse_column_and_table_checks(): + sql = """ + CREATE TABLE people ( + age INTEGER CONSTRAINT positive CHECK (age > 0), + status TEXT CHECK(status IN ('active', 'inactive')), + CONSTRAINT adult CHECK(age >= 18) + ) + """ + assert parse_checks(sql) == [ + Check("age > 0", name="positive", column="age"), + Check( + "status IN ('active', 'inactive')", + column="status", + options=["active", "inactive"], + ), + Check("age >= 18", name="adult"), + ] + checks = parse_checks(sql) + assert checks[0].sql == "CONSTRAINT positive CHECK (age > 0)" + assert sql[checks[0].start : checks[0].end] == checks[0].sql + assert checks[1].sql == "CHECK(status IN ('active', 'inactive'))" + assert sql[checks[2].start : checks[2].end] == checks[2].sql + + +def test_comments_are_trivia_not_constraints(): + sql = """ + CREATE /* fake CHECK (nope), ( */ TABLE t ( + a INTEGER /* CHECK (a < 0), phantom */, + b INTEGER CHECK /* between keyword and expression */ (b > 0), + /* CHECK (also_fake) */ CONSTRAINT upper CHECK(b < 10) + ) + """ + sqlite3.connect(":memory:").execute(sql) + assert parse_checks(sql) == [ + Check("b > 0", column="b"), + Check("b < 10", name="upper"), + ] + + +@pytest.mark.parametrize( + "expression,expected", + [ + ("value IN ('one', 'two')", ["one", "two"]), + ("((value IN ('one', 'two')))", ["one", "two"]), + ("value NOT IN ('one', 'two')", None), + ("value IN ('one', 'two') OR enabled", None), + ("other IN ('one', 'two')", None), + ("value IN (lower('one'), 'two')", None), + ('value IN ("other")', None), + ], +) +def test_options_only_for_exact_literal_in_check(expression, expected): + sql = f"CREATE TABLE t(value TEXT CHECK({expression}), enabled INTEGER, other TEXT)" + sqlite3.connect(":memory:").execute(sql) + assert parse_checks(sql)[0].options == expected + + +@pytest.mark.parametrize("column", ["💩x", "e\u0301"]) +def test_unquoted_unicode_identifiers(column): + sql = f"CREATE TABLE t({column} INTEGER CHECK({column} > 0))" + sqlite3.connect(":memory:").execute(sql) + assert parse_checks(sql) == [Check(f"{column} > 0", column=column)] + + +@pytest.mark.parametrize( + "sql", + [ + "SELECT CHECK(x > 0)", + "CREATE TABLE t(x INTEGER CHECK(x > 0)", + "CREATE TABLE t(x TEXT CHECK(x != 'unterminated))", + "CREATE TABLE t(x INTEGER /* unterminated)", + ], +) +def test_invalid_sql_raises_parse_error(sql): + with pytest.raises(ParseError): + parse_checks(sql) + + +def test_virtual_table_has_no_checks(): + assert ( + parse_checks("CREATE /* comment */ VIRTUAL TABLE search USING fts5(text)") == [] + ) + + +comment_or_space = st.sampled_from( + [ + " ", + "\n ", + "/* comment with , ( ) and CHECK(fake) */", + "-- comment with , ( ) and CHECK(fake)\n", + ] +) + + +@given(gaps=st.lists(comment_or_space, min_size=5, max_size=5)) +def test_comments_and_whitespace_can_separate_check_tokens(gaps): + sql = ( + f"CREATE{gaps[0]}TABLE{gaps[1]}t{gaps[2]}(" + f"value INTEGER CHECK{gaps[3]}(value{gaps[4]}> 0))" + ) + connection = sqlite3.connect(":memory:") + connection.execute(sql) + stored_sql = connection.execute( + "select sql from sqlite_schema where name = 't'" + ).fetchone()[0] + assert parse_checks(stored_sql) == [Check(f"value{gaps[4]}> 0", column="value")] + + +safe_string_text = st.text( + alphabet=st.characters( + blacklist_categories=("Cc", "Cs"), + blacklist_characters=("'",), + ), + max_size=40, +) + + +@given(value=safe_string_text) +def test_check_like_text_inside_strings_is_opaque(value): + sql = f"CREATE TABLE t(value TEXT CHECK(value != '{value}'))" + connection = sqlite3.connect(":memory:") + connection.execute(sql) + stored_sql = connection.execute( + "select sql from sqlite_schema where name = 't'" + ).fetchone()[0] + checks = parse_checks(stored_sql) + assert len(checks) == 1 + assert checks[0].column == "value" + assert checks[0].check == f"value != '{value}'" diff --git a/tests/test_introspect.py b/tests/test_introspect.py index b7e8fc2..2a8d579 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,6 +1,6 @@ import pytest -from sqlite_utils.db import Database, Index, View, XIndex, XIndexColumn +from sqlite_utils.db import Check, Database, Index, View, XIndex, XIndexColumn def _check_supports_strict(): @@ -177,6 +177,31 @@ def test_pks(fresh_db, pk, expected): assert expected == fresh_db["foo"].pks +def test_checks(fresh_db): + fresh_db.execute(""" + CREATE TABLE scores ( + score INTEGER CONSTRAINT positive CHECK(score > 0), + maximum INTEGER, + CONSTRAINT within_maximum CHECK(score <= maximum) + ) + """) + scores = fresh_db["scores"] + expected_column = Check("score > 0", name="positive", column="score") + expected_table = Check("score <= maximum", name="within_maximum") + assert scores.checks == [expected_column, expected_table] + assert scores.column_checks == {"score": [expected_column]} + assert scores.table_checks == [expected_table] + assert scores.checks[0].sql == "CONSTRAINT positive CHECK(score > 0)" + + +def test_checks_nonexistent_and_virtual_tables(fresh_db): + assert fresh_db["does_not_exist"].checks == [] + fresh_db["searchable"].insert({"text": "hello"}).enable_fts( + ["text"], fts_version="FTS5" + ) + assert fresh_db["searchable_fts"].checks == [] + + def test_triggers_and_triggers_dict(fresh_db): assert [] == fresh_db.triggers authors = fresh_db["authors"] From 2303b80aef69abe418636dc980c5350958d55411 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Aug 2026 21:59:57 -0700 Subject: [PATCH 13/33] .transform() preserves check constraints, refs #762 --- docs/changelog.rst | 1 + docs/python-api.rst | 9 ++ sqlite_utils/create_table_parser.py | 83 +++++++++++++++ sqlite_utils/db.py | 65 +++++++++++- tests/test_mutator_transactions.py | 154 ++++++++++++++++++++++++++++ tests/test_transform.py | 137 ++++++++++++++++++++++++- 6 files changed, 447 insertions(+), 2 deletions(-) create mode 100644 tests/test_mutator_transactions.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 77f644c..938e9db 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) +- ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in them without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) - ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) .. _v3_39_1: diff --git a/docs/python-api.rst b/docs/python-api.rst index 93f8a3d..75900b6 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1986,6 +1986,15 @@ A bare column name drops any foreign key that column participates in, including Renaming a column with ``rename=`` updates any foreign keys that use it, and dropping a column with ``drop=`` also drops any foreign keys it participates in - for a compound foreign key this removes the whole constraint. +.. _python_api_transform_check_constraints: + +CHECK constraints +----------------- + +``.transform()`` preserves both column-level and table-level ``CHECK`` constraints. If a column is renamed, references to that column in the check expression are renamed too. + +A column-level check is removed if its owning column is dropped. Dropping a column referenced by any remaining check raises ``TransformError`` instead of creating an invalid or unexpectedly weakened schema. + .. _python_api_transform_views: Tables referenced by views diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py index 9c0a9aa..082c2c8 100644 --- a/sqlite_utils/create_table_parser.py +++ b/sqlite_utils/create_table_parser.py @@ -549,3 +549,86 @@ def parse_checks(create_sql: str) -> list[Check]: _column_checks(item, item_tokens, column, body_start + item_start) ) return checks + + +def _is_identifier_token(tokens: list[_Token], index: int) -> bool: + token = tokens[index] + if index + 1 < len(tokens) and tokens[index + 1].text in ("(", "."): + return False + if index and ( + tokens[index - 1].is_keyword("COLLATE") or tokens[index - 1].is_keyword("AS") + ): + return False + if token.kind == "identifier": + return True + if token.kind != "word" or token.text.upper() in _SQLITE_KEYWORDS: + return False + return True + + +def check_references_identifier(expression: str, identifier: str) -> bool: + tokens = _meaningful(_lex(expression)) + folded = _ascii_fold(identifier) + return any( + _is_identifier_token(tokens, index) + and _ascii_fold(_unquote(token.text)) == folded + for index, token in enumerate(tokens) + ) + + +def check_expression_ends_in_line_comment(expression: str) -> bool: + """Return True if appended SQL would be swallowed by a ``--`` comment.""" + tokens = _lex(expression) + if not tokens: + return False + final = tokens[-1] + return ( + final.kind == "comment" + and final.text.startswith("--") + and not final.text.endswith(("\n", "\r")) + ) + + +def _valid_bare_identifier(identifier: str) -> bool: + if not identifier or identifier.upper() in _SQLITE_KEYWORDS: + return False + first = identifier[0] + if not (first == "_" or first.isalpha() or ord(first) >= 0x80): + return False + return all( + char == "_" or char == "$" or char.isalnum() or ord(char) >= 0x80 + for char in identifier[1:] + ) + + +def _quote_replacement(original: str, replacement: str) -> str: + if original.startswith('"'): + return '"{}"'.format(replacement.replace('"', '""')) + if original.startswith("`"): + return "`{}`".format(replacement.replace("`", "``")) + if original.startswith("[") and "]" not in replacement: + return f"[{replacement}]" + if _valid_bare_identifier(replacement): + return replacement + return '"{}"'.format(replacement.replace('"', '""')) + + +def rewrite_check_expression(expression: str, rename: dict[str, str]) -> str: + """Rewrite column identifiers in a CHECK expression, preserving trivia.""" + if not rename: + return expression + tokens = _lex(expression) + meaningful = _meaningful(tokens) + replacements = {_ascii_fold(key): value for key, value in rename.items()} + edits: list[tuple[int, int, str]] = [] + for index, token in enumerate(meaningful): + if not _is_identifier_token(meaningful, index): + continue + replacement = replacements.get(_ascii_fold(_unquote(token.text))) + if replacement is not None: + edits.append( + (token.start, token.end, _quote_replacement(token.text, replacement)) + ) + for start, end, replacement in reversed(edits): + expression = expression[:start] + replacement + expression[end:] + return expression diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d85ca41..949f574 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -27,7 +27,14 @@ from typing_extensions import Self from sqlite_utils.plugins import ensure_plugins_loaded, pm -from .create_table_parser import Check, parse_checks +from .create_table_parser import ( + Check, + ParseError, + check_expression_ends_in_line_comment, + check_references_identifier, + parse_checks, + rewrite_check_expression, +) from .utils import ( OperationalError, chunks, @@ -86,6 +93,12 @@ def quote_identifier(identifier: str) -> str: return '"{}"'.format(identifier.replace('"', '""')) +def _check_constraint_sql(check: Check) -> str: + prefix = f"CONSTRAINT {quote_identifier(check.name)} " if check.name else "" + newline = "\n" if check_expression_ends_in_line_comment(check.check) else "" + return f"{prefix}CHECK ({check.check}{newline})" + + _IDENTIFIER_CASEFOLD = str.maketrans( "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" ) @@ -1380,6 +1393,7 @@ class Database: extracts: dict[str, str] | list[str] | None = None, if_not_exists: bool = False, strict: bool = False, + _checks: Iterable[Check] | None = None, ) -> str: """ Returns the SQL ``CREATE TABLE`` statement for creating the specified table. @@ -1425,6 +1439,19 @@ class Database: defaults = {resolve_casing(n, columns): v for n, v in (defaults or {}).items()} if column_order is not None: column_order = [resolve_casing(c, columns) for c in column_order] + checks = list(_checks or ()) + checks_by_column: dict[str, list[Check]] = {} + table_checks: list[Check] = [] + for check in checks: + if check.column: + column = resolve_casing(check.column, columns) + if column not in columns: + raise AlterError( + f"No such column for CHECK constraint: {check.column}" + ) + checks_by_column.setdefault(column, []).append(check) + else: + table_checks.append(check) if not columns: raise ValueError("Tables must have at least one column") if not all(n in columns for n in not_null): @@ -1481,6 +1508,10 @@ class Database: column_extras.append( f"REFERENCES {quote_identifier(fk.other_table)}({quote_identifier(cast(str, fk.other_column))}){_fk_actions_sql(fk)}" ) + column_extras.extend( + _check_constraint_sql(check) + for check in checks_by_column.get(column_name, ()) + ) column_type_str = COLUMN_TYPE_MAPPING[column_type] # Special case for strict tables to map FLOAT to REAL # Refs https://github.com/simonw/sqlite-utils/issues/644 @@ -1520,6 +1551,9 @@ class Database: actions=_fk_actions_sql(fk), ) ) + column_defs.extend( + f" {_check_constraint_sql(check)}" for check in table_checks + ) columns_sql = ",\n".join(column_defs) sql = """CREATE TABLE {if_not_exists}{table} ( {columns_sql}{extra_pk} @@ -2677,6 +2711,34 @@ class Table(Queryable): if column_order is not None: column_order = [resolve_casing(c, existing_columns) for c in column_order] + try: + existing_checks = self.checks + except ParseError as ex: + raise TransformError( + f"Could not parse CHECK constraints for table {self.name!r}: {ex}" + ) from ex + create_table_checks: list[Check] = [] + for check in existing_checks: + owner = ( + resolve_casing(check.column, existing_columns) if check.column else "" + ) + # A column-level constraint disappears with the column that owns it. + if owner and owner in drop: + continue + for dropped_column in drop: + if check_references_identifier(check.check, dropped_column): + raise TransformError( + f"Cannot drop column {dropped_column!r}: it is used by " + f"CHECK constraint {check.name or check.check!r}" + ) + create_table_checks.append( + Check( + rewrite_check_expression(check.check, rename), + name=check.name, + column=rename.get(owner) or owner, + ) + ) + create_table_foreign_keys: list[ForeignKeyIndicator] = [] if foreign_keys is not None: @@ -2826,6 +2888,7 @@ class Table(Queryable): foreign_keys=create_table_foreign_keys, column_order=column_order, strict=self.strict if strict is None else strict, + _checks=create_table_checks, ).strip() ) diff --git a/tests/test_mutator_transactions.py b/tests/test_mutator_transactions.py new file mode 100644 index 0000000..37ae1b6 --- /dev/null +++ b/tests/test_mutator_transactions.py @@ -0,0 +1,154 @@ +import pytest + +from sqlite_utils import Database +from sqlite_utils.utils import sqlite3 + +BASELINE_ROWS = [(1, "one"), (2, "two")] + + +def insert(table): + table.insert({"id": 3, "value": "three"}, pk="id") + + +def insert_all(table): + table.insert_all( + [ + {"id": 3, "value": "three"}, + {"id": 4, "value": "four"}, + ], + pk="id", + batch_size=1, + ) + + +def upsert(table): + table.upsert({"id": 2, "value": "TWO"}, pk="id") + + +def upsert_all(table): + table.upsert_all( + [ + {"id": 2, "value": "TWO"}, + {"id": 3, "value": "three"}, + ], + pk="id", + batch_size=1, + ) + + +def update(table): + table.update(2, {"value": "TWO"}) + + +def delete(table): + table.delete(2) + + +def delete_where(table): + table.delete_where("id > ?", [1]) + + +MUTATOR_CASES = ( + pytest.param( + insert, + [(1, "one"), (2, "two"), (3, "three")], + id="insert", + ), + pytest.param( + insert_all, + [(1, "one"), (2, "two"), (3, "three"), (4, "four")], + id="insert_all", + ), + pytest.param( + upsert, + [(1, "one"), (2, "TWO")], + id="upsert", + ), + pytest.param( + upsert_all, + [(1, "one"), (2, "TWO"), (3, "three")], + id="upsert_all", + ), + pytest.param( + update, + [(1, "one"), (2, "TWO")], + id="update", + ), + pytest.param(delete, [(1, "one")], id="delete"), + pytest.param(delete_where, [(1, "one")], id="delete_where"), +) + + +class RollbackTest(Exception): + pass + + +def seed_database(path): + conn = sqlite3.connect(str(path)) + try: + conn.execute("create table items (id integer primary key, value text)") + conn.executemany("insert into items values (?, ?)", BASELINE_ROWS) + conn.commit() + finally: + conn.close() + return Database(path) + + +def current_rows(db): + return db.conn.execute("select id, value from items order by id").fetchall() + + +def persisted_rows(path): + conn = sqlite3.connect(str(path)) + try: + return conn.execute("select id, value from items order by id").fetchall() + finally: + conn.close() + + +@pytest.mark.parametrize("mutate,expected_rows", MUTATOR_CASES) +def test_mutator_commits_by_default(tmp_path, mutate, expected_rows): + path = tmp_path / "default.db" + db = seed_database(path) + + assert not db.conn.in_transaction + mutate(db["items"]) + assert current_rows(db) == expected_rows + assert not db.conn.in_transaction + + db.close() + assert persisted_rows(path) == expected_rows + + +@pytest.mark.parametrize("mutate,expected_rows", MUTATOR_CASES) +def test_mutator_commits_with_outer_atomic(tmp_path, mutate, expected_rows): + path = tmp_path / "atomic.db" + db = seed_database(path) + + with db.atomic(): + assert db.conn.in_transaction + mutate(db["items"]) + assert current_rows(db) == expected_rows + assert db.conn.in_transaction + + assert current_rows(db) == expected_rows + assert not db.conn.in_transaction + db.close() + assert persisted_rows(path) == expected_rows + + +@pytest.mark.parametrize("mutate,expected_rows", MUTATOR_CASES) +def test_mutator_rolls_back_outer_atomic(tmp_path, mutate, expected_rows): + path = tmp_path / "rollback.db" + db = seed_database(path) + + with pytest.raises(RollbackTest), db.atomic(): + mutate(db["items"]) + assert current_rows(db) == expected_rows + assert db.conn.in_transaction + raise RollbackTest + + assert current_rows(db) == BASELINE_ROWS + assert not db.conn.in_transaction + db.close() + assert persisted_rows(path) == BASELINE_ROWS diff --git a/tests/test_transform.py b/tests/test_transform.py index 7874421..44fec99 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -2,7 +2,7 @@ import sqlite3 import pytest -from sqlite_utils.db import ForeignKey, TransactionError, TransformError +from sqlite_utils.db import Check, ForeignKey, TransactionError, TransformError from sqlite_utils.utils import OperationalError @@ -1065,3 +1065,138 @@ def test_transform_restores_legacy_alter_table_setting(fresh_db): assert sqls[-1] == "PRAGMA legacy_alter_table=ON;" dogs.transform(types={"name": str}) assert fresh_db.execute("PRAGMA legacy_alter_table").fetchone()[0] == 1 + + +def test_transform_preserves_check_constraints(fresh_db): + fresh_db.execute(""" + CREATE TABLE scores ( + id INTEGER PRIMARY KEY, + score INTEGER CONSTRAINT valid_score CHECK(score BETWEEN 0 AND 100), + CONSTRAINT nonzero_id CHECK(id != 0) + ) + """) + scores = fresh_db["scores"] + scores.insert({"id": 1, "score": 50}) + scores.transform() + assert scores.checks == [ + Check("score BETWEEN 0 AND 100", name="valid_score", column="score"), + Check("id != 0", name="nonzero_id"), + ] + with pytest.raises(sqlite3.IntegrityError, match="CHECK constraint failed"): + scores.insert({"id": 2, "score": 101}) + + +def test_transform_preserves_check_ending_in_line_comment(fresh_db): + fresh_db.execute(""" + CREATE TABLE inventory ( + quantity INTEGER, + CHECK ( + quantity >= 0 -- Quantity cannot be negative + ) + ) + """) + inventory = fresh_db["inventory"] + inventory.transform(types={"quantity": float}) + assert inventory.checks == [Check("quantity >= 0 -- Quantity cannot be negative")] + with pytest.raises(sqlite3.IntegrityError, match="CHECK constraint failed"): + inventory.insert({"quantity": -1}) + + +def test_transform_renames_columns_inside_check_constraints(fresh_db): + fresh_db.execute(""" + CREATE TABLE inventory ( + quantity INTEGER CONSTRAINT positive + CHECK(quantity > 0 AND 'quantity' != ''), + maximum INTEGER, + CONSTRAINT within_maximum CHECK(quantity <= maximum) + ) + """) + inventory = fresh_db["inventory"] + inventory.insert({"quantity": 2, "maximum": 3}) + inventory.transform(rename={"quantity": "amount"}) + assert inventory.checks == [ + Check( + "amount > 0 AND 'quantity' != ''", + name="positive", + column="amount", + ), + Check("amount <= maximum", name="within_maximum"), + ] + with pytest.raises(sqlite3.IntegrityError, match="CHECK constraint failed"): + inventory.insert({"amount": 4, "maximum": 3}) + + +def test_transform_check_rewrite_preserves_functions_and_quotes(fresh_db): + fresh_db.execute(""" + CREATE TABLE items ( + length TEXT, + "old name" TEXT, + CHECK(length("old name") > 0 AND length != '') + ) + """) + items = fresh_db["items"] + items.insert({"length": "label", "old name": "hello"}) + items.transform(rename={"length": "description", "old name": "new name"}) + assert items.checks == [Check("length(\"new name\") > 0 AND description != ''")] + + +def test_transform_check_rewrite_quotes_keyword_column(fresh_db): + fresh_db.execute("CREATE TABLE t(old_name TEXT CHECK(old_name != ''))") + fresh_db["t"].insert({"old_name": "value"}) + fresh_db["t"].transform(rename={"old_name": "select"}) + assert fresh_db["t"].checks == [Check("\"select\" != ''", column="select")] + + +def test_transform_check_rewrite_does_not_rename_collations_or_cast_types(fresh_db): + fresh_db.execute(""" + CREATE TABLE t ( + nocase TEXT, + kind TEXT, + other TEXT, + CHECK( + other COLLATE nocase != '' + AND CAST(other AS kind) != '' + AND nocase != '' + AND kind != '' + ) + ) + """) + fresh_db["t"].insert({"nocase": "n", "kind": "k", "other": "o"}) + fresh_db["t"].transform(rename={"nocase": "label", "kind": "category"}) + check = fresh_db["t"].checks[0].check + assert "COLLATE nocase" in check + assert "AS kind" in check + assert "AND label != ''" in check + assert "AND category != ''" in check + + +def test_transform_drops_check_owned_by_dropped_column(fresh_db): + fresh_db.execute(""" + CREATE TABLE t ( + id INTEGER, + obsolete INTEGER CHECK(obsolete > 0), + CHECK(id > 0) + ) + """) + fresh_db["t"].insert({"id": 1, "obsolete": 2}) + fresh_db["t"].transform(drop={"obsolete"}) + assert fresh_db["t"].checks == [Check("id > 0")] + + +def test_transform_refuses_to_drop_column_used_by_remaining_check(fresh_db): + fresh_db.execute(""" + CREATE TABLE ranges ( + minimum INTEGER, + maximum INTEGER, + CHECK(minimum <= maximum) + ) + """) + ranges = fresh_db["ranges"] + ranges.insert({"minimum": 1, "maximum": 2}) + schema_before = ranges.schema + with pytest.raises( + TransformError, + match="Cannot drop column 'maximum'.*CHECK constraint", + ): + ranges.transform(drop={"maximum"}) + assert ranges.schema == schema_before From b432e686ca3d3449df74899393ab05cb4f40f6d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Aug 2026 22:02:28 -0700 Subject: [PATCH 14/33] Use sqlite_master not sqlite_schema for older SQLite compatibility --- tests/test_create_table_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_create_table_parser.py b/tests/test_create_table_parser.py index e7ab3e8..3b878f3 100644 --- a/tests/test_create_table_parser.py +++ b/tests/test_create_table_parser.py @@ -110,7 +110,7 @@ def test_comments_and_whitespace_can_separate_check_tokens(gaps): connection = sqlite3.connect(":memory:") connection.execute(sql) stored_sql = connection.execute( - "select sql from sqlite_schema where name = 't'" + "select sql from sqlite_master where name = 't'" ).fetchone()[0] assert parse_checks(stored_sql) == [Check(f"value{gaps[4]}> 0", column="value")] @@ -130,7 +130,7 @@ def test_check_like_text_inside_strings_is_opaque(value): connection = sqlite3.connect(":memory:") connection.execute(sql) stored_sql = connection.execute( - "select sql from sqlite_schema where name = 't'" + "select sql from sqlite_master where name = 't'" ).fetchone()[0] checks = parse_checks(stored_sql) assert len(checks) == 1 From b37b8cf8c83515a022e7ec0b1599004bb2f7eb55 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Aug 2026 22:37:13 -0700 Subject: [PATCH 15/33] Preserve column before/after comments through .transform() Refs #762 The before comment comes before the column definition - the after comment is anything after it but before its trailing comma. --- docs/changelog.rst | 3 +- docs/python-api.rst | 2 ++ sqlite_utils/create_table_parser.py | 56 +++++++++++++++++++++++++---- sqlite_utils/db.py | 52 ++++++++++++++++++++++----- tests/test_create_table_parser.py | 28 ++++++++++++++- tests/test_transform.py | 48 +++++++++++++++++++++++++ 6 files changed, 171 insertions(+), 18 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 938e9db..2950dd4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,8 @@ Unreleased ---------- - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) -- ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in them without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) +- ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) +- ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) - ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) .. _v3_39_1: diff --git a/docs/python-api.rst b/docs/python-api.rst index 75900b6..90c5165 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1995,6 +1995,8 @@ CHECK constraints A column-level check is removed if its owning column is dropped. Dropping a column referenced by any remaining check raises ``TransformError`` instead of creating an invalid or unexpectedly weakened schema. +Comments immediately before or after a column definition are preserved too. They move with that column if it is renamed or reordered, and are removed if the column is dropped. A comment between two column definitions is treated as belonging to the following column. + .. _python_api_transform_views: Tables referenced by views diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py index 082c2c8..2d891ae 100644 --- a/sqlite_utils/create_table_parser.py +++ b/sqlite_utils/create_table_parser.py @@ -26,6 +26,12 @@ class Check: end: int = field(default=-1, compare=False, repr=False) +@dataclass(frozen=True) +class ColumnComments: + before: str = "" + after: str = "" + + class ParseError(ValueError): pass @@ -472,8 +478,7 @@ def _column_checks( return checks -def parse_checks(create_sql: str) -> list[Check]: - """Return CHECK constraints from a valid SQLite CREATE TABLE statement.""" +def _table_body(create_sql: str) -> tuple[str, int] | None: all_tokens = _lex(create_sql) tokens = _meaningful(all_tokens) if not tokens or not tokens[0].is_keyword("CREATE"): @@ -484,7 +489,7 @@ def parse_checks(create_sql: str) -> list[Check]: ): index += 1 if index < len(tokens) and tokens[index].is_keyword("VIRTUAL"): - return [] + return None if index >= len(tokens) or not tokens[index].is_keyword("TABLE"): raise ParseError("Expected CREATE TABLE") index += 1 @@ -501,7 +506,7 @@ def parse_checks(create_sql: str) -> list[Check]: if index + 1 < len(tokens) and tokens[index].text == ".": index += 2 if index < len(tokens) and tokens[index].is_keyword("AS"): - return [] + return None if index >= len(tokens) or tokens[index].text != "(": raise ParseError("CREATE TABLE is missing its column list") close = _matching_paren(tokens, index) @@ -512,7 +517,15 @@ def parse_checks(create_sql: str) -> list[Check]: body_start = tokens[index].end body_end = tokens[close].start - body = create_sql[body_start:body_end] + return create_sql[body_start:body_end], body_start + + +def parse_checks(create_sql: str) -> list[Check]: + """Return CHECK constraints from a valid SQLite CREATE TABLE statement.""" + body_info = _table_body(create_sql) + if body_info is None: + return [] + body, body_start = body_info body_tokens = _lex(body) checks: list[Check] = [] for item, item_start, _ in _split_spans(body, body_tokens): @@ -551,6 +564,35 @@ def parse_checks(create_sql: str) -> list[Check]: return checks +def parse_column_comments(create_sql: str) -> dict[str, ColumnComments]: + """Return comments immediately before and after each column definition.""" + body_info = _table_body(create_sql) + if body_info is None: + return {} + body, _ = body_info + comments: dict[str, ColumnComments] = {} + for item, _, _ in _split_spans(body, _lex(body)): + item_tokens = _meaningful(_lex(item)) + if not item_tokens: + continue + item_index = 0 + if item_tokens[item_index].is_keyword("CONSTRAINT"): + item_index = 2 + head = item_tokens[item_index] if item_index < len(item_tokens) else None + if ( + head + and head.kind == "word" + and head.text.upper() in _TABLE_CONSTRAINT_KEYWORDS + ): + continue + column = _unquote(item_tokens[0].text) + before = item[: item_tokens[0].start].strip() + after = item[item_tokens[-1].end :].strip() + if before or after: + comments[column] = ColumnComments(before=before, after=after) + return comments + + def _is_identifier_token(tokens: list[_Token], index: int) -> bool: token = tokens[index] if index + 1 < len(tokens) and tokens[index + 1].text in ("(", "."): @@ -576,9 +618,9 @@ def check_references_identifier(expression: str, identifier: str) -> bool: ) -def check_expression_ends_in_line_comment(expression: str) -> bool: +def sql_ends_in_line_comment(sql: str) -> bool: """Return True if appended SQL would be swallowed by a ``--`` comment.""" - tokens = _lex(expression) + tokens = _lex(sql) if not tokens: return False final = tokens[-1] diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 949f574..2e9b570 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -29,11 +29,13 @@ from sqlite_utils.plugins import ensure_plugins_loaded, pm from .create_table_parser import ( Check, + ColumnComments, ParseError, - check_expression_ends_in_line_comment, check_references_identifier, parse_checks, + parse_column_comments, rewrite_check_expression, + sql_ends_in_line_comment, ) from .utils import ( OperationalError, @@ -95,10 +97,26 @@ def quote_identifier(identifier: str) -> str: def _check_constraint_sql(check: Check) -> str: prefix = f"CONSTRAINT {quote_identifier(check.name)} " if check.name else "" - newline = "\n" if check_expression_ends_in_line_comment(check.check) else "" + newline = "\n" if sql_ends_in_line_comment(check.check) else "" return f"{prefix}CHECK ({check.check}{newline})" +def _column_definition_with_comments( + definition: str, comments: ColumnComments | None +) -> str: + if comments is None: + return definition + before = textwrap.dedent(comments.before).strip() + after = textwrap.dedent(comments.after).strip() + if before: + definition = f"{textwrap.indent(before, ' ')}\n{definition}" + if after: + definition = f"{definition} {after}" + if sql_ends_in_line_comment(after): + definition += "\n" + return definition + + _IDENTIFIER_CASEFOLD = str.maketrans( "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" ) @@ -1394,6 +1412,7 @@ class Database: if_not_exists: bool = False, strict: bool = False, _checks: Iterable[Check] | None = None, + _column_comments: Mapping[str, ColumnComments] | None = None, ) -> str: """ Returns the SQL ``CREATE TABLE`` statement for creating the specified table. @@ -1439,6 +1458,10 @@ class Database: defaults = {resolve_casing(n, columns): v for n, v in (defaults or {}).items()} if column_order is not None: column_order = [resolve_casing(c, columns) for c in column_order] + column_comments = { + resolve_casing(name, columns): comments + for name, comments in (_column_comments or {}).items() + } checks = list(_checks or ()) checks_by_column: dict[str, list[Check]] = {} table_checks: list[Check] = [] @@ -1517,13 +1540,16 @@ class Database: # Refs https://github.com/simonw/sqlite-utils/issues/644 if strict and column_type_str == "FLOAT": column_type_str = "REAL" + column_definition = " {} {column_type}{column_extras}".format( + quote_identifier(column_name), + column_type=column_type_str, + column_extras=( + (" " + " ".join(column_extras)) if column_extras else "" + ), + ) column_defs.append( - " {} {column_type}{column_extras}".format( - quote_identifier(column_name), - column_type=column_type_str, - column_extras=( - (" " + " ".join(column_extras)) if column_extras else "" - ), + _column_definition_with_comments( + column_definition, column_comments.get(column_name) ) ) extra_pk = "" @@ -2713,9 +2739,10 @@ class Table(Queryable): try: existing_checks = self.checks + existing_column_comments = parse_column_comments(self.schema) except ParseError as ex: raise TransformError( - f"Could not parse CHECK constraints for table {self.name!r}: {ex}" + f"Could not parse table schema for table {self.name!r}: {ex}" ) from ex create_table_checks: list[Check] = [] for check in existing_checks: @@ -2739,6 +2766,12 @@ class Table(Queryable): ) ) + create_table_column_comments: dict[str, ColumnComments] = {} + for column, comments in existing_column_comments.items(): + owner = resolve_casing(column, existing_columns) + if owner not in drop: + create_table_column_comments[rename.get(owner) or owner] = comments + create_table_foreign_keys: list[ForeignKeyIndicator] = [] if foreign_keys is not None: @@ -2889,6 +2922,7 @@ class Table(Queryable): column_order=column_order, strict=self.strict if strict is None else strict, _checks=create_table_checks, + _column_comments=create_table_column_comments, ).strip() ) diff --git a/tests/test_create_table_parser.py b/tests/test_create_table_parser.py index 3b878f3..a7aa0c0 100644 --- a/tests/test_create_table_parser.py +++ b/tests/test_create_table_parser.py @@ -4,7 +4,13 @@ import hypothesis.strategies as st import pytest from hypothesis import given -from sqlite_utils.create_table_parser import Check, ParseError, parse_checks +from sqlite_utils.create_table_parser import ( + Check, + ColumnComments, + ParseError, + parse_checks, + parse_column_comments, +) def test_parse_column_and_table_checks(): @@ -46,6 +52,26 @@ def test_comments_are_trivia_not_constraints(): ] +def test_parse_comments_owned_by_columns(): + sql = """ + CREATE TABLE t ( + -- Before id + id /* Between name and type */ INTEGER /* After id */, + /* Between column definitions */ + value TEXT CHECK(value != '') /* After value */, + /* Before a table constraint, not a column */ + CHECK(value != 'forbidden') + ) + """ + assert parse_column_comments(sql) == { + "id": ColumnComments(before="-- Before id", after="/* After id */"), + "value": ColumnComments( + before="/* Between column definitions */", + after="/* After value */", + ), + } + + @pytest.mark.parametrize( "expression,expected", [ diff --git a/tests/test_transform.py b/tests/test_transform.py index 44fec99..980ee9d 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1102,6 +1102,54 @@ def test_transform_preserves_check_ending_in_line_comment(fresh_db): inventory.insert({"quantity": -1}) +def test_transform_preserves_comments_owned_by_columns(fresh_db): + fresh_db.execute(""" + CREATE TABLE people ( + -- Primary identifier + id INTEGER PRIMARY KEY /* IDs are stable */, + /* Displayed to users */ + name TEXT /* May contain spaces */, + -- Age in years + age INTEGER -- May be NULL + ) + """) + people = fresh_db["people"] + people.insert({"id": 1, "name": "Cleo", "age": 5}) + people.transform( + rename={"name": "display_name"}, + types={"age": float}, + column_order=("age", "id", "name"), + ) + assert people.get(1) == {"age": 5.0, "id": 1, "display_name": "Cleo"} + schema = people.schema + assert schema.index("-- Age in years") < schema.index('"age" REAL') + assert schema.index('"age" REAL') < schema.index("-- May be NULL") + assert schema.index("-- Primary identifier") < schema.index('"id" INTEGER') + assert schema.index('"id" INTEGER') < schema.index("/* IDs are stable */") + assert schema.index("/* Displayed to users */") < schema.index( + '"display_name" TEXT' + ) + assert schema.index('"display_name" TEXT') < schema.index( + "/* May contain spaces */" + ) + + +def test_transform_drops_comments_owned_by_dropped_column(fresh_db): + fresh_db.execute(""" + CREATE TABLE t ( + /* Keep this explanation */ + id INTEGER, + /* Drop this explanation */ + obsolete TEXT /* Drop this too */ + ) + """) + fresh_db["t"].transform(drop={"obsolete"}) + schema = fresh_db["t"].schema + assert "Keep this explanation" in schema + assert "Drop this explanation" not in schema + assert "Drop this too" not in schema + + def test_transform_renames_columns_inside_check_constraints(fresh_db): fresh_db.execute(""" CREATE TABLE inventory ( From 2d3c6b9a1e5068fcee6923c9ed74cbd158ee9db4 Mon Sep 17 00:00:00 2001 From: Bunlong Heng Date: Wed, 12 Aug 2026 01:48:06 -0400 Subject: [PATCH 16/33] Escape tokenize argument in enable_fts (#828) The tokenize value passed to Table.enable_fts() was interpolated directly into the CREATE VIRTUAL TABLE statement inside a single-quoted string literal. A value containing a single quote could break out of that literal and inject arbitrary SQL, which executes via executescript(). This is reachable from the CLI via 'enable-fts --tokenize'. Route the value through the existing Database.quote() helper so SQLite itself escapes it. Legitimate tokenizers such as 'porter' are unaffected. Adds a regression test. --- sqlite_utils/db.py | 4 +++- tests/test_fts.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2e9b570..45482f3 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3514,7 +3514,9 @@ class Table(Queryable): table_fts=quote_identifier(self.name + "_fts"), columns=", ".join(quote_identifier(c) for c in columns), fts_version=fts_version, - tokenize=f"\n tokenize='{tokenize}'," if tokenize else "", + tokenize=( + f"\n tokenize={self.db.quote(tokenize)}," if tokenize else "" + ), ) ) should_recreate = False diff --git a/tests/test_fts.py b/tests/test_fts.py index 50c1770..395fc66 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -252,6 +252,18 @@ def test_fts_tokenize(fresh_db, fts_version): }.items() <= rows[0].items() +def test_fts_tokenize_escaped(fresh_db): + # A malicious tokenize value must not be able to break out of the + # string literal in the CREATE VIRTUAL TABLE statement. + table = fresh_db["searchable"] + table.insert_all(search_records) + malicious = "porter'); CREATE TABLE injected(x); --" + with pytest.raises(Exception): + table.enable_fts(["text"], tokenize=malicious) + # The injected statement must not have executed + assert "injected" not in fresh_db.table_names() + + def test_optimize_fts(fresh_db): for fts_version in ("4", "5"): table_name = f"searchable_{fts_version}" From 43d5d3331f5bd056d20ee61903142b86a2ee0efb Mon Sep 17 00:00:00 2001 From: ethanhawkes-gif Date: Wed, 12 Aug 2026 01:52:43 -0400 Subject: [PATCH 17/33] Emit LIMIT -1 when offset is used without limit (#821) * Emit LIMIT -1 when offset is used without limit, closes #816 SQLite requires a LIMIT clause to appear before OFFSET, so passing offset without limit generated invalid SQL such as: select * from "t" offset 2 which raised OperationalError: near "2": syntax error. A negative limit means "no upper bound" in SQLite, so "limit -1 offset N" returns all rows from position N onwards. Fixed in three places that build LIMIT/OFFSET SQL: - Queryable.rows_where() - also covers pks_and_rows_where() - Table.search_sql() - also covers search() - the "sqlite-utils rows" CLI command * Remove duplicate comments --------- Co-authored-by: ethanhawkes-gif <259455325+ethanhawkes-gif@users.noreply.github.com> --- sqlite_utils/cli.py | 2 ++ sqlite_utils/db.py | 6 ++++++ tests/test_cli.py | 5 +++++ tests/test_fts.py | 11 +++++++++++ tests/test_rows.py | 9 +++++++++ 5 files changed, 33 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index dab4b67..d9c7728 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2478,6 +2478,8 @@ def rows( if limit: sql += f" limit {limit}" if offset: + if not limit: + sql += " limit -1" sql += f" offset {offset}" ctx.invoke( query, diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 45482f3..82a95c1 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2064,6 +2064,10 @@ class Queryable: if limit is not None: sql += f" limit {limit}" if offset is not None: + # SQLite requires a limit clause before offset - a negative limit + # means "no upper bound", so offset works without an explicit limit + if limit is None: + sql += " limit -1" sql += f" offset {offset}" cursor = self.db.execute(sql, where_args or []) columns = dedupe_keys(c[0] for c in cursor.description) @@ -3732,6 +3736,8 @@ class Table(Queryable): if limit is not None: limit_offset += f" limit {limit}" if offset is not None: + if limit is None: + limit_offset += " limit -1" limit_offset += f" offset {offset}" return sql.format( dbtable=quote_identifier(self.name), diff --git a/tests/test_cli.py b/tests/test_cli.py index a1e072f..d3ad228 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1183,6 +1183,11 @@ def test_query_memory_does_not_create_file(tmpdir): ["-c", "name", "--limit", "1", "--offset", "1"], '[{"name": "Pancakes"}]', ), + # --offset without --limit + ( + ["-c", "name", "--offset", "1"], + '[{"name": "Pancakes"}]', + ), # --where ( ["-c", "name", "--where", "id = 1"], diff --git a/tests/test_fts.py b/tests/test_fts.py index 395fc66..79af042 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -112,6 +112,17 @@ def test_search_limit_offset(fresh_db): ) +def test_search_offset_without_limit(fresh_db): + table = fresh_db["t"] + table.insert_all(search_records) + table.enable_fts(["text", "country"], fts_version="FTS4") + assert [row["rowid"] for row in table.search("are", order_by="rowid")] == [1, 2] + assert [ + row["rowid"] for row in table.search("are", offset=1, order_by="rowid") + ] == [2] + assert table.search_sql(offset=1).strip().endswith("limit -1 offset 1") + + @pytest.mark.parametrize("fts_version", ("FTS4", "FTS5")) def test_search_where(fresh_db, fts_version): table = fresh_db["t"] diff --git a/tests/test_rows.py b/tests/test_rows.py index 46d4f53..dccb6ad 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -59,6 +59,9 @@ def test_rows_where_order_by(where, order_by, expected_ids, fresh_db): (None, 3, [1, 2, 3]), (0, 3, [1, 2, 3]), (3, 3, [4, 5, 6]), + # offset without limit should return every remaining row + (97, None, [98, 99, 100]), + (0, None, list(range(1, 101))), ], ) def test_rows_where_offset_limit(fresh_db, offset, limit, expected): @@ -70,6 +73,12 @@ def test_rows_where_offset_limit(fresh_db, offset, limit, expected): ] +def test_pks_and_rows_where_offset_without_limit(fresh_db): + table = fresh_db["rows"] + table.insert_all([{"id": id} for id in range(1, 6)], pk="id") + assert [pk for pk, _ in table.pks_and_rows_where(offset=3, order_by="id")] == [4, 5] + + def test_pks_and_rows_where_rowid(fresh_db): table = fresh_db["rowid_table"] table.insert_all({"number": i + 10} for i in range(3)) From 38fe4667006bc03b0dc22852de3bd891645b5707 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 13:40:55 -0700 Subject: [PATCH 18/33] Use db.table() and db.view() in tests, closes #838 --- tests/test_analyze.py | 20 +- tests/test_analyze_tables.py | 14 +- tests/test_atomic.py | 114 +++++---- tests/test_attach.py | 4 +- tests/test_cli.py | 308 +++++++++++----------- tests/test_cli_bulk.py | 10 +- tests/test_cli_convert.py | 74 +++--- tests/test_cli_insert.py | 68 ++--- tests/test_cli_memory.py | 2 +- tests/test_cli_migrate.py | 76 +++--- tests/test_column_affinity.py | 2 +- tests/test_column_casing.py | 154 +++++------ tests/test_constructor.py | 12 +- tests/test_conversions.py | 10 +- tests/test_convert.py | 20 +- tests/test_create.py | 397 +++++++++++++++-------------- tests/test_default_value.py | 6 +- tests/test_delete.py | 22 +- tests/test_duplicate.py | 4 +- tests/test_enable_counts.py | 40 +-- tests/test_extract.py | 118 ++++----- tests/test_extracts.py | 26 +- tests/test_foreign_keys.py | 220 ++++++++-------- tests/test_fts.py | 112 ++++---- tests/test_get.py | 8 +- tests/test_gis.py | 14 +- tests/test_hypothesis.py | 16 +- tests/test_insert_files.py | 6 +- tests/test_introspect.py | 114 +++++---- tests/test_list_mode.py | 62 ++--- tests/test_lookup.py | 18 +- tests/test_m2m.py | 50 ++-- tests/test_migrations.py | 36 +-- tests/test_mutator_transactions.py | 6 +- tests/test_query.py | 36 +-- tests/test_recipes.py | 40 +-- tests/test_recreate.py | 4 +- tests/test_rows.py | 28 +- tests/test_sniff.py | 2 +- tests/test_transform.py | 238 +++++++++-------- tests/test_update.py | 16 +- tests/test_upsert.py | 26 +- tests/test_wal.py | 22 +- 43 files changed, 1321 insertions(+), 1254 deletions(-) diff --git a/tests/test_analyze.py b/tests/test_analyze.py index a4cd8a2..edd5174 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -3,11 +3,13 @@ import pytest @pytest.fixture def db(fresh_db): - fresh_db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id") - fresh_db["one_index"].create_index(["name"]) - fresh_db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") - fresh_db["two_indexes"].create_index(["name"]) - fresh_db["two_indexes"].create_index(["species"]) + fresh_db.table("one_index").insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.table("one_index").create_index(["name"]) + fresh_db.table("two_indexes").insert( + {"id": 1, "name": "Cleo", "species": "dog"}, pk="id" + ) + fresh_db.table("two_indexes").create_index(["name"]) + fresh_db.table("two_indexes").create_index(["species"]) return fresh_db @@ -17,7 +19,7 @@ def test_analyze_whole_database(db): assert set(db.table_names()).issuperset( {"one_index", "two_indexes", "sqlite_stat1"} ) - assert list(db["sqlite_stat1"].rows) == [ + assert list(db.table("sqlite_stat1").rows) == [ {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, @@ -30,12 +32,12 @@ def test_analyze_one_table(db, method): if method == "db_method_with_name": db.analyze("one_index") elif method == "table_method": - db["one_index"].analyze() + db.table("one_index").analyze() assert set(db.table_names()).issuperset( {"one_index", "two_indexes", "sqlite_stat1"} ) - assert list(db["sqlite_stat1"].rows) == [ + assert list(db.table("sqlite_stat1").rows) == [ {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"} ] @@ -46,6 +48,6 @@ def test_analyze_index_by_name(db): assert set(db.table_names()).issuperset( {"one_index", "two_indexes", "sqlite_stat1"} ) - assert list(db["sqlite_stat1"].rows) == [ + assert list(db.table("sqlite_stat1").rows) == [ {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, ] diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index a51bba6..9e4799c 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -9,7 +9,7 @@ from sqlite_utils.db import ColumnDetails, Database @pytest.fixture def db_to_analyze(fresh_db): - stuff = fresh_db["stuff"] + stuff = fresh_db.table("stuff") stuff.insert_all( [ {"id": 1, "owner": "Terryterryterry", "size": 5}, @@ -45,7 +45,7 @@ def big_db_to_analyze_path(tmpdir): "all_null": None, } ) - db["stuff"].insert_all(to_insert) + db.table("stuff").insert_all(to_insert) return path @@ -126,7 +126,7 @@ def big_db_to_analyze_path(tmpdir): ) def test_analyze_column(db_to_analyze, column, extra_kwargs, expected): assert ( - db_to_analyze["stuff"].analyze_column( + db_to_analyze.table("stuff").analyze_column( column, common_limit=2, value_truncate=5, **extra_kwargs ) == expected @@ -186,7 +186,7 @@ def test_analyze_table_save(db_to_analyze_path): cli.cli, ["analyze-tables", db_to_analyze_path, "--save"] ) assert result.exit_code == 0 - rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows) + rows = list(Database(db_to_analyze_path).table("_analyze_tables_").rows) assert rows == [ { "table": "stuff", @@ -248,7 +248,7 @@ def test_analyze_table_save_no_most_no_least_options( args.append("--no-least") result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0 - rows = list(Database(big_db_to_analyze_path)["_analyze_tables_"].rows) + rows = list(Database(big_db_to_analyze_path).table("_analyze_tables_").rows) expected = { "table": "stuff", "column": "category", @@ -297,13 +297,13 @@ def test_analyze_table_column_all_nulls(big_db_to_analyze_path): def test_analyze_table_validate_columns(tmpdir, args, expected_error): path = str(tmpdir / "test_validate_columns.db") db = Database(path) - db["one"].insert( + db.table("one").insert( { "id": 1, "name": "one", } ) - db["two"].insert( + db.table("two").insert( { "id": 1, "age": 5, diff --git a/tests/test_atomic.py b/tests/test_atomic.py index ba16ca5..89a318a 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -45,30 +45,30 @@ def test_iter_complete_sql_statements(sql, expected): def test_atomic_commits(fresh_db): with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") - assert list(fresh_db["dogs"].rows) == [{"id": 1, "name": "Cleo"}] + assert list(fresh_db.table("dogs").rows) == [{"id": 1, "name": "Cleo"}] def test_atomic_rolls_back(fresh_db): with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") raise RuntimeError("boom") - assert not fresh_db["dogs"].exists() + assert not fresh_db.table("dogs").exists() def test_nested_atomic_rolls_back_to_savepoint(fresh_db): - fresh_db["dogs"].create({"id": int, "name": str}, pk="id") + fresh_db.table("dogs").create({"id": int, "name": str}, pk="id") with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}) + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo"}) with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + fresh_db.table("dogs").insert({"id": 2, "name": "Pancakes"}) raise RuntimeError("boom") - fresh_db["dogs"].insert({"id": 3, "name": "Marnie"}) + fresh_db.table("dogs").insert({"id": 3, "name": "Marnie"}) - assert list(fresh_db["dogs"].rows) == [ + assert list(fresh_db.table("dogs").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 3, "name": "Marnie"}, ] @@ -76,12 +76,12 @@ def test_nested_atomic_rolls_back_to_savepoint(fresh_db): def test_outer_atomic_rolls_back_released_savepoint(fresh_db): with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + fresh_db.table("dogs").insert({"id": 2, "name": "Pancakes"}) raise RuntimeError("boom") - assert not fresh_db["dogs"].exists() + assert not fresh_db.table("dogs").exists() def test_executescript_does_not_commit_open_atomic_block(fresh_db): @@ -97,41 +97,41 @@ def test_executescript_does_not_commit_open_atomic_block(fresh_db): """) raise RuntimeError("boom") - assert not fresh_db["dogs"].exists() + assert not fresh_db.table("dogs").exists() def test_transform_does_not_commit_open_atomic_block(fresh_db): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"}) - fresh_db["dogs"].transform(rename={"age": "dog_age"}) + fresh_db.table("dogs").insert({"id": 2, "name": "Pancakes", "age": "6"}) + fresh_db.table("dogs").transform(rename={"age": "dog_age"}) raise RuntimeError("boom") assert ( - fresh_db["dogs"].schema + fresh_db.table("dogs").schema == 'CREATE TABLE "dogs" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n)' ) - assert list(fresh_db["dogs"].rows) == [ + assert list(fresh_db.table("dogs").rows) == [ {"id": 1, "name": "Cleo", "age": "5"}, ] def test_transform_parent_table_with_foreign_keys_in_atomic(fresh_db): fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") - fresh_db["books"].insert( + fresh_db.table("authors").insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db.table("books").insert( {"id": 1, "title": "Book", "author_id": 1}, pk="id", foreign_keys={"author_id"}, ) with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "full_name"}) + fresh_db.table("authors").transform(rename={"name": "full_name"}) assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] assert ( - fresh_db["authors"].schema + fresh_db.table("authors").schema == 'CREATE TABLE "authors" (\n "id" INTEGER PRIMARY KEY,\n "full_name" TEXT\n)' ) assert fresh_db.execute("PRAGMA foreign_key_check").fetchall() == [] @@ -139,19 +139,19 @@ def test_transform_parent_table_with_foreign_keys_in_atomic(fresh_db): def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db): fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") - fresh_db["books"].insert( + fresh_db.table("authors").insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db.table("books").insert( {"id": 1, "title": "Book", "author_id": 1}, pk="id", foreign_keys={"author_id"}, ) with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "full_name"}) + fresh_db.table("authors").transform(rename={"name": "full_name"}) raise RuntimeError("boom") assert ( - fresh_db["authors"].schema + fresh_db.table("authors").schema == 'CREATE TABLE "authors" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n)' ) assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] @@ -160,49 +160,51 @@ def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db): def test_transform_detects_foreign_key_check_violations(fresh_db): fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 2}, pk="id") + fresh_db.table("authors").insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db.table("books").insert({"id": 1, "author_id": 2}, pk="id") with pytest.raises(sqlite3.IntegrityError): - fresh_db["books"].transform(add_foreign_keys=(("author_id", "authors", "id"),)) + fresh_db.table("books").transform( + add_foreign_keys=(("author_id", "authors", "id"),) + ) - assert fresh_db["books"].foreign_keys == [] + assert fresh_db.table("books").foreign_keys == [] assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] def test_atomic_inside_manual_transaction_uses_savepoint(fresh_db): - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.execute("begin") with fresh_db.atomic(): - fresh_db["t"].insert({"id": 2}, pk="id") + fresh_db.table("t").insert({"id": 2}, pk="id") # Nothing is committed until the user's own transaction commits assert fresh_db.conn.in_transaction fresh_db.rollback() - assert [r["id"] for r in fresh_db["t"].rows] == [1] + assert [r["id"] for r in fresh_db.table("t").rows] == [1] # And with a commit instead, the atomic block's writes persist fresh_db.execute("begin") with fresh_db.atomic(): - fresh_db["t"].insert({"id": 3}, pk="id") + fresh_db.table("t").insert({"id": 3}, pk="id") fresh_db.commit() - assert [r["id"] for r in fresh_db["t"].rows] == [1, 3] + assert [r["id"] for r in fresh_db.table("t").rows] == [1, 3] def test_begin_commit_rollback(tmpdir): path = str(tmpdir / "test.db") db = Database(path) - db["t"].insert({"id": 1}, pk="id") + db.table("t").insert({"id": 1}, pk="id") db.begin() - db["t"].insert({"id": 2}, pk="id") + db.table("t").insert({"id": 2}, pk="id") assert db.conn.in_transaction db.rollback() assert not db.conn.in_transaction - assert [r["id"] for r in db["t"].rows] == [1] + assert [r["id"] for r in db.table("t").rows] == [1] db.begin() - db["t"].insert({"id": 3}, pk="id") + db.table("t").insert({"id": 3}, pk="id") db.commit() db.close() db2 = Database(path) - assert [r["id"] for r in db2["t"].rows] == [1, 3] + assert [r["id"] for r in db2.table("t").rows] == [1, 3] db2.close() @@ -222,7 +224,7 @@ def test_commit_and_rollback_without_transaction_are_noops(fresh_db): def test_execute_write_commits_immediately(tmpdir): path = str(tmpdir / "test.db") db = Database(path) - db["t"].insert({"id": 1}, pk="id") + db.table("t").insert({"id": 1}, pk="id") db.execute("insert into t (id) values (2)") # No implicit transaction is left open assert not db.conn.in_transaction @@ -234,24 +236,24 @@ def test_execute_write_commits_immediately(tmpdir): def test_execute_write_respects_explicit_transaction(fresh_db): - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.begin() fresh_db.execute("insert into t (id) values (2)") # Still inside the explicit transaction - not committed assert fresh_db.conn.in_transaction fresh_db.rollback() - assert [r["id"] for r in fresh_db["t"].rows] == [1] + assert [r["id"] for r in fresh_db.table("t").rows] == [1] def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): # A BEGIN hidden behind a leading comment must not be auto-committed # out from under the caller - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.execute("-- start a transaction\nbegin") assert fresh_db.conn.in_transaction fresh_db.execute("insert into t (id) values (2)") fresh_db.rollback() - assert [r["id"] for r in fresh_db["t"].rows] == [1] + assert [r["id"] for r in fresh_db.table("t").rows] == [1] def _sqlite_accepts_bom(): @@ -269,12 +271,12 @@ def test_execute_prefixed_begin_leaves_transaction_open(fresh_db, begin_sql): # out from under the caller if begin_sql.startswith("\ufeff") and not _sqlite_accepts_bom(): pytest.skip("This SQLite version rejects a leading byte order mark") - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.execute(begin_sql) assert fresh_db.conn.in_transaction fresh_db.execute("insert into t (id) values (2)") fresh_db.rollback() - assert [r["id"] for r in fresh_db["t"].rows] == [1] + assert [r["id"] for r in fresh_db.table("t").rows] == [1] def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir): @@ -282,40 +284,40 @@ def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir): # that would silently disable auto-commit for every subsequent write path = str(tmpdir / "test.db") db = Database(path) - db["t"].insert({"id": 1}, pk="id") + db.table("t").insert({"id": 1}, pk="id") with pytest.raises(sqlite3.IntegrityError): db.execute("insert into t (id) values (1)") assert not db.conn.in_transaction # Subsequent writes commit as normal and survive closing the connection - db["other"].insert({"id": 2}) + db.table("other").insert({"id": 2}) db.close() db2 = Database(path) - assert db2["other"].exists() + assert db2.table("other").exists() db2.close() def test_execute_failed_write_preserves_explicit_transaction(fresh_db): # A failed write inside an explicit transaction must not roll back # the caller's earlier work - only the caller decides that - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.begin() fresh_db.execute("insert into t (id) values (2)") with pytest.raises(sqlite3.IntegrityError): fresh_db.execute("insert into t (id) values (1)") assert fresh_db.conn.in_transaction fresh_db.commit() - assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] + assert [r["id"] for r in fresh_db.table("t").rows] == [1, 2] def test_execute_failed_write_inside_atomic_preserves_block(fresh_db): # A caught failure inside an atomic() block must leave the block's # transaction open so its other work still commits - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") with fresh_db.atomic(): fresh_db.execute("insert into t (id) values (2)") with pytest.raises(sqlite3.IntegrityError): fresh_db.execute("insert into t (id) values (1)") - assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] + assert [r["id"] for r in fresh_db.table("t").rows] == [1, 2] def test_query_returning_commits_after_iteration(tmpdir): @@ -325,7 +327,7 @@ def test_query_returning_commits_after_iteration(tmpdir): _pytest.skip("RETURNING requires SQLite 3.35.0 or higher") path = str(tmpdir / "test.db") db = Database(path) - db["t"].insert({"id": 1}, pk="id") + db.table("t").insert({"id": 1}, pk="id") rows = list(db.query("insert into t (id) values (2) returning id")) assert rows == [{"id": 2}] assert not db.conn.in_transaction @@ -375,7 +377,7 @@ def test_nested_atomic_preserves_error_from_transaction_destroying_trigger( def test_atomic_preserves_error_from_insert_or_rollback(fresh_db): - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") with pytest.raises(sqlite3.IntegrityError), fresh_db.atomic(): fresh_db.execute("insert or rollback into t (id) values (1)") assert not fresh_db.conn.in_transaction diff --git a/tests/test_attach.py b/tests/test_attach.py index b594b3b..2b11e36 100644 --- a/tests/test_attach.py +++ b/tests/test_attach.py @@ -6,10 +6,10 @@ def test_attach(tmpdir): bar_path = str(tmpdir / "bar.db") db = Database(foo_path) with db.conn: - db["foo"].insert({"id": 1, "text": "foo"}) + db.table("foo").insert({"id": 1, "text": "foo"}) db2 = Database(bar_path) with db2.conn: - db2["bar"].insert({"id": 1, "text": "bar"}) + db2.table("bar").insert({"id": 1, "text": "bar"}) db.attach("bar", bar_path) assert db.execute( "select * from foo union all select * from bar.bar" diff --git a/tests/test_cli.py b/tests/test_cli.py index d3ad228..012900c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -72,13 +72,13 @@ def test_views(db_path): def test_tables_fts4(db_path): - Database(db_path)["Gosh"].enable_fts(["c2"], fts_version="FTS4") + Database(db_path).table("Gosh").enable_fts(["c2"], fts_version="FTS4") result = CliRunner().invoke(cli.cli, ["tables", "--fts4", db_path]) assert '[{"table": "Gosh_fts"}]' == result.output.strip() def test_tables_fts5(db_path): - Database(db_path)["Gosh"].enable_fts(["c2"], fts_version="FTS5") + Database(db_path).table("Gosh").enable_fts(["c2"], fts_version="FTS5") result = CliRunner().invoke(cli.cli, ["tables", "--fts5", db_path]) assert '[{"table": "Gosh_fts"}]' == result.output.strip() @@ -86,7 +86,7 @@ def test_tables_fts5(db_path): def test_tables_counts_and_columns(db_path): db = Database(db_path) with db.conn: - db["lots"].insert_all([{"id": i, "age": i + 1} for i in range(30)]) + db.table("lots").insert_all([{"id": i, "age": i + 1} for i in range(30)]) result = CliRunner().invoke(cli.cli, ["tables", "--counts", "--columns", db_path]) assert ( '[{"table": "Gosh", "count": 0, "columns": ["c1", "c2", "c3"]},\n' @@ -121,7 +121,7 @@ def test_tables_counts_and_columns(db_path): def test_tables_counts_and_columns_csv(db_path, format, expected): db = Database(db_path) with db.conn: - db["lots"].insert_all([{"id": i, "age": i + 1} for i in range(30)]) + db.table("lots").insert_all([{"id": i, "age": i + 1} for i in range(30)]) result = CliRunner().invoke( cli.cli, ["tables", "--counts", "--columns", format, db_path] ) @@ -131,7 +131,7 @@ def test_tables_counts_and_columns_csv(db_path, format, expected): def test_tables_schema(db_path): db = Database(db_path) with db.conn: - db["lots"].insert_all([{"id": i, "age": i + 1} for i in range(30)]) + db.table("lots").insert_all([{"id": i, "age": i + 1} for i in range(30)]) result = CliRunner().invoke(cli.cli, ["tables", "--schema", db_path]) assert ( '[{"table": "Gosh", "schema": "CREATE TABLE Gosh (c1 text, c2 text, c3 text)"},\n' @@ -183,7 +183,7 @@ def test_tables_schema(db_path): def test_output_table(db_path, options, expected): db = Database(db_path) with db.conn: - db["rows"].insert_all( + db.table("rows").insert_all( [ { "c1": f"verb{i}", @@ -207,7 +207,7 @@ def test_output_table_no_headers(db_path, fmt_option): # tabulate formats and the column names were always printed. db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "name": "Cleo", "age": 4}, {"id": 2, "name": "Pancakes", "age": 2}, @@ -244,14 +244,14 @@ def test_output_table_no_headers(db_path, fmt_option): def test_create_index(db_path): db = Database(db_path) - assert [] == db["Gosh"].indexes + assert [] == db.table("Gosh").indexes result = CliRunner().invoke(cli.cli, ["create-index", db_path, "Gosh", "c1"]) assert result.exit_code == 0 assert [ Index( seq=0, name="idx_Gosh_c1", unique=0, origin="c", partial=0, columns=["c1"] ) - ] == db["Gosh"].indexes + ] == db.table("Gosh").indexes # Try with a custom name result = CliRunner().invoke( cli.cli, ["create-index", db_path, "Gosh", "c2", "--name", "blah"] @@ -262,7 +262,7 @@ def test_create_index(db_path): Index( seq=1, name="idx_Gosh_c1", unique=0, origin="c", partial=0, columns=["c1"] ), - ] == db["Gosh"].indexes + ] == db.table("Gosh").indexes # Try a two-column unique index create_index_unique_args = [ "create-index", @@ -283,7 +283,7 @@ def test_create_index(db_path): partial=0, columns=["c1", "c2"], ) - ] == db["Gosh2"].indexes + ] == db.table("Gosh2").indexes # Trying to create the same index should fail assert CliRunner().invoke(cli.cli, create_index_unique_args).exit_code != 0 # ... unless we use --if-not-exists or --ignore @@ -296,11 +296,11 @@ def test_create_index(db_path): def test_drop_index(db_path): db = Database(db_path) - db["Gosh"].create_index(["c1"]) - assert [index.name for index in db["Gosh"].indexes] == ["idx_Gosh_c1"] + db.table("Gosh").create_index(["c1"]) + assert [index.name for index in db.table("Gosh").indexes] == ["idx_Gosh_c1"] result = CliRunner().invoke(cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1"]) assert result.exit_code == 0 - assert db["Gosh"].indexes == [] + assert db.table("Gosh").indexes == [] result = CliRunner().invoke(cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1"]) assert result.exit_code == 1 @@ -315,7 +315,7 @@ def test_drop_index(db_path): def test_create_index_analyze(db_path): db = Database(db_path) assert "sqlite_stat1" not in db.table_names() - assert [] == db["Gosh"].indexes + assert [] == db.table("Gosh").indexes result = CliRunner().invoke( cli.cli, ["create-index", db_path, "Gosh", "c1", "--analyze"] ) @@ -325,7 +325,7 @@ def test_create_index_analyze(db_path): def test_create_index_desc(db_path): db = Database(db_path) - assert [] == db["Gosh"].indexes + assert [] == db.table("Gosh").indexes result = CliRunner().invoke(cli.cli, ["create-index", db_path, "Gosh", "--", "-c1"]) assert result.exit_code == 0 assert ( @@ -361,12 +361,12 @@ def test_create_index_desc(db_path): def test_add_column(db_path, col_name, col_type, expected_schema): db = Database(db_path) db.create_table("dogs", {"name": str}) - assert db["dogs"].schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' + assert db.table("dogs").schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' args = ["add-column", db_path, "dogs", col_name] if col_type is not None: args.append(col_type) assert CliRunner().invoke(cli.cli, args).exit_code == 0 - assert db["dogs"].schema == expected_schema + assert db.table("dogs").schema == expected_schema @pytest.mark.parametrize("ignore", (True, False)) @@ -385,7 +385,7 @@ def test_add_column_ignore(db_path, ignore): def test_add_column_not_null_default(db_path): db = Database(db_path) db.create_table("dogs", {"name": str}) - assert db["dogs"].schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' + assert db.table("dogs").schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' args = [ "add-column", db_path, @@ -395,7 +395,7 @@ def test_add_column_not_null_default(db_path): "dogs'dawg", ] assert CliRunner().invoke(cli.cli, args).exit_code == 0 - assert db["dogs"].schema == ( + assert db.table("dogs").schema == ( 'CREATE TABLE "dogs" (\n' ' "name" TEXT\n' ", \"nickname\" TEXT NOT NULL DEFAULT 'dogs''dawg')" @@ -415,10 +415,10 @@ def test_add_column_not_null_default(db_path): ) def test_add_foreign_key(db_path, args, assert_message): db = Database(db_path) - db["authors"].insert_all( + db.table("authors").insert_all( [{"id": 1, "name": "Sally"}, {"id": 2, "name": "Asheesh"}], pk="id" ) - db["books"].insert_all( + db.table("books").insert_all( [ {"title": "Hedgehogs of the world", "author_id": 1}, {"title": "How to train your wolf", "author_id": 2}, @@ -431,7 +431,7 @@ def test_add_foreign_key(db_path, args, assert_message): ForeignKey( table="books", column="author_id", other_table="authors", other_column="id" ) - ] == db["books"].foreign_keys + ] == db.table("books").foreign_keys # Error if we try to add it twice: result = CliRunner().invoke( @@ -460,14 +460,14 @@ def test_add_foreign_key(db_path, args, assert_message): def test_add_column_foreign_key(db_path): db = Database(db_path) - db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - db["books"].insert({"title": "Hedgehogs of the world"}) + db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + db.table("books").insert({"title": "Hedgehogs of the world"}) # Add an author_id foreign key column to the books table result = CliRunner().invoke( cli.cli, ["add-column", db_path, "books", "author_id", "--fk", "authors"] ) assert result.exit_code == 0, result.output - assert db["books"].schema == ( + assert db.table("books").schema == ( 'CREATE TABLE "books" (\n' ' "title" TEXT,\n' ' "author_id" INTEGER REFERENCES "authors"("id")\n' @@ -488,7 +488,7 @@ def test_add_column_foreign_key(db_path): ], ) assert result.exit_code == 0, result.output - assert db["books"].schema == ( + assert db.table("books").schema == ( 'CREATE TABLE "books" (\n' ' "title" TEXT,\n' ' "author_id" INTEGER REFERENCES "authors"("id"),\n' @@ -505,7 +505,7 @@ def test_add_column_foreign_key(db_path): def test_suggest_alter_if_column_missing(db_path): db = Database(db_path) - db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") result = CliRunner().invoke( cli.cli, ["insert", db_path, "authors", "-"], @@ -521,27 +521,27 @@ def test_suggest_alter_if_column_missing(db_path): def test_index_foreign_keys(db_path): test_add_column_foreign_key(db_path) db = Database(db_path) - assert [] == db["books"].indexes + assert [] == db.table("books").indexes result = CliRunner().invoke(cli.cli, ["index-foreign-keys", db_path]) assert result.exit_code == 0 assert [["author_id"], ["author_name_ref"]] == [ - i.columns for i in db["books"].indexes + i.columns for i in db.table("books").indexes ] def test_enable_fts(db_path): db = Database(db_path) - assert db["Gosh"].detect_fts() is None + assert db.table("Gosh").detect_fts() is None result = CliRunner().invoke( cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] ) assert result.exit_code == 0 - assert "Gosh_fts" == db["Gosh"].detect_fts() + assert "Gosh_fts" == db.table("Gosh").detect_fts() # Table names with restricted chars are handled correctly. # colons and dots are restricted characters for table names. - db["http://example.com"].create({"c1": str, "c2": str, "c3": str}) - assert db["http://example.com"].detect_fts() is None + db.table("http://example.com").create({"c1": str, "c2": str, "c3": str}) + assert db.table("http://example.com").detect_fts() is None result = CliRunner().invoke( cli.cli, [ @@ -555,7 +555,7 @@ def test_enable_fts(db_path): ], ) assert result.exit_code == 0 - assert "http://example.com_fts" == db["http://example.com"].detect_fts() + assert "http://example.com_fts" == db.table("http://example.com").detect_fts() # Check tokenize was set to porter assert ( 'CREATE VIRTUAL TABLE "http://example.com_fts" USING FTS4 (\n' @@ -563,19 +563,19 @@ def test_enable_fts(db_path): " tokenize='porter',\n" ' content="http://example.com"' "\n)" - ) == db["http://example.com_fts"].schema - db["http://example.com"].drop() + ) == db.table("http://example.com_fts").schema + db.table("http://example.com").drop() def test_enable_fts_replace(db_path): db = Database(db_path) - assert db["Gosh"].detect_fts() is None + assert db.table("Gosh").detect_fts() is None result = CliRunner().invoke( cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] ) assert result.exit_code == 0 - assert "Gosh_fts" == db["Gosh"].detect_fts() - assert db["Gosh_fts"].columns_dict == {"c1": str} + assert "Gosh_fts" == db.table("Gosh").detect_fts() + assert db.table("Gosh_fts").columns_dict == {"c1": str} # This should throw an error result2 = CliRunner().invoke( @@ -589,11 +589,11 @@ def test_enable_fts_replace(db_path): cli.cli, ["enable-fts", db_path, "Gosh", "c2", "--fts4", "--replace"] ) assert result3.exit_code == 0 - assert db["Gosh_fts"].columns_dict == {"c2": str} + assert db.table("Gosh_fts").columns_dict == {"c2": str} def test_enable_fts_with_triggers(db_path): - Database(db_path)["Gosh"].insert_all([{"c1": "baz"}]) + Database(db_path).table("Gosh").insert_all([{"c1": "baz"}]) exit_code = ( CliRunner() .invoke( @@ -612,12 +612,12 @@ def test_enable_fts_with_triggers(db_path): ) assert [("baz",)] == search("baz") - Database(db_path)["Gosh"].insert_all([{"c1": "martha"}]) + Database(db_path).table("Gosh").insert_all([{"c1": "martha"}]) assert [("martha",)] == search("martha") def test_populate_fts(db_path): - Database(db_path)["Gosh"].insert_all([{"c1": "baz"}]) + Database(db_path).table("Gosh").insert_all([{"c1": "baz"}]) exit_code = ( CliRunner() .invoke(cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"]) @@ -633,7 +633,7 @@ def test_populate_fts(db_path): ) assert [("baz",)] == search("baz") - Database(db_path)["Gosh"].insert_all([{"c1": "martha"}]) + Database(db_path).table("Gosh").insert_all([{"c1": "martha"}]) assert [] == search("martha") exit_code = ( CliRunner().invoke(cli.cli, ["populate-fts", db_path, "Gosh", "c1"]).exit_code @@ -645,7 +645,7 @@ def test_populate_fts(db_path): def test_disable_fts(db_path): db = Database(db_path) assert {"Gosh", "Gosh2"} == set(db.table_names()) - db["Gosh"].enable_fts(["c1"], create_triggers=True) + db.table("Gosh").enable_fts(["c1"], create_triggers=True) assert { "Gosh_fts", "Gosh_fts_idx", @@ -677,7 +677,7 @@ def test_optimize(db_path, tables): db = Database(db_path) with db.conn: for table in ("Gosh", "Gosh2"): - db[table].insert_all( + db.table(table).insert_all( [ { "c1": f"verb{i}", @@ -687,8 +687,8 @@ def test_optimize(db_path, tables): for i in range(10000) ] ) - db["Gosh"].enable_fts(["c1", "c2", "c3"], fts_version="FTS4") - db["Gosh2"].enable_fts(["c1", "c2", "c3"], fts_version="FTS5") + db.table("Gosh").enable_fts(["c1", "c2", "c3"], fts_version="FTS4") + db.table("Gosh2").enable_fts(["c1", "c2", "c3"], fts_version="FTS5") size_before_optimize = os.stat(db_path).st_size result = CliRunner().invoke(cli.cli, ["optimize", db_path] + tables) assert result.exit_code == 0 @@ -713,22 +713,22 @@ def test_rebuild_fts_fixes_docsize_error(db_path): for i in range(10000) ] with db.conn: - db["fts5_table"].insert_all(records, pk="c1") - db["fts5_table"].enable_fts( + db.table("fts5_table").insert_all(records, pk="c1") + db.table("fts5_table").enable_fts( ["c1", "c2", "c3"], fts_version="FTS5", create_triggers=True ) # Search should work - assert list(db["fts5_table"].search("verb1")) + assert list(db.table("fts5_table").search("verb1")) # Replicate docsize error from this issue for FTS5 # https://github.com/simonw/sqlite-utils/issues/149 - assert db["fts5_table_fts_docsize"].count == 10000 - db["fts5_table"].insert_all(records, replace=True) - assert db["fts5_table"].count == 10000 - assert db["fts5_table_fts_docsize"].count == 20000 + assert db.table("fts5_table_fts_docsize").count == 10000 + db.table("fts5_table").insert_all(records, replace=True) + assert db.table("fts5_table").count == 10000 + assert db.table("fts5_table_fts_docsize").count == 20000 # Running rebuild-fts should fix this result = CliRunner().invoke(cli.cli, ["rebuild-fts", db_path, "fts5_table"]) assert result.exit_code == 0 - assert db["fts5_table_fts_docsize"].count == 10000 + assert db.table("fts5_table_fts_docsize").count == 10000 @pytest.mark.parametrize( @@ -741,7 +741,7 @@ def test_rebuild_fts_fixes_docsize_error(db_path): def test_query_csv(db_path, format, expected): db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}, @@ -793,7 +793,7 @@ _one_query = "select id, name, age from dogs where id = 1" def test_query_json(db_path, sql, args, expected): db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}, @@ -807,7 +807,7 @@ def test_query_sql_from_stdin(db_path): # https://github.com/simonw/sqlite-utils/issues/765 db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}, @@ -1004,7 +1004,7 @@ LOREM_IPSUM_COMPRESSED = ( def test_query_json_binary(db_path): db = Database(db_path) with db.conn: - db["files"].insert( + db.table("files").insert( { "name": "lorem.txt", "sz": 16984, @@ -1059,7 +1059,7 @@ def test_query_params(db_path, sql, params, expected): def test_query_json_with_json_cols(db_path): db = Database(db_path) with db.conn: - db["dogs"].insert( + db.table("dogs").insert( { "id": 1, "name": "Cleo", @@ -1088,7 +1088,7 @@ def test_query_json_with_json_cols(db_path): def test_query_json_unicode_not_escaped_by_default(db_path): db = Database(db_path) with db.conn: - db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + db.table("text").insert({"id": 1, "text": "Japanese 日本語"}, pk="id") result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text"]) assert result.exit_code == 0 assert result.output.strip() == '[{"id": 1, "text": "Japanese 日本語"}]' @@ -1102,7 +1102,7 @@ def test_query_json_unicode_not_escaped_by_default(db_path): def test_query_json_ascii_option(db_path, command): db = Database(db_path) with db.conn: - db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + db.table("text").insert({"id": 1, "text": "Japanese 日本語"}, pk="id") if command == "query": args = [db_path, "select id, text from text", "--ascii"] else: @@ -1118,7 +1118,7 @@ def test_query_json_ascii_option(db_path, command): [(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)], ) def test_query_raw(db_path, content, is_binary): - Database(db_path)["files"].insert({"content": content}) + Database(db_path).table("files").insert({"content": content}) result = CliRunner().invoke( cli.cli, [db_path, "select content from files", "--raw"] ) @@ -1133,7 +1133,7 @@ def test_query_raw(db_path, content, is_binary): [(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)], ) def test_query_raw_lines(db_path, content, is_binary): - Database(db_path)["files"].insert_all({"content": content} for _ in range(3)) + Database(db_path).table("files").insert_all({"content": content} for _ in range(3)) result = CliRunner().invoke( cli.cli, [db_path, "select content from files", "--raw-lines"] ) @@ -1215,7 +1215,7 @@ def test_query_memory_does_not_create_file(tmpdir): def test_rows(db_path, args, expected): db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}, @@ -1240,7 +1240,7 @@ def test_upsert(db_path, tmpdir): catch_exceptions=False, ) assert result.exit_code == 0, result.output - assert 2 == db["dogs"].count + assert 2 == db.table("dogs").count # Now run the upsert to update just their ages upsert_dogs = [ {"id": 1, "age": 5}, @@ -1295,8 +1295,8 @@ def test_upsert_pk_inferred_from_existing_table(db_path, tmpdir): def test_upsert_analyze(db_path, tmpdir): db = Database(db_path) - db["rows"].insert({"id": 1, "foo": "x", "n": 3}, pk="id") - db["rows"].create_index(["n"]) + db.table("rows").insert({"id": 1, "foo": "x", "n": 3}, pk="id") + db.table("rows").create_index(["n"]) assert "sqlite_stat1" not in db.table_names() result = CliRunner().invoke( cli.cli, @@ -1310,7 +1310,7 @@ def test_upsert_analyze(db_path, tmpdir): def test_upsert_flatten(tmpdir): db_path = str(tmpdir / "flat.db") db = Database(db_path) - db["upsert_me"].insert({"id": 1, "name": "Example"}, pk="id") + db.table("upsert_me").insert({"id": 1, "name": "Example"}, pk="id") result = CliRunner().invoke( cli.cli, ["upsert", db_path, "upsert_me", "-", "--flatten", "--pk", "id", "--alter"], @@ -1424,7 +1424,7 @@ def test_create_table(args, schema): ) assert result.exit_code == 0 db = Database("test.db") - assert schema == db["t"].schema + assert schema == db.table("t").schema def test_create_table_foreign_key(): @@ -1459,21 +1459,21 @@ def test_create_table_foreign_key(): ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT\n' ")" - ) == db["authors"].schema + ) == db.table("authors").schema assert ( 'CREATE TABLE "books" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "title" TEXT,\n' ' "author_id" INTEGER REFERENCES "authors"("id")\n' ")" - ) == db["books"].schema + ) == db.table("books").schema def test_create_table_error_if_table_exists(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) result = runner.invoke( cli.cli, ["create-table", "test.db", "dogs", "id", "integer"] ) @@ -1488,24 +1488,24 @@ def test_create_table_ignore(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) result = runner.invoke( cli.cli, ["create-table", "test.db", "dogs", "id", "integer", "--ignore"] ) assert result.exit_code == 0 - assert 'CREATE TABLE "dogs" (\n "name" TEXT\n)' == db["dogs"].schema + assert 'CREATE TABLE "dogs" (\n "name" TEXT\n)' == db.table("dogs").schema def test_create_table_replace(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) result = runner.invoke( cli.cli, ["create-table", "test.db", "dogs", "id", "integer", "--replace"] ) assert result.exit_code == 0 - assert 'CREATE TABLE "dogs" (\n "id" INTEGER\n)' == db["dogs"].schema + assert 'CREATE TABLE "dogs" (\n "id" INTEGER\n)' == db.table("dogs").schema def test_create_view(): @@ -1517,7 +1517,8 @@ def test_create_view(): ) assert result.exit_code == 0 assert ( - 'CREATE VIEW "version" AS select sqlite_version()' == db["version"].schema + 'CREATE VIEW "version" AS select sqlite_version()' + == db.view("version").schema ) @@ -1554,7 +1555,7 @@ def test_create_view_ignore(): assert result.exit_code == 0 assert ( 'CREATE VIEW "version" AS select sqlite_version() + 1' - == db["version"].schema + == db.view("version").schema ) @@ -1575,7 +1576,8 @@ def test_create_view_replace(): ) assert result.exit_code == 0 assert ( - 'CREATE VIEW "version" AS select sqlite_version()' == db["version"].schema + 'CREATE VIEW "version" AS select sqlite_version()' + == db.view("version").schema ) @@ -1583,7 +1585,7 @@ def test_drop_table(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["t"].create({"pk": int}, pk="pk") + db.table("t").create({"pk": int}, pk="pk") assert "t" in db.table_names() result = runner.invoke( cli.cli, @@ -1601,7 +1603,7 @@ def test_drop_table_error(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["t"].create({"pk": int}, pk="pk") + db.table("t").create({"pk": int}, pk="pk") result = runner.invoke( cli.cli, [ @@ -1624,7 +1626,7 @@ def test_drop_table_on_view_errors(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["t"].insert({"id": 1}) + db.table("t").insert({"id": 1}) db.create_view("v", "select * from t") result = runner.invoke(cli.cli, ["drop-table", "test.db", "v"]) assert result.exit_code == 1 @@ -1660,7 +1662,7 @@ def test_drop_view_on_table_errors(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["t"].insert({"id": 1}) + db.table("t").insert({"id": 1}) result = runner.invoke(cli.cli, ["drop-view", "test.db", "t"]) assert result.exit_code == 1 assert 'Error: "t" is a table, not a view - use drop-table to drop it' == ( @@ -1677,7 +1679,7 @@ def test_drop_view_error(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["t"].create({"pk": int}, pk="pk") + db.table("t").create({"pk": int}, pk="pk") result = runner.invoke( cli.cli, [ @@ -1702,7 +1704,7 @@ def test_enable_wal(): with runner.isolated_filesystem(): for dbname in dbs: db = Database(dbname) - db["t"].create({"pk": int}, pk="pk") + db.table("t").create({"pk": int}, pk="pk") assert db.journal_mode == "delete" result = runner.invoke(cli.cli, ["enable-wal"] + dbs, catch_exceptions=False) assert result.exit_code == 0 @@ -1717,7 +1719,7 @@ def test_disable_wal(): with runner.isolated_filesystem(): for dbname in dbs: db = Database(dbname) - db["t"].create({"pk": int}, pk="pk") + db.table("t").create({"pk": int}, pk="pk") db.enable_wal() assert db.journal_mode == "wal" result = runner.invoke(cli.cli, ["disable-wal"] + dbs) @@ -1740,7 +1742,7 @@ def test_disable_wal(): def test_query_update(db_path, args, expected): db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "age": 4, "name": "Cleo"}, ] @@ -1756,11 +1758,13 @@ def test_query_update(db_path, args, expected): def test_add_foreign_keys(db_path): db = Database(db_path) - db["countries"].insert({"id": 7, "name": "Panama"}, pk="id") - db["authors"].insert({"id": 3, "name": "Matilda", "country_id": 7}, pk="id") - db["books"].insert({"id": 2, "title": "Wolf anatomy", "author_id": 3}, pk="id") - assert db["authors"].foreign_keys == [] - assert db["books"].foreign_keys == [] + db.table("countries").insert({"id": 7, "name": "Panama"}, pk="id") + db.table("authors").insert({"id": 3, "name": "Matilda", "country_id": 7}, pk="id") + db.table("books").insert( + {"id": 2, "title": "Wolf anatomy", "author_id": 3}, pk="id" + ) + assert db.table("authors").foreign_keys == [] + assert db.table("books").foreign_keys == [] result = CliRunner().invoke( cli.cli, [ @@ -1777,7 +1781,7 @@ def test_add_foreign_keys(db_path): ], ) assert result.exit_code == 0 - assert db["authors"].foreign_keys == [ + assert db.table("authors").foreign_keys == [ ForeignKey( table="authors", column="country_id", @@ -1785,7 +1789,7 @@ def test_add_foreign_keys(db_path): other_column="id", ) ] - assert db["books"].foreign_keys == [ + assert db.table("books").foreign_keys == [ ForeignKey( table="books", column="author_id", other_table="authors", other_column="id" ) @@ -1909,7 +1913,7 @@ def test_add_foreign_keys(db_path): def test_transform(db_path, args, expected_schema): db = Database(db_path) with db.conn: - db["dogs"].insert( + db.table("dogs").insert( {"id": 1, "age": 4, "name": "Cleo"}, not_null={"age"}, defaults={"age": 1}, @@ -1918,20 +1922,20 @@ def test_transform(db_path, args, expected_schema): result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs"] + args) print(result.output) assert result.exit_code == 0 - schema = db["dogs"].schema + schema = db.table("dogs").schema assert schema == expected_schema def test_transform_sql(db_path): db = Database(db_path) with db.conn: - db["dogs"].insert( + db.table("dogs").insert( {"id": 1, "age": 4, "name": "Cleo"}, not_null={"age"}, defaults={"age": 1}, pk="id", ) - original_schema = db["dogs"].schema + original_schema = db.table("dogs").schema result = CliRunner().invoke( cli.cli, ["transform", db_path, "dogs", "--drop", "name", "--sql"] @@ -1942,7 +1946,7 @@ def test_transform_sql(db_path): assert '"age" INTEGER NOT NULL DEFAULT' in result.output assert 'DROP TABLE "dogs";' in result.output assert 'ALTER TABLE "dogs_new_' in result.output - assert db["dogs"].schema == original_schema + assert db.table("dogs").schema == original_schema @pytest.mark.parametrize( @@ -1958,12 +1962,12 @@ def test_transform_strict_option(db_path, initial_strict, args, expected_strict) db = Database(db_path) if not db.supports_strict: pytest.skip("SQLite version does not support strict tables") - db["dogs"].create({"id": int}, strict=initial_strict) + db.table("dogs").create({"id": int}, strict=initial_strict) result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs"] + args) assert result.exit_code == 0, result.output - assert db["dogs"].strict is expected_strict + assert db.table("dogs").strict is expected_strict @pytest.mark.parametrize( @@ -1977,20 +1981,20 @@ def test_transform_strict_option_sql(db_path, initial_strict, flag, sql_is_stric db = Database(db_path) if not db.supports_strict: pytest.skip("SQLite version does not support strict tables") - db["dogs"].create({"id": int}, strict=initial_strict) + db.table("dogs").create({"id": int}, strict=initial_strict) result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs", flag, "--sql"]) assert result.exit_code == 0, result.output assert (") STRICT;" in result.output) is sql_is_strict - assert db["dogs"].strict is initial_strict + assert db.table("dogs").strict is initial_strict def test_transform_strict_option_with_invalid_data(db_path): db = Database(db_path) if not db.supports_strict: pytest.skip("SQLite version does not support strict tables") - dogs = db["dogs"] + dogs = db.table("dogs") dogs.create({"id": int}) dogs.insert({"id": "not-an-integer"}) @@ -2048,10 +2052,10 @@ def test_transform_add_or_drop_foreign_key(db_path, extra_args, expected_schema) db = Database(db_path) with db.conn: # Create table with three foreign keys so we can drop two of them - db["continent"].insert({"id": 1, "name": "Europe"}, pk="id") - db["country"].insert({"id": 1, "name": "France"}, pk="id") - db["city"].insert({"id": 24, "name": "Paris"}, pk="id") - db["places"].insert( + db.table("continent").insert({"id": 1, "name": "Europe"}, pk="id") + db.table("country").insert({"id": 1, "name": "France"}, pk="id") + db.table("city").insert({"id": 24, "name": "Paris"}, pk="id") + db.table("places").insert( { "id": 32, "name": "Caveau de la Huchette", @@ -2072,7 +2076,7 @@ def test_transform_add_or_drop_foreign_key(db_path, extra_args, expected_schema) + extra_args, ) assert result.exit_code == 0 - schema = db["places"].schema + schema = db.table("places").schema assert schema == expected_schema @@ -2133,7 +2137,7 @@ _common_other_schema = ( def test_extract(db_path, args, expected_table_schema, expected_other_schema): db = Database(db_path) with db.conn: - db["trees"].insert( + db.table("trees").insert( {"id": 1, "address": "4 Park Ave", "species": "Palm"}, pk="id", ) @@ -2142,7 +2146,7 @@ def test_extract(db_path, args, expected_table_schema, expected_other_schema): ) print(result.output) assert result.exit_code == 0 - schema = db["trees"].schema + schema = db.table("trees").schema assert schema == expected_table_schema other_schema = next( t for t in db.tables if t.name not in ("trees", "Gosh", "Gosh2") @@ -2190,7 +2194,7 @@ def test_insert_encoding(tmpdir): ) assert good_result.exit_code == 0 db = Database(db_path) - assert list(db["places"].rows) == [ + assert list(db.table("places").rows) == [ { "date": "2020-01-01", "name": "Barra da Lagoa", @@ -2226,7 +2230,7 @@ def test_insert_encoding(tmpdir): def test_search(tmpdir, fts, extra_arg, expected): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["articles"].insert_all( + db.table("articles").insert_all( [ {"id": 1, "title": "Title the first"}, {"id": 2, "title": "Title the second"}, @@ -2234,7 +2238,7 @@ def test_search(tmpdir, fts, extra_arg, expected): ], pk="id", ) - db["articles"].enable_fts(["title"], fts_version=fts) + db.table("articles").enable_fts(["title"], fts_version=fts) result = CliRunner().invoke( cli.cli, ["search", db_path, "articles", "second"] + ([extra_arg] if extra_arg else []), @@ -2247,7 +2251,7 @@ def test_search(tmpdir, fts, extra_arg, expected): def test_search_quote(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["creatures"].insert({"name": "dog."}).enable_fts(["name"]) + db.table("creatures").insert({"name": "dog."}).enable_fts(["name"]) # Without --quote should return an error error_result = CliRunner().invoke(cli.cli, ["search", db_path, "creatures", 'dog"']) assert error_result.exit_code == 1 @@ -2355,11 +2359,11 @@ _TRIGGERS_EXPECTED = ( def test_triggers(tmpdir, extra_args, expected): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["articles"].insert( + db.table("articles").insert( {"id": 1, "title": "Title the first"}, pk="id", ) - db["counter"].insert({"count": 1}) + db.table("counter").insert({"count": 1}) db.conn.execute(textwrap.dedent(""" CREATE TRIGGER blah AFTER INSERT ON articles BEGIN @@ -2420,9 +2424,9 @@ def test_triggers(tmpdir, extra_args, expected): def test_schema(tmpdir, options, expected): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["dogs"].create({"id": int, "name": str}) - db["chickens"].create({"id": int, "name": str, "breed": str}) - db["chickens"].create_index(["breed"]) + db.table("dogs").create({"id": int, "name": str}) + db.table("chickens").create({"id": int, "name": str, "breed": str}) + db.table("chickens").create_index(["breed"]) result = CliRunner().invoke( cli.cli, ["schema", db_path] + options, @@ -2446,7 +2450,7 @@ def test_long_csv_column_value(tmpdir): ) assert result.exit_code == 0 db = Database(db_path) - rows = list(db["bigtable"].rows) + rows = list(db.table("bigtable").rows) assert len(rows) == 1 assert rows[0]["text"] == long_string @@ -2473,7 +2477,7 @@ def test_import_no_headers(tmpdir, args, tsv): ) assert result.exit_code == 0, result.output db = Database(db_path) - schema = db["creatures"].schema + schema = db.table("creatures").schema assert schema == ( 'CREATE TABLE "creatures" (\n' ' "untitled_1" TEXT,\n' @@ -2481,7 +2485,7 @@ def test_import_no_headers(tmpdir, args, tsv): ' "untitled_3" TEXT\n' ")" ) - rows = list(db["creatures"].rows) + rows = list(db.table("creatures").rows) assert rows == [ {"untitled_1": "Cleo", "untitled_2": "Dog", "untitled_3": "5"}, {"untitled_1": "Tracy", "untitled_2": "Spider", "untitled_3": "7"}, @@ -2493,10 +2497,10 @@ def test_attach(tmpdir): bar_path = str(tmpdir / "bar.db") db = Database(foo_path) with db.conn: - db["foo"].insert({"id": 1, "text": "foo"}) + db.table("foo").insert({"id": 1, "text": "foo"}) db2 = Database(bar_path) with db2.conn: - db2["bar"].insert({"id": 1, "text": "bar"}) + db2.table("bar").insert({"id": 1, "text": "bar"}) db.attach("bar", bar_path) sql = "select * from foo union all select * from bar.bar" result = CliRunner().invoke( @@ -2557,7 +2561,7 @@ def test_insert_detect_types(tmpdir): ) assert result.exit_code == 0 db = Database(db_path) - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"name": "Cleo", "age": 6, "weight": 45.5}, {"name": "Dori", "age": 1, "weight": 3.5}, ] @@ -2589,7 +2593,7 @@ def test_upsert_detect_types(tmpdir): ) assert result.exit_code == 0 db = Database(db_path) - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"id": 1, "name": "Cleo", "age": 6, "weight": 45.5}, {"id": 2, "name": "Dori", "age": 1, "weight": 3.5}, ] @@ -2608,7 +2612,7 @@ def test_csv_detect_types_creates_real_columns(tmpdir): assert result.exit_code == 0 db = Database(db_path) # Check that the schema uses REAL for the weight column - assert db["creatures"].schema == ( + assert db.table("creatures").schema == ( 'CREATE TABLE "creatures" (\n' ' "name" TEXT,\n' ' "age" INTEGER,\n' @@ -2630,11 +2634,11 @@ def test_insert_no_detect_types(tmpdir): assert result.exit_code == 0 db = Database(db_path) # All columns should be TEXT when --no-detect-types is used - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"name": "Cleo", "age": "6", "weight": "45.5"}, {"name": "Dori", "age": "1", "weight": "3.5"}, ] - assert db["creatures"].schema == ( + assert db.table("creatures").schema == ( 'CREATE TABLE "creatures" (\n' ' "name" TEXT,\n' ' "age" TEXT,\n' @@ -2665,11 +2669,11 @@ def test_upsert_no_detect_types(tmpdir): assert result.exit_code == 0 db = Database(db_path) # All columns should be TEXT when --no-detect-types is used - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"id": "1", "name": "Cleo", "age": "6", "weight": "45.5"}, {"id": "2", "name": "Dori", "age": "1", "weight": "3.5"}, ] - assert db["creatures"].schema == ( + assert db.table("creatures").schema == ( 'CREATE TABLE "creatures" (\n' ' "id" TEXT PRIMARY KEY,\n' ' "name" TEXT,\n' @@ -2751,20 +2755,20 @@ def test_create_database(tmpdir, enable_wal): def test_analyze(tmpdir, options, expected): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id") - db["one_index"].create_index(["name"]) - db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") - db["two_indexes"].create_index(["name"]) - db["two_indexes"].create_index(["species"]) + db.table("one_index").insert({"id": 1, "name": "Cleo"}, pk="id") + db.table("one_index").create_index(["name"]) + db.table("two_indexes").insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") + db.table("two_indexes").create_index(["name"]) + db.table("two_indexes").create_index(["species"]) result = CliRunner().invoke(cli.cli, ["analyze", db_path] + options) assert result.exit_code == 0 - assert list(db["sqlite_stat1"].rows) == expected + assert list(db.table("sqlite_stat1").rows) == expected def test_rename_table(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["one"].insert({"id": 1, "name": "Cleo"}, pk="id") + db.table("one").insert({"id": 1, "name": "Cleo"}, pk="id") # First try a non-existent table result_error = CliRunner().invoke( cli.cli, @@ -2782,7 +2786,7 @@ def test_rename_table(tmpdir): catch_exceptions=False, ) assert result_error2.exit_code == 0 - previous_columns = db["one"].columns_dict + previous_columns = db.table("one").columns_dict # Now try for a table that exists result = CliRunner().invoke( cli.cli, @@ -2790,13 +2794,13 @@ def test_rename_table(tmpdir): catch_exceptions=False, ) assert result.exit_code == 0 - assert db["two"].columns_dict == previous_columns + assert db.table("two").columns_dict == previous_columns def test_duplicate_table(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["one"].insert({"id": 1, "name": "Cleo"}, pk="id") + db.table("one").insert({"id": 1, "name": "Cleo"}, pk="id") # First try a non-existent table result_error = CliRunner().invoke( cli.cli, @@ -2819,8 +2823,8 @@ def test_duplicate_table(tmpdir): catch_exceptions=False, ) assert result.exit_code == 0 - assert db["one"].columns_dict == db["two"].columns_dict - assert list(db["one"].rows) == list(db["two"].rows) + assert db.table("one").columns_dict == db.table("two").columns_dict + assert list(db.table("one").rows) == list(db.table("two").rows) @pytest.mark.skipif(not _has_compiled_ext(), reason="Requires compiled ext.c") @@ -2863,9 +2867,9 @@ def test_create_table_strict(strict): + (["--strict"] if strict else []), ) assert result.exit_code == 0 - assert db["items"].strict == strict or not db.supports_strict + assert db.table("items").strict == strict or not db.supports_strict # Should have a floating point column - assert db["items"].columns_dict == {"id": int, "w": float} + assert db.table("items").columns_dict == {"id": int, "w": float} @pytest.mark.parametrize("method", ("insert", "upsert")) @@ -2880,12 +2884,12 @@ def test_insert_upsert_strict(tmpdir, method, strict): ) assert result.exit_code == 0 db = Database(db_path) - assert db["items"].strict == strict or not db.supports_strict + assert db.table("items").strict == strict or not db.supports_strict def test_extract_bad_column_clean_error(db_path): db = Database(db_path) - db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") + db.table("trees").insert({"id": 1, "species": "Palm"}, pk="id") result = CliRunner().invoke(cli.cli, ["extract", db_path, "trees", "nope"]) assert result.exit_code == 1 assert result.exception is None or isinstance(result.exception, SystemExit) @@ -2894,7 +2898,7 @@ def test_extract_bad_column_clean_error(db_path): def test_extract_view_clean_error(db_path): db = Database(db_path) - db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") + db.table("trees").insert({"id": 1, "species": "Palm"}, pk="id") db.create_view("v", "select * from trees") result = CliRunner().invoke(cli.cli, ["extract", db_path, "v", "species"]) assert result.exit_code == 1 diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py index 932269b..24889b3 100644 --- a/tests/test_cli_bulk.py +++ b/tests/test_cli_bulk.py @@ -13,7 +13,7 @@ from sqlite_utils import Database, cli def test_db_and_path(tmpdir): db_path = str(pathlib.Path(tmpdir) / "data.db") db = Database(db_path) - db["example"].insert_all( + db.table("example").insert_all( [ {"id": 1, "name": "One"}, {"id": 2, "name": "Two"}, @@ -44,7 +44,7 @@ def test_cli_bulk(test_db_and_path): {"id": 2, "name": "Two"}, {"id": 3, "name": "THREE"}, {"id": 4, "name": "FOUR"}, - ] == list(db["example"].rows) + ] == list(db.table("example").rows) def test_cli_bulk_multiple_functions(test_db_and_path): @@ -70,7 +70,7 @@ def test_cli_bulk_multiple_functions(test_db_and_path): {"id": 2, "name": "Two"}, {"id": 3, "name": "THREE"}, {"id": 4, "name": "FOUR"}, - ] == list(db["example"].rows) + ] == list(db.table("example").rows) def test_cli_bulk_batch_size(test_db_and_path): @@ -95,13 +95,13 @@ def test_cli_bulk_batch_size(test_db_and_path): proc.stdin.write(b'{"id": 3, "name": "Three"}\n\n') proc.stdin.flush() time.sleep(1) - assert db["example"].count == 2 + assert db.table("example").count == 2 # Writing another should trigger a commit: proc.stdin.write(b'{"id": 4, "name": "Four"}\n\n') proc.stdin.flush() time.sleep(1) - assert db["example"].count == 4 + assert db.table("example").count == 4 proc.stdin.close() proc.wait() diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 65543b1..1101f0f 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -12,7 +12,7 @@ from sqlite_utils import cli @pytest.fixture def test_db_and_path(fresh_db_and_path): db, db_path = fresh_db_and_path - db["example"].insert_all( + db.table("example").insert_all( [ {"id": 1, "dt": "5th October 2019 12:04"}, {"id": 2, "dt": "6th October 2019 00:05:06"}, @@ -47,12 +47,12 @@ def fresh_db_and_path(tmpdir): ) def test_convert_code(fresh_db_and_path, code): db, db_path = fresh_db_and_path - db["t"].insert({"text": "October"}) + db.table("t").insert({"text": "October"}) result = CliRunner().invoke( cli.cli, ["convert", db_path, "t", "text", code], catch_exceptions=False ) assert result.exit_code == 0, result.output - value = next(iter(db["t"].rows))["text"] + value = next(iter(db.table("t").rows))["text"] assert value == "Spooktober" @@ -65,7 +65,7 @@ def test_convert_code(fresh_db_and_path, code): ) def test_convert_code_errors(fresh_db_and_path, bad_code): db, db_path = fresh_db_and_path - db["t"].insert({"text": "October"}) + db.table("t").insert({"text": "October"}) result = CliRunner().invoke( cli.cli, ["convert", db_path, "t", "text", bad_code], catch_exceptions=False ) @@ -93,12 +93,12 @@ def test_convert_import(test_db_and_path): {"id": 2, "dt": "6th OXXober 2019 00:05:06"}, {"id": 3, "dt": ""}, {"id": 4, "dt": None}, - ] == list(db["example"].rows) + ] == list(db.table("example").rows) def test_convert_import_nested(fresh_db_and_path): db, db_path = fresh_db_and_path - db["example"].insert({"xml": ''}) + db.table("example").insert({"xml": ''}) result = CliRunner().invoke( cli.cli, [ @@ -114,7 +114,7 @@ def test_convert_import_nested(fresh_db_and_path): assert result.exit_code == 0, result.output assert [ {"xml": "Cleo"}, - ] == list(db["example"].rows) + ] == list(db.table("example").rows) def test_convert_dryrun(test_db_and_path): @@ -152,7 +152,7 @@ def test_convert_dryrun(test_db_and_path): "Would affect 4 rows" ) # But it should not have actually modified the table data - assert list(db["example"].rows) == [ + assert list(db.table("example").rows) == [ {"id": 1, "dt": "5th October 2019 12:04"}, {"id": 2, "dt": "6th October 2019 00:05:06"}, {"id": 3, "dt": ""}, @@ -269,7 +269,7 @@ def test_convert_output_column(test_db_and_path, drop): if drop: for row in expected: del row["dt"] - assert list(db["example"].rows) == expected + assert list(db.table("example").rows) == expected @pytest.mark.parametrize( @@ -352,7 +352,7 @@ def test_convert_output_error(test_db_and_path, options, expected_error): @pytest.mark.parametrize("drop", (True, False)) def test_convert_multi(fresh_db_and_path, drop): db, db_path = fresh_db_and_path - db["creatures"].insert_all( + db.table("creatures").insert_all( [ {"id": 1, "name": "Simon"}, {"id": 2, "name": "Cleo"}, @@ -378,12 +378,12 @@ def test_convert_multi(fresh_db_and_path, drop): if drop: for row in expected: del row["name"] - assert list(db["creatures"].rows) == expected + assert list(db.table("creatures").rows) == expected def test_convert_multi_complex_column_types(fresh_db_and_path): db, db_path = fresh_db_and_path - db["rows"].insert_all( + db.table("rows").insert_all( [ {"id": 1}, {"id": 2}, @@ -412,7 +412,7 @@ def test_convert_multi_complex_column_types(fresh_db_and_path): ], ) assert result.exit_code == 0, result.output - assert list(db["rows"].rows) == [ + assert list(db.table("rows").rows) == [ {"id": 1, "is_str": "", "is_float": 1.2, "is_int": None, "is_bytes": None}, {"id": 2, "is_str": None, "is_float": 1.0, "is_int": 12, "is_bytes": None}, { @@ -424,7 +424,7 @@ def test_convert_multi_complex_column_types(fresh_db_and_path): }, {"id": 4, "is_str": None, "is_float": None, "is_int": None, "is_bytes": None}, ] - assert db["rows"].schema == ( + assert db.table("rows").schema == ( 'CREATE TABLE "rows" (\n' ' "id" INTEGER PRIMARY KEY\n' ', "is_str" TEXT, "is_float" REAL, "is_int" INTEGER, "is_bytes" BLOB)' @@ -435,7 +435,7 @@ def test_convert_multi_complex_column_types(fresh_db_and_path): def test_recipe_jsonsplit(tmpdir, delimiter): db_path = str(pathlib.Path(tmpdir) / "data.db") db = sqlite_utils.Database(db_path) - db["example"].insert_all( + db.table("example").insert_all( [ {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])}, {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])}, @@ -448,7 +448,7 @@ def test_recipe_jsonsplit(tmpdir, delimiter): args = ["convert", db_path, "example", "tags", code] result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0, result.output - assert list(db["example"].rows) == [ + assert list(db.table("example").rows) == [ {"id": 1, "tags": '["foo", "bar"]'}, {"id": 2, "tags": '["bar", "baz"]'}, ] @@ -464,7 +464,7 @@ def test_recipe_jsonsplit(tmpdir, delimiter): ) def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array): db, db_path = fresh_db_and_path - db["example"].insert_all( + db.table("example").insert_all( [ {"id": 1, "records": "1,2,3"}, ], @@ -476,13 +476,13 @@ def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array): args = ["convert", db_path, "example", "records", code] result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0, result.output - assert json.loads(db["example"].get(1)["records"]) == expected_array + assert json.loads(db.table("example").get(1)["records"]) == expected_array @pytest.mark.parametrize("drop", (True, False)) def test_recipe_jsonsplit_output(fresh_db_and_path, drop): db, db_path = fresh_db_and_path - db["example"].insert_all( + db.table("example").insert_all( [ {"id": 1, "records": "1,2,3"}, ], @@ -501,7 +501,7 @@ def test_recipe_jsonsplit_output(fresh_db_and_path, drop): } if drop: del expected["records"] - assert db["example"].get(1) == expected + assert db.table("example").get(1) == expected def test_cannot_use_drop_without_multi_or_output(fresh_db_and_path): @@ -558,7 +558,7 @@ def test_convert_where(test_db_and_path): ], ) assert result.exit_code == 0, result.output - assert list(db["example"].rows) == [ + assert list(db.table("example").rows) == [ {"id": 1, "dt": "5th October 2019 12:04"}, {"id": 2, "dt": "6TH OCTOBER 2019 00:05:06"}, {"id": 3, "dt": ""}, @@ -568,7 +568,7 @@ def test_convert_where(test_db_and_path): def test_convert_where_multi(fresh_db_and_path): db, db_path = fresh_db_and_path - db["names"].insert_all( + db.table("names").insert_all( [{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Bants"}], pk="id" ) result = CliRunner().invoke( @@ -588,7 +588,7 @@ def test_convert_where_multi(fresh_db_and_path): ], ) assert result.exit_code == 0, result.output - assert list(db["names"].rows) == [ + assert list(db.table("names").rows) == [ {"id": 1, "name": "Cleo", "upper": None}, {"id": 2, "name": "Bants", "upper": "BANTS"}, ] @@ -596,7 +596,7 @@ def test_convert_where_multi(fresh_db_and_path): def test_convert_code_standard_input(fresh_db_and_path): db, db_path = fresh_db_and_path - db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + db.table("names").insert_all([{"id": 1, "name": "Cleo"}], pk="id") result = CliRunner().invoke( cli.cli, [ @@ -609,27 +609,27 @@ def test_convert_code_standard_input(fresh_db_and_path): input="value.upper()", ) assert result.exit_code == 0, result.output - assert list(db["names"].rows) == [ + assert list(db.table("names").rows) == [ {"id": 1, "name": "CLEO"}, ] def test_convert_hyphen_workaround(fresh_db_and_path): db, db_path = fresh_db_and_path - db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + db.table("names").insert_all([{"id": 1, "name": "Cleo"}], pk="id") result = CliRunner().invoke( cli.cli, ["convert", db_path, "names", "name", '"-"'], ) assert result.exit_code == 0, result.output - assert list(db["names"].rows) == [ + assert list(db.table("names").rows) == [ {"id": 1, "name": "-"}, ] def test_convert_initialization_pattern(fresh_db_and_path): db, db_path = fresh_db_and_path - db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + db.table("names").insert_all([{"id": 1, "name": "Cleo"}], pk="id") result = CliRunner().invoke( cli.cli, [ @@ -642,7 +642,7 @@ def test_convert_initialization_pattern(fresh_db_and_path): input="import random\nrandom.seed(1)\ndef convert(value): return random.randint(0, 100)", ) assert result.exit_code == 0, result.output - assert list(db["names"].rows) == [ + assert list(db.table("names").rows) == [ {"id": 1, "name": "17"}, ] @@ -657,13 +657,13 @@ def test_convert_handles_falsey_values(fresh_db_and_path): "x", "-", ] - db["t"].insert_all([{"x": 0}, {"x": 1}]) - assert db["t"].get(1)["x"] == 0 - assert db["t"].get(2)["x"] == 1 + db.table("t").insert_all([{"x": 0}, {"x": 1}]) + assert db.table("t").get(1)["x"] == 0 + assert db.table("t").get(2)["x"] == 1 result = CliRunner().invoke(cli.cli, args, input="value + 1") assert result.exit_code == 0, result.output - assert db["t"].get(1)["x"] == 1 - assert db["t"].get(2)["x"] == 2 + assert db.table("t").get(1)["x"] == 1 + assert db.table("t").get(2)["x"] == 2 @pytest.mark.parametrize( @@ -684,7 +684,7 @@ def test_convert_callable_reference(test_db_and_path, code): cli.cli, ["convert", db_path, "example", "dt", code], catch_exceptions=False ) assert result.exit_code == 0, result.output - rows = list(db["example"].rows) + rows = list(db.table("example").rows) assert rows[0]["dt"] == "2019-10-05" assert rows[1]["dt"] == "2019-10-06" assert rows[2]["dt"] == "" @@ -694,7 +694,7 @@ def test_convert_callable_reference(test_db_and_path, code): def test_convert_callable_reference_with_import(fresh_db_and_path): """Test callable reference from an imported module""" db, db_path = fresh_db_and_path - db["example"].insert({"id": 1, "data": '{"name": "test"}'}) + db.table("example").insert({"id": 1, "data": '{"name": "test"}'}) result = CliRunner().invoke( cli.cli, [ @@ -710,5 +710,5 @@ def test_convert_callable_reference_with_import(fresh_db_and_path): ) assert result.exit_code == 0, result.output # json.loads returns a dict, which sqlite stores as JSON string - row = db["example"].get(1) + row = db.table("example").get(1) assert row["data"] == '{"name": "test"}' diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index eefb3fa..01e7e94 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -21,7 +21,7 @@ def test_insert_simple(tmpdir): ) db = Database(db_path) assert ["dogs"] == db.table_names() - assert [] == db["dogs"].indexes + assert [] == db.table("dogs").indexes def test_insert_from_stdin(tmpdir): @@ -96,7 +96,7 @@ def test_insert_with_primary_keys(db_path, tmpdir, args, expected_pks): Database(db_path).query("select * from dogs") ) db = Database(db_path) - assert db["dogs"].pks == expected_pks + assert db.table("dogs").pks == expected_pks def test_insert_multiple_with_primary_key(db_path, tmpdir): @@ -110,7 +110,7 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir): assert result.exit_code == 0 db = Database(db_path) assert dogs == list(db.query("select * from dogs order by id")) - assert ["id"] == db["dogs"].pks + assert ["id"] == db.table("dogs").pks def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): @@ -127,7 +127,7 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): assert result.exit_code == 0 db = Database(db_path) assert dogs == list(db.query("select * from dogs order by breed, id")) - assert {"breed", "id"} == set(db["dogs"].pks) + assert {"breed", "id"} == set(db.table("dogs").pks) assert ( 'CREATE TABLE "dogs" (\n' ' "breed" TEXT,\n' @@ -136,7 +136,7 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): ' "age" INTEGER,\n' ' PRIMARY KEY ("id", "breed")\n' ")" - ) == db["dogs"].schema + ) == db.table("dogs").schema def test_insert_not_null_default(db_path, tmpdir): @@ -160,7 +160,7 @@ def test_insert_not_null_default(db_path, tmpdir): ' "name" TEXT NOT NULL,\n' " \"age\" INTEGER NOT NULL DEFAULT '1',\n" " \"score\" INTEGER DEFAULT '5'\n)" - ) == db["dogs"].schema + ) == db.table("dogs").schema def test_insert_binary_base64(db_path): @@ -191,7 +191,7 @@ def test_insert_newline_delimited(db_path): def test_insert_ignore(db_path, tmpdir): db = Database(db_path) - db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") json_path = str(tmpdir / "dogs.json") with open(json_path, "w") as fp: fp.write(json.dumps([{"id": 1, "name": "Bailey"}])) @@ -232,7 +232,7 @@ def test_insert_csv_tsv(content, options, db_path, tmpdir): catch_exceptions=False, ) assert result.exit_code == 0 - assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows) + assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db.table("data").rows) @pytest.mark.parametrize("empty_null", (True, False)) @@ -248,7 +248,7 @@ def test_insert_csv_empty_null(db_path, empty_null): ) assert result.exit_code == 0 db = Database(db_path) - assert [r for r in db["data"].rows] == [ + assert [r for r in db.table("data").rows] == [ {"foo": "1", "bar": None if empty_null else "", "baz": "cat"} ] @@ -302,7 +302,7 @@ def test_insert_replace(db_path, tmpdir): test_insert_multiple_with_primary_key(db_path, tmpdir) json_path = str(tmpdir / "insert-replace.json") db = Database(db_path) - assert db["dogs"].count == 20 + assert db.table("dogs").count == 20 insert_replace_dogs = [ {"id": 1, "name": "Insert replaced 1", "age": 4}, {"id": 2, "name": "Insert replaced 2", "age": 4}, @@ -314,7 +314,7 @@ def test_insert_replace(db_path, tmpdir): cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"] ) assert result.exit_code == 0, result.output - assert db["dogs"].count == 21 + assert db.table("dogs").count == 21 assert ( list(db.query("select * from dogs where id in (1, 2, 21) order by id")) == insert_replace_dogs @@ -377,7 +377,7 @@ def test_insert_alter(db_path, tmpdir): assert result.exit_code == 0, result.output # Soundness check the database itself db = Database(db_path) - assert {"foo": str, "n": int, "baz": int} == db["from_json_nl"].columns_dict + assert {"foo": str, "n": int, "baz": int} == db.table("from_json_nl").columns_dict assert [ {"foo": "bar", "n": 1, "baz": None}, {"foo": "baz", "n": 2, "baz": None}, @@ -387,8 +387,8 @@ def test_insert_alter(db_path, tmpdir): def test_insert_analyze(db_path): db = Database(db_path) - db["rows"].insert({"foo": "x", "n": 3}) - db["rows"].create_index(["n"]) + db.table("rows").insert({"foo": "x", "n": 3}) + db.table("rows").create_index(["n"]) assert "sqlite_stat1" not in db.table_names() result = CliRunner().invoke( cli.cli, @@ -583,7 +583,7 @@ def test_insert_streaming_batch_size_1(db_path): def try_until(expected): tries = 0 while True: - rows = list(Database(db_path)["rows"].rows) + rows = list(Database(db_path).table("rows").rows) if rows == expected: return tries += 1 @@ -615,13 +615,13 @@ def test_insert_csv_headers_only(tmpdir): assert result.exit_code == 0 # Table should not exist since there were no data rows db = Database(db_path) - assert not db["data"].exists() + assert not db.table("data").exists() def test_insert_into_view_errors(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["t"].insert({"id": 1}) + db.table("t").insert({"id": 1}) db.create_view("v", "select * from t") db.close() result = CliRunner().invoke( @@ -637,7 +637,7 @@ def test_insert_csv_detect_types_leaves_existing_table_alone(db_path): # table would rewrite its column types and corrupt data such as # TEXT zip codes with leading zeros db = Database(db_path) - db["places"].insert({"name": "Boston", "zip": "01234"}) + db.table("places").insert({"name": "Boston", "zip": "01234"}) result = CliRunner().invoke( cli.cli, ["insert", db_path, "places", "-", "--csv"], @@ -645,8 +645,8 @@ def test_insert_csv_detect_types_leaves_existing_table_alone(db_path): input="name,zip\nSF,94107", ) assert result.exit_code == 0, result.output - assert db["places"].columns_dict["zip"] is str - assert list(db["places"].rows) == [ + assert db.table("places").columns_dict["zip"] is str + assert list(db.table("places").rows) == [ {"name": "Boston", "zip": "01234"}, {"name": "SF", "zip": "94107"}, ] @@ -662,7 +662,7 @@ def test_insert_csv_detect_types_new_table(db_path): ) assert result.exit_code == 0, result.output db = Database(db_path) - assert db["data"].columns_dict == {"name": str, "age": int, "weight": float} + assert db.table("data").columns_dict == {"name": str, "age": int, "weight": float} @pytest.mark.parametrize( @@ -708,13 +708,13 @@ def test_insert_upsert_csv_type_overrides_detected_types( expected_columns = {"zipcode": str, "score": float} if command == "upsert": expected_columns = {"id": int, **expected_columns} - assert db["places"].columns_dict == expected_columns - assert list(db["places"].rows) == [expected_row] + assert db.table("places").columns_dict == expected_columns + assert list(db.table("places").rows) == [expected_row] def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): db = Database(db_path) - db["places"].insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id") + db.table("places").insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id") result = CliRunner().invoke( cli.cli, ["upsert", db_path, "places", "-", "--csv", "--pk", "id"], @@ -722,15 +722,15 @@ def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): input="id,name,zip\n2,SF,94107", ) assert result.exit_code == 0, result.output - assert db["places"].columns_dict["zip"] is str - assert db["places"].get(1)["zip"] == "01234" + assert db.table("places").columns_dict["zip"] is str + assert db.table("places").get(1)["zip"] == "01234" def test_insert_invalid_pk_clean_error(db_path): # An invalid --pk against an existing table should be a clean CLI # error, not a raw InvalidColumns traceback db = Database(db_path) - db["t"].insert({"a": 1}) + db.table("t").insert({"a": 1}) result = CliRunner().invoke( cli.cli, ["insert", db_path, "t", "-", "--pk", "badcol"], @@ -765,8 +765,8 @@ def test_insert_code(tmpdir, code): ) assert result.exit_code == 0, result.output db = Database(db_path) - assert db["creatures"].pks == ["id"] - assert list(db["creatures"].rows) == [ + assert db.table("creatures").pks == ["id"] + assert list(db.table("creatures").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 2, "name": "Suna"}, ] @@ -782,7 +782,7 @@ def test_insert_code_from_file(tmpdir): ["insert", db_path, "creatures", "--code", code_path], ) assert result.exit_code == 0, result.output - assert list(Database(db_path)["creatures"].rows) == [ + assert list(Database(db_path).table("creatures").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 2, "name": "Suna"}, ] @@ -791,7 +791,7 @@ def test_insert_code_from_file(tmpdir): def test_upsert_code(tmpdir): db_path = str(tmpdir / "dogs.db") db = Database(db_path) - db["creatures"].insert_all( + db.table("creatures").insert_all( [{"id": 1, "name": "old"}, {"id": 2, "name": "Suna"}], pk="id" ) result = CliRunner().invoke( @@ -799,7 +799,7 @@ def test_upsert_code(tmpdir): ["upsert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--pk", "id"], ) assert result.exit_code == 0, result.output - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 2, "name": "Suna"}, ] @@ -858,7 +858,9 @@ def test_insert_code_single_dict(tmpdir): ], ) assert result.exit_code == 0, result.output - assert list(Database(db_path)["creatures"].rows) == [{"id": 1, "name": "Cleo"}] + assert list(Database(db_path).table("creatures").rows) == [ + {"id": 1, "name": "Cleo"} + ] def test_insert_code_not_iterable(tmpdir): diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index 4fb4fb3..445f50b 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -228,7 +228,7 @@ def test_memory_save(tmpdir, extra_args): ) assert result.exit_code == 0 db = Database(save_to) - assert list(db["stdin"].rows) == [ + assert list(db.table("stdin").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 2, "name": "Bants"}, ] diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index f49ef10..7439887 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -13,11 +13,11 @@ m = Migrations("hello") @m() def foo(db): - db["foo"].insert({"hello": "world"}) + db.table("foo").insert({"hello": "world"}) @m() def bar(db): - db["bar"].insert({"hello": "world"}) + db.table("bar").insert({"hello": "world"}) """ @@ -42,21 +42,21 @@ creatures = Migrations("creatures") @creatures() def create_table(db): - db["creatures"].insert({"name": "Cleo"}) + db.table("creatures").insert({"name": "Cleo"}) @creatures() def add_weight(db): - db["creature_weights"].insert({"weight": 4.2}) + db.table("creature_weights").insert({"weight": 4.2}) sales = Migrations("sales") @sales() def create_table(db): - db["sales"].insert({"id": 1}) + db.table("sales").insert({"id": 1}) @sales() def add_weight(db): - db["sales_weights"].insert({"weight": 10}) + db.table("sales_weights").insert({"weight": 10}) """, "utf-8", ) @@ -99,10 +99,10 @@ def test_basic(two_migrations, arg): assert " Pending:\n (none)" in list_output db = sqlite_utils.Database(db_path) - assert db["foo"].exists() - assert db["bar"].exists() - assert db["_sqlite_migrations"].exists() - rows = list(db["_sqlite_migrations"].rows) + assert db.table("foo").exists() + assert db.table("bar").exists() + assert db.table("_sqlite_migrations").exists() + rows = list(db.table("_sqlite_migrations").rows) assert len(rows) == 2 assert rows[0]["name"] == "foo" assert rows[1]["name"] == "bar" @@ -113,13 +113,13 @@ def test_list_same_migration_names_in_different_sets(capsys): @applied(name="foo") def applied_foo(db): - db["applied"].insert({"hello": "world"}) + db.table("applied").insert({"hello": "world"}) pending = sqlite_utils.Migrations("pending") @pending(name="foo") def pending_foo(db): - db["pending"].insert({"hello": "world"}) + db.table("pending").insert({"hello": "world"}) db = sqlite_utils.Database(memory=True) applied.apply(db) @@ -144,7 +144,7 @@ m = Migrations("hello") @m() def foo(db): - db["dogs"].insert({"id": 1, "name": "Cleo"}) + db.table("dogs").insert({"id": 1, "name": "Cleo"}) """, "utf-8", ) @@ -184,9 +184,9 @@ Schema after: new_migration = """ @m() def bar(db): - db["dogs"].add_column("age", int) - db["dogs"].add_column("weight", float) - db["dogs"].transform() + db.table("dogs").add_column("age", int) + db.table("dogs").add_column("weight", float) + db.table("dogs").transform() """ migrations_py.write_text(migrations_py.read_text("utf-8") + new_migration) @@ -224,8 +224,8 @@ def test_stop_before(two_migrations): ) assert result.exit_code == 0 db = sqlite_utils.Database(db_path) - assert db["foo"].exists() - assert not db["bar"].exists() + assert db.table("foo").exists() + assert not db.table("bar").exists() def test_stop_before_multiple_sets_unqualified(two_migrations): @@ -239,7 +239,7 @@ m = Migrations("hello2") @m() def foo(db): - db["foo"].insert({"hello": "world"}) + db.table("foo").insert({"hello": "world"}) """, "utf-8", ) @@ -257,7 +257,7 @@ def foo(db): assert result.exit_code == 0, result.output db = sqlite_utils.Database(db_path) assert db.table_names() == ["_sqlite_migrations"] - assert list(db["_sqlite_migrations"].rows) == [] + assert list(db.table("_sqlite_migrations").rows) == [] def test_stop_before_qualified_only_affects_named_set(two_sets_same_migration_name): @@ -275,10 +275,10 @@ def test_stop_before_qualified_only_affects_named_set(two_sets_same_migration_na ) assert result.exit_code == 0, result.output db = sqlite_utils.Database(db_path) - assert db["creatures"].exists() - assert not db["creature_weights"].exists() - assert db["sales"].exists() - assert db["sales_weights"].exists() + assert db.table("creatures").exists() + assert not db.table("creature_weights").exists() + assert db.table("sales").exists() + assert db.table("sales_weights").exists() def test_stop_before_multiple_qualified(two_sets_same_migration_name): @@ -298,10 +298,10 @@ def test_stop_before_multiple_qualified(two_sets_same_migration_name): ) assert result.exit_code == 0, result.output db = sqlite_utils.Database(db_path) - assert db["creatures"].exists() - assert not db["creature_weights"].exists() - assert db["sales"].exists() - assert not db["sales_weights"].exists() + assert db.table("creatures").exists() + assert not db.table("creature_weights").exists() + assert db.table("sales").exists() + assert not db.table("sales_weights").exists() LEGACY_MIGRATIONS = """ @@ -331,7 +331,7 @@ class LegacyMigrations: return fn def ensure_migrations_table(self, db): - db[self.migrations_table].create( + db.table(self.migrations_table).create( {"migration_set": str, "name": str, "applied_at": str}, pk=("migration_set", "name"), if_not_exists=True, @@ -341,7 +341,7 @@ class LegacyMigrations: self.ensure_migrations_table(db) return [ _Applied(row["name"], row["applied_at"]) - for row in db[self.migrations_table].rows_where( + for row in db.table(self.migrations_table).rows_where( "migration_set = ?", [self.name] ) ] @@ -355,7 +355,7 @@ class LegacyMigrations: if migration.name == stop_before: return migration.fn(db) - db[self.migrations_table].insert( + db.table(self.migrations_table).insert( { "migration_set": self.name, "name": migration.name, @@ -369,11 +369,11 @@ legacy = LegacyMigrations("legacy_set") @legacy def first(db): - db["first"].insert({"hello": "world"}) + db.table("first").insert({"hello": "world"}) @legacy def second(db): - db["second"].insert({"hello": "world"}) + db.table("second").insert({"hello": "world"}) """ @@ -446,11 +446,11 @@ def test_list_does_not_upgrade_legacy_migrations_table(two_migrations): path, _ = two_migrations db_path = str(path / "test.db") db = sqlite_utils.Database(db_path) - db["_sqlite_migrations"].create( + db.table("_sqlite_migrations").create( {"migration_set": str, "name": str, "applied_at": str}, pk=("migration_set", "name"), ) - db["_sqlite_migrations"].insert( + db.table("_sqlite_migrations").insert( {"migration_set": "hello", "name": "foo", "applied_at": "x"} ) db.close() @@ -462,7 +462,7 @@ def test_list_does_not_upgrade_legacy_migrations_table(two_migrations): assert "foo - x" in result.output # --list must not perform the one-way legacy schema upgrade db2 = sqlite_utils.Database(db_path) - assert db2["_sqlite_migrations"].pks == ["migration_set", "name"] + assert db2.table("_sqlite_migrations").pks == ["migration_set", "name"] db2.close() @@ -485,7 +485,7 @@ def test_stop_before_applied_migration_errors(two_migrations): assert result.exit_code != 0 assert "already been applied" in result.output db = sqlite_utils.Database(db_path) - assert not db["bar"].exists() + assert not db.table("bar").exists() def test_list_with_legacy_class_is_read_only(tmpdir): @@ -496,7 +496,7 @@ def test_list_with_legacy_class_is_read_only(tmpdir): (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8") db_path = str(path / "test.db") db = sqlite_utils.Database(db_path) - db["existing"].insert({"id": 1}) + db.table("existing").insert({"id": 1}) db.close() result = CliRunner().invoke( sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"] diff --git a/tests/test_column_affinity.py b/tests/test_column_affinity.py index fa23345..8c619e1 100644 --- a/tests/test_column_affinity.py +++ b/tests/test_column_affinity.py @@ -43,4 +43,4 @@ def test_column_affinity(column_def, expected_type): @pytest.mark.parametrize("column_def,expected_type", EXAMPLES) def test_columns_dict(fresh_db, column_def, expected_type): fresh_db.execute(f"create table foo (col {column_def})") - assert {"col": expected_type} == fresh_db["foo"].columns_dict + assert {"col": expected_type} == fresh_db.table("foo").columns_dict diff --git a/tests/test_column_casing.py b/tests/test_column_casing.py index ce11345..b3f03c9 100644 --- a/tests/test_column_casing.py +++ b/tests/test_column_casing.py @@ -13,14 +13,14 @@ from sqlite_utils.db import ForeignKey def test_insert_populates_last_pk_case_insensitively(fresh_db): - books = fresh_db["books"] + books = fresh_db.table("books") books.create({"Id": int, "Title": str}, pk="Id") books.insert({"Id": 1, "Title": "One"}, pk="id") assert books.last_pk == 1 def test_insert_populates_last_pk_compound_pk_case_insensitively(fresh_db): - books = fresh_db["books"] + books = fresh_db.table("books") books.create({"Author": str, "Position": int, "Title": str}) books.insert( {"Author": "Sue", "Position": 1, "Title": "One"}, pk=("author", "position") @@ -31,7 +31,7 @@ def test_insert_populates_last_pk_compound_pk_case_insensitively(fresh_db): @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_upsert_pk_case_differs_from_schema(use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) - books = db["books"] + books = db.table("books") books.create({"Id": int, "Title": str}, pk="Id") books.insert({"Id": 1, "Title": "One"}) books.upsert({"id": 1, "title": "Won"}, pk="id") @@ -43,7 +43,7 @@ def test_upsert_pk_case_differs_from_schema(use_old_upsert): def test_upsert_record_key_case_differs_from_pk(use_old_upsert): # all_columns comes from the record keys, pk= from the caller db = Database(memory=True, use_old_upsert=use_old_upsert) - books = db["books"] + books = db.table("books") books.create({"Id": int, "Title": str}, pk="Id") books.upsert({"ID": 1, "Title": "One"}, pk="id") assert list(books.rows) == [{"Id": 1, "Title": "One"}] @@ -52,7 +52,7 @@ def test_upsert_record_key_case_differs_from_pk(use_old_upsert): def test_upsert_inferred_pk_case_differs_from_record_keys(fresh_db): # pk is inferred from the existing schema as "Id", records use "id" - books = fresh_db["books"] + books = fresh_db.table("books") books.create({"Id": int, "Title": str}, pk="Id") books.upsert({"id": 1, "title": "One"}) assert list(books.rows) == [{"Id": 1, "Title": "One"}] @@ -60,7 +60,7 @@ def test_upsert_inferred_pk_case_differs_from_record_keys(fresh_db): def test_upsert_list_mode_pk_case_insensitive(fresh_db): - books = fresh_db["books"] + books = fresh_db.table("books") books.create({"Id": int, "Title": str}, pk="Id") books.upsert_all([["id", "title"], [1, "One"]], pk="Id") assert list(books.rows) == [{"Id": 1, "Title": "One"}] @@ -68,84 +68,84 @@ def test_upsert_list_mode_pk_case_insensitive(fresh_db): def test_lookup_pk_case_insensitive(fresh_db): - fresh_db["species"].create({"ID": int, "Name": str}, pk="ID") - fresh_db["species"].insert({"ID": 5, "Name": "Palm"}) - fresh_db["species"].create_index(["Name"], unique=True) - assert fresh_db["species"].lookup({"Name": "Palm"}, pk="id") == 5 + fresh_db.table("species").create({"ID": int, "Name": str}, pk="ID") + fresh_db.table("species").insert({"ID": 5, "Name": "Palm"}) + fresh_db.table("species").create_index(["Name"], unique=True) + assert fresh_db.table("species").lookup({"Name": "Palm"}, pk="id") == 5 def test_lookup_does_not_create_redundant_index(fresh_db): - fresh_db["species"].create({"id": int, "Name": str}, pk="id") - fresh_db["species"].create_index(["Name"], unique=True) - fresh_db["species"].lookup({"name": "Palm"}) - assert len(fresh_db["species"].indexes) == 1 + fresh_db.table("species").create({"id": int, "Name": str}, pk="id") + fresh_db.table("species").create_index(["Name"], unique=True) + fresh_db.table("species").lookup({"name": "Palm"}) + assert len(fresh_db.table("species").indexes) == 1 def test_create_table_transform_same_columns_different_case(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) - fresh_db["t"].insert({"Name": "Cleo", "Age": 5}) + fresh_db.table("t").create({"Name": str, "Age": int}) + fresh_db.table("t").insert({"Name": "Cleo", "Age": 5}) fresh_db.create_table("t", {"name": str, "age": int}, transform=True) # Schema casing is preserved - SQLite considers these the same columns - assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} - assert list(fresh_db["t"].rows) == [{"Name": "Cleo", "Age": 5}] + assert fresh_db.table("t").columns_dict == {"Name": str, "Age": int} + assert list(fresh_db.table("t").rows) == [{"Name": "Cleo", "Age": 5}] def test_create_table_transform_case_insensitive_with_changes(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db.table("t").create({"Name": str, "Age": int}) fresh_db.create_table("t", {"name": str, "age": str, "size": int}, transform=True) # age changed type, size added, Name untouched - assert fresh_db["t"].columns_dict == {"Name": str, "Age": str, "size": int} + assert fresh_db.table("t").columns_dict == {"Name": str, "Age": str, "size": int} def test_transform_types_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str, "Age": str}) - fresh_db["t"].transform(types={"age": int}) - assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} + fresh_db.table("t").create({"Name": str, "Age": str}) + fresh_db.table("t").transform(types={"age": int}) + assert fresh_db.table("t").columns_dict == {"Name": str, "Age": int} def test_transform_rename_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str}) - fresh_db["t"].transform(rename={"name": "title"}) - assert fresh_db["t"].columns_dict == {"title": str} + fresh_db.table("t").create({"Name": str}) + fresh_db.table("t").transform(rename={"name": "title"}) + assert fresh_db.table("t").columns_dict == {"title": str} def test_transform_drop_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) - fresh_db["t"].transform(drop=["name"]) - assert fresh_db["t"].columns_dict == {"Age": int} + fresh_db.table("t").create({"Name": str, "Age": int}) + fresh_db.table("t").transform(drop=["name"]) + assert fresh_db.table("t").columns_dict == {"Age": int} def test_transform_not_null_and_defaults_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) - fresh_db["t"].transform(not_null={"name"}, defaults={"age": 3}) - columns = {c.name: c for c in fresh_db["t"].columns} + fresh_db.table("t").create({"Name": str, "Age": int}) + fresh_db.table("t").transform(not_null={"name"}, defaults={"age": 3}) + columns = {c.name: c for c in fresh_db.table("t").columns} assert columns["Name"].notnull - assert fresh_db["t"].default_values == {"Age": 3} + assert fresh_db.table("t").default_values == {"Age": 3} def test_transform_pk_case_insensitive(fresh_db): - fresh_db["t"].create({"Id": int, "Name": str}) - fresh_db["t"].transform(pk="id") - assert fresh_db["t"].pks == ["Id"] - assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} + fresh_db.table("t").create({"Id": int, "Name": str}) + fresh_db.table("t").transform(pk="id") + assert fresh_db.table("t").pks == ["Id"] + assert fresh_db.table("t").columns_dict == {"Id": int, "Name": str} def test_transform_drop_foreign_keys_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create( + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create( {"id": int, "Parent_ID": int}, pk="id", foreign_keys=[("Parent_ID", "parent", "Id")], ) - fresh_db["child"].transform(drop_foreign_keys=["parent_id"]) - assert fresh_db["child"].foreign_keys == [] + fresh_db.table("child").transform(drop_foreign_keys=["parent_id"]) + assert fresh_db.table("child").foreign_keys == [] def test_add_foreign_key_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") - fresh_db["child"].add_foreign_key("parent_id", "parent", "id") - fks = fresh_db["child"].foreign_keys + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create({"id": int, "Parent_ID": int}, pk="id") + fresh_db.table("child").add_foreign_key("parent_id", "parent", "id") + fks = fresh_db.table("child").foreign_keys assert len(fks) == 1 # The foreign key should use the schema casing of the columns assert fks[0].column == "Parent_ID" @@ -153,79 +153,83 @@ def test_add_foreign_key_case_insensitive(fresh_db): def test_add_foreign_keys_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create({"id": int, "Parent_ID": int}, pk="id") fresh_db.add_foreign_keys([("child", "parent_id", "parent", "id")]) - fks = fresh_db["child"].foreign_keys + fks = fresh_db.table("child").foreign_keys assert len(fks) == 1 assert fks[0].column == "Parent_ID" assert fks[0].other_column == "Id" def test_add_foreign_key_detects_existing_case_insensitively(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create( + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create( {"id": int, "Parent_ID": int}, pk="id", foreign_keys=[("Parent_ID", "parent", "Id")], ) # ignore=True should treat this as already existing, not add a duplicate - fresh_db["child"].add_foreign_key("parent_id", "parent", "id", ignore=True) - assert len(fresh_db["child"].foreign_keys) == 1 + fresh_db.table("child").add_foreign_key("parent_id", "parent", "id", ignore=True) + assert len(fresh_db.table("child").foreign_keys) == 1 def test_add_column_fk_col_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create({"id": int}, pk="id") - fresh_db["child"].add_column("parent_id", int, fk="parent", fk_col="id") - fks = fresh_db["child"].foreign_keys + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create({"id": int}, pk="id") + fresh_db.table("child").add_column("parent_id", int, fk="parent", fk_col="id") + fks = fresh_db.table("child").foreign_keys assert len(fks) == 1 assert fks[0].other_column == "Id" def test_extract_case_insensitive(fresh_db): - fresh_db["trees"].insert({"id": 1, "Species": "Palm"}, pk="id") - fresh_db["trees"].extract("species") - assert fresh_db["trees"].columns_dict == {"id": int, "Species_id": int} - assert list(fresh_db["Species"].rows) == [{"id": 1, "Species": "Palm"}] + fresh_db.table("trees").insert({"id": 1, "Species": "Palm"}, pk="id") + fresh_db.table("trees").extract("species") + assert fresh_db.table("trees").columns_dict == {"id": int, "Species_id": int} + assert list(fresh_db.table("Species").rows) == [{"id": 1, "Species": "Palm"}] def test_convert_multi_case_insensitive(fresh_db): - fresh_db["t"].insert({"id": 1, "Name": "Cleo"}, pk="id") - fresh_db["t"].convert("name", lambda v: {"upper": v.upper()}, multi=True) - assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "upper": "CLEO"}] + fresh_db.table("t").insert({"id": 1, "Name": "Cleo"}, pk="id") + fresh_db.table("t").convert("name", lambda v: {"upper": v.upper()}, multi=True) + assert list(fresh_db.table("t").rows) == [ + {"id": 1, "Name": "Cleo", "upper": "CLEO"} + ] def test_convert_output_case_insensitive(fresh_db): - fresh_db["t"].insert({"id": 1, "Name": "Cleo", "Upper": None}, pk="id") - fresh_db["t"].convert("name", lambda v: v.upper(), output="upper") - assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "Upper": "CLEO"}] + fresh_db.table("t").insert({"id": 1, "Name": "Cleo", "Upper": None}, pk="id") + fresh_db.table("t").convert("name", lambda v: v.upper(), output="upper") + assert list(fresh_db.table("t").rows) == [ + {"id": 1, "Name": "Cleo", "Upper": "CLEO"} + ] def test_create_table_sql_pk_case_insensitive(fresh_db): - fresh_db["t"].create({"Id": int, "Name": str}, pk="id") + fresh_db.table("t").create({"Id": int, "Name": str}, pk="id") # Should not have created an extra lowercase "id" column - assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} - assert fresh_db["t"].pks == ["Id"] + assert fresh_db.table("t").columns_dict == {"Id": int, "Name": str} + assert fresh_db.table("t").pks == ["Id"] def test_create_table_not_null_and_defaults_case_insensitive(fresh_db): - fresh_db["t"].create( + fresh_db.table("t").create( {"Name": str, "Age": int}, not_null={"name"}, defaults={"age": 1} ) - columns = {c.name: c for c in fresh_db["t"].columns} + columns = {c.name: c for c in fresh_db.table("t").columns} assert columns["Name"].notnull - assert fresh_db["t"].default_values == {"Age": 1} + assert fresh_db.table("t").default_values == {"Age": 1} def test_create_table_foreign_keys_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create( + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create( {"id": int, "Parent_ID": int}, pk="id", foreign_keys=[("parent_id", "parent", "id")], ) - fks = fresh_db["child"].foreign_keys + fks = fresh_db.table("child").foreign_keys assert fks == [ ForeignKey( table="child", column="Parent_ID", other_table="parent", other_column="Id" diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 4282969..2d0a298 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -20,8 +20,8 @@ def test_recursive_triggers_off(): def test_memory_name(): db1 = Database(memory_name="shared") db2 = Database(memory_name="shared") - db1["dogs"].insert({"name": "Cleo"}) - assert list(db2["dogs"].rows) == [{"name": "Cleo"}] + db1.table("dogs").insert({"name": "Cleo"}) + assert list(db2.table("dogs").rows) == [{"name": "Cleo"}] def test_sqlite_version(): @@ -36,7 +36,7 @@ def test_sqlite_version(): def test_database_context_manager(tmpdir): path = str(tmpdir / "test.db") with Database(path) as db: - db["t"].insert({"id": 1}) + db.table("t").insert({"id": 1}) # Raw writes commit automatically too db.execute("insert into t (id) values (2)") # An explicitly opened transaction left uncommitted on purpose: @@ -47,7 +47,7 @@ def test_database_context_manager(tmpdir): db.execute("select 1") # ... and the open explicit transaction was rolled back, not committed db2 = Database(path) - assert [r["id"] for r in db2["t"].rows] == [1, 2] + assert [r["id"] for r in db2.table("t").rows] == [1, 2] db2.close() @@ -86,8 +86,8 @@ def test_legacy_transaction_control_connection_is_accepted(tmpdir): str(tmpdir / "test.db"), autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL ) db = Database(conn) - db["t"].insert({"id": 1}, pk="id") - assert [r["id"] for r in db["t"].rows] == [1] + db.table("t").insert({"id": 1}, pk="id") + assert [r["id"] for r in db.table("t").rows] == [1] db.close() diff --git a/tests/test_conversions.py b/tests/test_conversions.py index d70f5c8..bb58df4 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -1,17 +1,17 @@ def test_insert_conversion(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"foo": "bar"}, conversions={"foo": "upper(?)"}) assert [{"foo": "BAR"}] == list(table.rows) def test_insert_all_conversion(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all([{"foo": "bar"}], conversions={"foo": "upper(?)"}) assert [{"foo": "BAR"}] == list(table.rows) def test_upsert_conversion(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert({"id": 1, "foo": "bar"}, pk="id", conversions={"foo": "upper(?)"}) assert [{"id": 1, "foo": "BAR"}] == list(table.rows) table.upsert( @@ -21,7 +21,7 @@ def test_upsert_conversion(fresh_db): def test_upsert_all_conversion(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert_all( [{"id": 1, "foo": "bar"}], pk="id", conversions={"foo": "upper(?)"} ) @@ -29,7 +29,7 @@ def test_upsert_all_conversion(fresh_db): def test_update_conversion(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"id": 5, "foo": "bar"}, pk="id") table.update(5, {"foo": "baz"}, conversions={"foo": "upper(?)"}) assert [{"id": 5, "foo": "BAZ"}] == list(table.rows) diff --git a/tests/test_convert.py b/tests/test_convert.py index 879267a..1f9e9ed 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -27,7 +27,7 @@ from sqlite_utils.db import BadMultiValues ), ) def test_convert(fresh_db, columns, fn, expected): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"title": "Mixed Case", "abstract": "Abstract"}) table.convert(columns, fn) assert list(table.rows) == [expected] @@ -37,7 +37,7 @@ def test_convert(fresh_db, columns, fn, expected): "where,where_args", (("id > 1", None), ("id > :id", {"id": 1}), ("id > ?", [1])) ) def test_convert_where(fresh_db, where, where_args): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all( [ {"id": 1, "title": "One"}, @@ -53,7 +53,7 @@ def test_convert_where(fresh_db, where, where_args): def test_convert_handles_falsey_values(fresh_db): # Falsey values like 0 should be converted (issue #527) - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all([{"x": 0}, {"x": 1}]) assert table.get(1)["x"] == 0 assert table.get(2)["x"] == 1 @@ -70,14 +70,14 @@ def test_convert_handles_falsey_values(fresh_db): ), ) def test_convert_output(fresh_db, drop, expected): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"title": "Mixed Case"}) table.convert("title", lambda v: v.upper(), output="other", drop=drop) assert list(table.rows) == [expected] def test_convert_output_multiple_column_error(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") with pytest.raises(ValueError) as excinfo: table.convert(["title", "other"], lambda v: v, output="out") assert "output= can only be used with a single column" in str(excinfo.value) @@ -91,14 +91,14 @@ def test_convert_output_multiple_column_error(fresh_db): ), ) def test_convert_output_type(fresh_db, type, expected): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"number": "123"}) table.convert("number", lambda v: v, output="other", output_type=type, drop=True) assert list(table.rows) == [expected] def test_convert_multi(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"title": "Mixed Case"}) table.convert( "title", @@ -123,7 +123,7 @@ def test_convert_multi(fresh_db): def test_convert_multi_where(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all( [ {"id": 1, "title": "One"}, @@ -145,14 +145,14 @@ def test_convert_multi_where(fresh_db): def test_convert_multi_exception(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"title": "Mixed Case"}) with pytest.raises(BadMultiValues): table.convert("title", lambda v: v.upper(), multi=True) def test_convert_repeated(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") col = "num" table.insert({col: 1}) table.convert(col, lambda x: x * 2) diff --git a/tests/test_create.py b/tests/test_create.py index 40746bf..0af68a6 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -80,9 +80,10 @@ def test_create_table_compound_primary_key(fresh_db): @pytest.mark.parametrize("pk", ("id", ["id"])) def test_create_table_with_single_primary_key(fresh_db, pk): - fresh_db["foo"].insert({"id": 1}, pk=pk) + fresh_db.table("foo").insert({"id": 1}, pk=pk) assert ( - fresh_db["foo"].schema == 'CREATE TABLE "foo" (\n "id" INTEGER PRIMARY KEY\n)' + fresh_db.table("foo").schema + == 'CREATE TABLE "foo" (\n "id" INTEGER PRIMARY KEY\n)' ) @@ -159,7 +160,7 @@ def test_create_table_with_not_null(fresh_db): ), ) def test_create_table_from_example(fresh_db, example, expected_columns): - people_table = fresh_db["people"] + people_table = fresh_db.table("people") assert people_table.last_rowid is None assert people_table.last_pk is None people_table.insert(example) @@ -167,13 +168,13 @@ def test_create_table_from_example(fresh_db, example, expected_columns): assert people_table.last_pk == 1 assert ["people"] == fresh_db.table_names() assert expected_columns == [ - {"name": col.name, "type": col.type} for col in fresh_db["people"].columns + {"name": col.name, "type": col.type} for col in fresh_db.table("people").columns ] def test_create_table_from_example_with_compound_primary_keys(fresh_db): record = {"name": "Zhang", "group": "staff", "employee_id": 2} - table = fresh_db["people"].insert(record, pk=("group", "employee_id")) + table = fresh_db.table("people").insert(record, pk=("group", "employee_id")) assert ["group", "employee_id"] == table.pks assert record == table.get(("staff", 2)) @@ -184,7 +185,7 @@ def test_create_table_from_example_with_compound_primary_keys(fresh_db): @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_create_table_with_custom_columns(method_name, use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) - table = db["dogs"] + table = db.table("dogs") method = getattr(table, method_name) record = {"id": 1, "name": "Cleo", "age": "5"} if method_name.endswith("_all"): @@ -218,14 +219,16 @@ def test_create_table_column_order(fresh_db, use_table_factory): if use_table_factory: fresh_db.table("table", column_order=column_order).insert(row) else: - fresh_db["table"].insert(row, column_order=column_order) + fresh_db.table("table").insert(row, column_order=column_order) assert [ {"name": "abc", "type": "TEXT"}, {"name": "ccc", "type": "TEXT"}, {"name": "zzz", "type": "TEXT"}, {"name": "bbb", "type": "TEXT"}, {"name": "aaa", "type": "TEXT"}, - ] == [{"name": col.name, "type": col.type} for col in fresh_db["table"].columns] + ] == [ + {"name": col.name, "type": col.type} for col in fresh_db.table("table").columns + ] @pytest.mark.parametrize( @@ -261,8 +264,8 @@ def test_create_table_works_for_m2m_with_only_foreign_keys( fresh_db.table("one", pk="id").insert({"id": 1}) fresh_db.table("two", pk="id").insert({"id": 1}) else: - fresh_db["one"].insert({"id": 1}, pk="id") - fresh_db["two"].insert({"id": 1}, pk="id") + fresh_db.table("one").insert({"id": 1}, pk="id") + fresh_db.table("two").insert({"id": 1}, pk="id") row = {"one_id": 1, "two_id": 1} @@ -270,7 +273,7 @@ def test_create_table_works_for_m2m_with_only_foreign_keys( if use_table_factory: fresh_db.table("m2m", foreign_keys=foreign_key_specification).insert(row) else: - fresh_db["m2m"].insert(row, foreign_keys=foreign_key_specification) + fresh_db.table("m2m").insert(row, foreign_keys=foreign_key_specification) if expected_exception: with pytest.raises(expected_exception): @@ -281,7 +284,7 @@ def test_create_table_works_for_m2m_with_only_foreign_keys( assert [ {"name": "one_id", "type": "INTEGER"}, {"name": "two_id", "type": "INTEGER"}, - ] == [{"name": col.name, "type": col.type} for col in fresh_db["m2m"].columns] + ] == [{"name": col.name, "type": col.type} for col in fresh_db.table("m2m").columns] assert sorted( [ {"column": "one_id", "other_table": "one", "other_column": "id"}, @@ -295,7 +298,7 @@ def test_create_table_works_for_m2m_with_only_foreign_keys( "other_table": fk.other_table, "other_column": fk.other_column, } - for fk in fresh_db["m2m"].foreign_keys + for fk in fresh_db.table("m2m").foreign_keys ], key=lambda s: repr(s), ) @@ -322,7 +325,7 @@ def test_self_referential_foreign_key(fresh_db): def test_create_error_if_invalid_foreign_keys(fresh_db): with pytest.raises(AlterError): - fresh_db["one"].insert( + fresh_db.table("one").insert( {"id": 1, "ref_id": 3}, pk="id", foreign_keys=(("ref_id", "bad_table", "bad_column"),), @@ -331,7 +334,7 @@ def test_create_error_if_invalid_foreign_keys(fresh_db): def test_create_error_if_invalid_self_referential_foreign_keys(fresh_db): with pytest.raises(AlterError) as ex: - fresh_db["one"].insert( + fresh_db.table("one").insert( {"id": 1, "ref_id": 3}, pk="id", foreign_keys=(("ref_id", "one", "bad_column"),), @@ -397,41 +400,43 @@ def test_create_error_if_invalid_self_referential_foreign_keys(fresh_db): ) def test_add_column(fresh_db, col_name, col_type, not_null_default, expected_schema): fresh_db.create_table("dogs", {"name": str}) - assert fresh_db["dogs"].schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' - fresh_db["dogs"].add_column(col_name, col_type, not_null_default=not_null_default) - assert fresh_db["dogs"].schema == expected_schema + assert fresh_db.table("dogs").schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' + fresh_db.table("dogs").add_column( + col_name, col_type, not_null_default=not_null_default + ) + assert fresh_db.table("dogs").schema == expected_schema def test_add_foreign_key(fresh_db): - fresh_db["authors"].insert_all( + fresh_db.table("authors").insert_all( [{"id": 1, "name": "Sally"}, {"id": 2, "name": "Asheesh"}], pk="id" ) - fresh_db["books"].insert_all( + fresh_db.table("books").insert_all( [ {"title": "Hedgehogs of the world", "author_id": 1}, {"title": "How to train your wolf", "author_id": 2}, ] ) - assert [] == fresh_db["books"].foreign_keys - t = fresh_db["books"].add_foreign_key("author_id", "authors", "id") + assert [] == fresh_db.table("books").foreign_keys + t = fresh_db.table("books").add_foreign_key("author_id", "authors", "id") # Ensure it returned self: assert isinstance(t, Table) and t.name == "books" assert [ ForeignKey( table="books", column="author_id", other_table="authors", other_column="id" ) - ] == fresh_db["books"].foreign_keys + ] == fresh_db.table("books").foreign_keys def test_add_foreign_key_if_column_contains_space(fresh_db): - fresh_db["authors"].insert_all([{"id": 1, "name": "Sally"}], pk="id") - fresh_db["books"].insert_all( + fresh_db.table("authors").insert_all([{"id": 1, "name": "Sally"}], pk="id") + fresh_db.table("books").insert_all( [ {"title": "Hedgehogs of the world", "author id": 1}, ] ) - fresh_db["books"].add_foreign_key("author id", "authors", "id") - assert fresh_db["books"].foreign_keys == [ + fresh_db.table("books").add_foreign_key("author id", "authors", "id") + assert fresh_db.table("books").foreign_keys == [ ForeignKey( table="books", column="author id", other_table="authors", other_column="id" ) @@ -439,44 +444,44 @@ def test_add_foreign_key_if_column_contains_space(fresh_db): def test_add_foreign_key_error_if_column_does_not_exist(fresh_db): - fresh_db["books"].insert( + fresh_db.table("books").insert( {"id": 1, "title": "Hedgehogs of the world", "author_id": 1} ) with pytest.raises(AlterError): - fresh_db["books"].add_foreign_key("author2_id", "books", "id") + fresh_db.table("books").add_foreign_key("author2_id", "books", "id") def test_add_foreign_key_error_if_other_table_does_not_exist(fresh_db): - fresh_db["books"].insert({"title": "Hedgehogs of the world", "author_id": 1}) + fresh_db.table("books").insert({"title": "Hedgehogs of the world", "author_id": 1}) with pytest.raises(AlterError): - fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") def test_add_foreign_key_error_if_already_exists(fresh_db): - fresh_db["books"].insert({"title": "Hedgehogs of the world", "author_id": 1}) - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fresh_db.table("books").insert({"title": "Hedgehogs of the world", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") with pytest.raises(AlterError) as ex: - fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") assert "Foreign key already exists for author_id => authors.id" == ex.value.args[0] def test_add_foreign_key_no_error_if_exists_and_ignore_true(fresh_db): - fresh_db["books"].insert({"title": "Hedgehogs of the world", "author_id": 1}) - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id", ignore=True) + fresh_db.table("books").insert({"title": "Hedgehogs of the world", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id", ignore=True) def test_add_foreign_keys(fresh_db): - fresh_db["authors"].insert_all( + fresh_db.table("authors").insert_all( [{"id": 1, "name": "Sally"}, {"id": 2, "name": "Asheesh"}], pk="id" ) - fresh_db["categories"].insert_all([{"id": 1, "name": "Wildlife"}], pk="id") - fresh_db["books"].insert_all( + fresh_db.table("categories").insert_all([{"id": 1, "name": "Wildlife"}], pk="id") + fresh_db.table("books").insert_all( [{"title": "Hedgehogs of the world", "author_id": 1, "category_id": 1}] ) - assert [] == fresh_db["books"].foreign_keys + assert [] == fresh_db.table("books").foreign_keys fresh_db.add_foreign_keys( [ ("books", "author_id", "authors", "id"), @@ -493,14 +498,14 @@ def test_add_foreign_keys(fresh_db): other_table="categories", other_column="id", ), - ] == sorted(fresh_db["books"].foreign_keys) + ] == sorted(fresh_db.table("books").foreign_keys) def test_add_column_foreign_key(fresh_db): fresh_db.create_table("dogs", {"name": str}) fresh_db.create_table("breeds", {"name": str}) - fresh_db["dogs"].add_column("breed_id", fk="breeds") - assert fresh_db["dogs"].schema == ( + fresh_db.table("dogs").add_column("breed_id", fk="breeds") + assert fresh_db.table("dogs").schema == ( 'CREATE TABLE "dogs" (\n' ' "name" TEXT,\n' ' "breed_id" INTEGER REFERENCES "breeds"("rowid")\n' @@ -508,8 +513,8 @@ def test_add_column_foreign_key(fresh_db): ) # And again with an explicit primary key column fresh_db.create_table("subbreeds", {"name": str, "primkey": str}, pk="primkey") - fresh_db["dogs"].add_column("subbreed_id", fk="subbreeds") - assert fresh_db["dogs"].schema == ( + fresh_db.table("dogs").add_column("subbreed_id", fk="subbreeds") + assert fresh_db.table("dogs").schema == ( 'CREATE TABLE "dogs" (\n' ' "name" TEXT,\n' ' "breed_id" INTEGER REFERENCES "breeds"("rowid"),\n' @@ -521,9 +526,9 @@ def test_add_column_foreign_key(fresh_db): def test_add_foreign_key_guess_table(fresh_db): fresh_db.create_table("dogs", {"name": str}) fresh_db.create_table("breeds", {"name": str, "id": int}, pk="id") - fresh_db["dogs"].add_column("breed_id", int) - fresh_db["dogs"].add_foreign_key("breed_id") - assert fresh_db["dogs"].schema == ( + fresh_db.table("dogs").add_column("breed_id", int) + fresh_db.table("dogs").add_foreign_key("breed_id") + assert fresh_db.table("dogs").schema == ( 'CREATE TABLE "dogs" (\n' ' "name" TEXT,\n' ' "breed_id" INTEGER REFERENCES "breeds"("id")\n' @@ -533,21 +538,23 @@ def test_add_foreign_key_guess_table(fresh_db): def test_index_foreign_keys(fresh_db): test_add_foreign_key_guess_table(fresh_db) - assert [] == fresh_db["dogs"].indexes + assert [] == fresh_db.table("dogs").indexes fresh_db.index_foreign_keys() - assert [["breed_id"]] == [i.columns for i in fresh_db["dogs"].indexes] + assert [["breed_id"]] == [i.columns for i in fresh_db.table("dogs").indexes] # Calling it a second time should do nothing fresh_db.index_foreign_keys() - assert [["breed_id"]] == [i.columns for i in fresh_db["dogs"].indexes] + assert [["breed_id"]] == [i.columns for i in fresh_db.table("dogs").indexes] def test_index_foreign_keys_if_index_name_is_already_used(fresh_db): # https://github.com/simonw/sqlite-utils/issues/335 test_add_foreign_key_guess_table(fresh_db) # Add index with a name that will conflict with index_foreign_keys() - fresh_db["dogs"].create_index(["name"], index_name="idx_dogs_breed_id") + fresh_db.table("dogs").create_index(["name"], index_name="idx_dogs_breed_id") fresh_db.index_foreign_keys() - assert {(idx.name, tuple(idx.columns)) for idx in fresh_db["dogs"].indexes} == { + assert { + (idx.name, tuple(idx.columns)) for idx in fresh_db.table("dogs").indexes + } == { ("idx_dogs_breed_id_2", ("breed_id",)), ("idx_dogs_breed_id", ("name",)), } @@ -571,7 +578,7 @@ def test_index_foreign_keys_if_index_name_is_already_used(fresh_db): def test_insert_row_alter_table( fresh_db, extra_data, expected_new_columns, use_table_factory ): - table = fresh_db["books"] + table = fresh_db.table("books") table.insert({"title": "Hedgehogs of the world", "author_id": 1}) assert [ {"name": "title", "type": "TEXT"}, @@ -582,7 +589,7 @@ def test_insert_row_alter_table( if use_table_factory: fresh_db.table("books", alter=True).insert(record) else: - fresh_db["books"].insert(record, alter=True) + fresh_db.table("books").insert(record, alter=True) assert [ {"name": "title", "type": "TEXT"}, {"name": "author_id", "type": "INTEGER"}, @@ -592,7 +599,7 @@ def test_insert_row_alter_table( def test_add_missing_columns_case_insensitive(fresh_db): - table = fresh_db["foo"] + table = fresh_db.table("foo") table.insert({"id": 1, "name": "Cleo"}, pk="id") table.add_missing_columns([{"Name": ".", "age": 4}]) assert ( @@ -618,7 +625,7 @@ def test_insert_replace_rows_alter_table(fresh_db, use_table_factory): table.insert(first_row) table.insert_all(next_rows, replace=True) else: - table = fresh_db["books"] + table = fresh_db.table("books") table.insert(first_row, pk="id") table.insert_all(next_rows, alter=True, replace=True) assert { @@ -664,8 +671,8 @@ def test_insert_all_with_extra_columns_in_later_chunks(fresh_db): {"record": "Record 3"}, {"record": "Record 4", "extra": 1}, ] - fresh_db["t"].insert_all(chunk, batch_size=2, alter=True) - assert list(fresh_db["t"].rows) == [ + fresh_db.table("t").insert_all(chunk, batch_size=2, alter=True) + assert list(fresh_db.table("t").rows) == [ {"record": "Record 1", "extra": None}, {"record": "Record 2", "extra": None}, {"record": "Record 3", "extra": None}, @@ -675,7 +682,7 @@ def test_insert_all_with_extra_columns_in_later_chunks(fresh_db): def test_bulk_insert_more_than_999_values(fresh_db): "Inserting 100 items with 11 columns should work" - fresh_db["big"].insert_all( + fresh_db.table("big").insert_all( ( { "id": i + 1, @@ -694,7 +701,7 @@ def test_bulk_insert_more_than_999_values(fresh_db): ), pk="id", ) - assert fresh_db["big"].count == 100 + assert fresh_db.table("big").count == 100 @pytest.mark.parametrize( @@ -704,9 +711,9 @@ def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): record = {f"c{i}": i for i in range(num_columns)} if should_error: with pytest.raises(ValueError): - fresh_db["big"].insert(record) + fresh_db.table("big").insert(record) else: - fresh_db["big"].insert(record) + fresh_db.table("big").insert(record) def test_columns_not_in_first_record_should_not_cause_batch_to_be_too_large(fresh_db): @@ -722,7 +729,9 @@ def test_columns_not_in_first_record_should_not_cause_batch_to_be_too_large(fres # fill out the batch with 99 records with enough columns to exceed THRESHOLD *[{f"c{i}": j for i in range(extra_columns)} for j in range(batch_size - 1)], ] - fresh_db["too_many_columns"].insert_all(records, alter=True, batch_size=batch_size) + fresh_db.table("too_many_columns").insert_all( + records, alter=True, batch_size=batch_size + ) @pytest.mark.parametrize( @@ -767,7 +776,7 @@ def test_columns_not_in_first_record_should_not_cause_batch_to_be_too_large(fres ), ) def test_create_index(fresh_db, columns, index_name, expected_index): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True}) assert [] == dogs.indexes dogs.create_index(columns, index_name) @@ -775,7 +784,7 @@ def test_create_index(fresh_db, columns, index_name, expected_index): def test_create_index_unique(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is_good_dog": True}) assert [] == dogs.indexes dogs.create_index(["name"], unique=True) @@ -793,7 +802,7 @@ def test_create_index_unique(fresh_db): def test_create_index_if_not_exists(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is_good_dog": True}) assert [] == dogs.indexes dogs.create_index(["name"]) @@ -804,7 +813,7 @@ def test_create_index_if_not_exists(fresh_db): def test_drop_index(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is_good_dog": True}) dogs.create_index(["name"]) assert [index.name for index in dogs.indexes] == ["idx_dogs_name"] @@ -813,7 +822,7 @@ def test_drop_index(fresh_db): def test_drop_index_ignore(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo"}) with pytest.raises(OperationalError, match="No index named idx_dogs_name"): dogs.drop_index("idx_dogs_name") @@ -821,8 +830,8 @@ def test_drop_index_ignore(fresh_db): def test_drop_index_wrong_table(fresh_db): - dogs = fresh_db["dogs"] - cats = fresh_db["cats"] + dogs = fresh_db.table("dogs") + cats = fresh_db.table("cats") dogs.insert({"name": "Cleo"}) cats.insert({"name": "Misty"}) dogs.create_index(["name"]) @@ -832,7 +841,7 @@ def test_drop_index_wrong_table(fresh_db): def test_create_index_desc(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True}) assert [] == dogs.indexes dogs.create_index([DescIndex("age"), "name"]) @@ -845,7 +854,7 @@ def test_create_index_desc(fresh_db): def test_create_index_find_unique_name(fresh_db): - table = fresh_db["t"] + table = fresh_db.table("t") table.insert({"id": 1}) table.create_index(["id"]) # Without find_unique_name should error @@ -860,12 +869,12 @@ def test_create_index_find_unique_name(fresh_db): def test_create_index_analyze(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") assert "sqlite_stat1" not in fresh_db.table_names() dogs.insert({"name": "Cleo", "twitter": "cleopaws"}) dogs.create_index(["name"], analyze=True) assert "sqlite_stat1" in fresh_db.table_names() - assert list(fresh_db["sqlite_stat1"].rows) == [ + assert list(fresh_db.table("sqlite_stat1").rows) == [ {"tbl": "dogs", "idx": "idx_dogs_name", "stat": "1 1"} ] @@ -887,14 +896,14 @@ def test_create_index_analyze(fresh_db): ), ) def test_insert_dictionaries_and_lists_as_json(fresh_db, data_structure): - fresh_db["test"].insert({"id": 1, "data": data_structure}, pk="id") + fresh_db.table("test").insert({"id": 1, "data": data_structure}, pk="id") row = fresh_db.execute("select id, data from test").fetchone() assert row[0] == 1 assert data_structure == json.loads(row[1]) def test_insert_list_nested_unicode(fresh_db): - fresh_db["test"].insert( + fresh_db.table("test").insert( {"id": 1, "data": {"key1": {"nested": ["cømplex"]}}}, pk="id" ) row = fresh_db.execute("select id, data from test").fetchone() @@ -903,33 +912,35 @@ def test_insert_list_nested_unicode(fresh_db): def test_insert_uuid(fresh_db): uuid4 = uuid.uuid4() - fresh_db["test"].insert({"uuid": uuid4}) - row = next(iter(fresh_db["test"].rows)) + fresh_db.table("test").insert({"uuid": uuid4}) + row = next(iter(fresh_db.table("test").rows)) assert {"uuid"} == row.keys() assert isinstance(row["uuid"], str) assert row["uuid"] == str(uuid4) def test_insert_memoryview(fresh_db): - fresh_db["test"].insert({"data": memoryview(b"hello")}) - row = next(iter(fresh_db["test"].rows)) + fresh_db.table("test").insert({"data": memoryview(b"hello")}) + row = next(iter(fresh_db.table("test").rows)) assert {"data"} == row.keys() assert isinstance(row["data"], bytes) assert row["data"] == b"hello" def test_insert_thousands_using_generator(fresh_db): - fresh_db["test"].insert_all({"i": i, "word": f"word_{i}"} for i in range(10000)) + fresh_db.table("test").insert_all( + {"i": i, "word": f"word_{i}"} for i in range(10000) + ) assert [{"name": "i", "type": "INTEGER"}, {"name": "word", "type": "TEXT"}] == [ - {"name": col.name, "type": col.type} for col in fresh_db["test"].columns + {"name": col.name, "type": col.type} for col in fresh_db.table("test").columns ] - assert fresh_db["test"].count == 10000 + assert fresh_db.table("test").count == 10000 def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fresh_db): # https://github.com/simonw/sqlite-utils/issues/139 with pytest.raises(Exception, match="table test has no column named extra"): - fresh_db["test"].insert_all( + fresh_db.table("test").insert_all( [{"i": i, "word": f"word_{i}"} for i in range(100)] + [{"i": 101, "extra": "This extra column should cause an exception"}], ) @@ -937,7 +948,7 @@ def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fr def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db): # https://github.com/simonw/sqlite-utils/issues/139 - fresh_db["test"].insert_all( + fresh_db.table("test").insert_all( [{"i": i, "word": f"word_{i}"} for i in range(100)] + [{"i": 101, "extra": "Should trigger ALTER"}], alter=True, @@ -953,12 +964,12 @@ def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): rows = [{"a": f"x{i}", "b": i} for i in range(num_rows)] with pytest.raises(InvalidColumns) as ex: - fresh_db["t"].insert_all(rows, pk="not_a_column") + fresh_db.table("t").insert_all(rows, pk="not_a_column") assert ex.value.args == ( "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", ) - assert fresh_db["t"].count == 0 + assert fresh_db.table("t").count == 0 @pytest.mark.parametrize("num_rows", (1, 2, 3, 10)) @@ -970,20 +981,20 @@ def test_insert_all_pk_not_in_records_alter_raises(fresh_db, num_rows): rows = [{"a": f"x{i}", "b": i} for i in range(num_rows)] with pytest.raises(InvalidColumns) as ex: - fresh_db["t"].insert_all(rows, pk="not_a_column", alter=True) + fresh_db.table("t").insert_all(rows, pk="not_a_column", alter=True) assert ex.value.args == ( "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", ) - assert fresh_db["t"].count == 0 + assert fresh_db.table("t").count == 0 def test_insert_pk_in_records_with_alter_adds_column(fresh_db): # 3.x allowed insert(pk=..., alter=True) to add the pk column from the # records - the InvalidColumns check must not fire in that case - fresh_db["t"].insert({"a": 1}) - fresh_db["t"].insert({"id": 5, "a": 2}, pk="id", alter=True) - assert fresh_db["t"].columns_dict.keys() == {"a", "id"} + fresh_db.table("t").insert({"a": 1}) + fresh_db.table("t").insert({"id": 5, "a": 2}, pk="id", alter=True) + assert fresh_db.table("t").columns_dict.keys() == {"a", "id"} assert list(fresh_db.query("select * from t order by a")) == [ {"a": 1, "id": None}, {"a": 2, "id": 5}, @@ -994,17 +1005,17 @@ def test_insert_all_invalid_pk_alter_empty_records_is_noop(fresh_db): # With alter=True the pk check needs record keys, so an empty iterator # returns without error - matching the 3.x no-op for empty inserts fresh_db.conn.execute("CREATE TABLE t (a TEXT)") - fresh_db["t"].insert_all([], pk="not_a_column", alter=True) - assert fresh_db["t"].count == 0 + fresh_db.table("t").insert_all([], pk="not_a_column", alter=True) + assert fresh_db.table("t").count == 0 def test_insert_ignore(fresh_db): - fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") + fresh_db.table("test").insert({"id": 1, "bar": 2}, pk="id") # Should raise an error if we try this again with pytest.raises(Exception, match="UNIQUE constraint failed"): - fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") + fresh_db.table("test").insert({"id": 1, "bar": 2}, pk="id") # Using ignore=True should cause our insert to be silently ignored - fresh_db["test"].insert({"id": 1, "bar": 3}, pk="id", ignore=True) + fresh_db.table("test").insert({"id": 1, "bar": 3}, pk="id", ignore=True) # Only one row, and it should be bar=2, not bar=3 rows = list(fresh_db.query("select * from test")) assert rows == [{"id": 1, "bar": 2}] @@ -1013,12 +1024,12 @@ def test_insert_ignore(fresh_db): def test_insert_ignore_reports_existing_row(fresh_db): # An ignored insert (row already exists) should point last_rowid and # last_pk at the existing conflicting row - see the Datasette insert API - fresh_db["docs"].insert({"id": 1, "title": "Exists"}, pk="id") + fresh_db.table("docs").insert({"id": 1, "title": "Exists"}, pk="id") # Insert a conflicting row with ignore=True and no explicit pk= - table = fresh_db["docs"].insert({"id": 1, "title": "One"}, ignore=True) + table = fresh_db.table("docs").insert({"id": 1, "title": "One"}, ignore=True) assert table.last_rowid == 1 assert table.last_pk == 1 - assert list(fresh_db["docs"].rows_where("rowid = ?", [table.last_rowid])) == [ + assert list(fresh_db.table("docs").rows_where("rowid = ?", [table.last_rowid])) == [ {"id": 1, "title": "Exists"} ] @@ -1029,51 +1040,51 @@ def test_pk_rowid_alias_on_rowid_table(fresh_db, rowid_alias, method): # rowid and its aliases are valid primary keys for a rowid table even # though they are not listed among the table's columns - see the Datasette # upsert API against tables without an explicit primary key - fresh_db["t"].insert({"title": "Hello"}) - assert fresh_db["t"].pks == ["rowid"] + fresh_db.table("t").insert({"title": "Hello"}) + assert fresh_db.table("t").pks == ["rowid"] record = {rowid_alias: 1, "title": "Updated"} if method == "upsert": - table = fresh_db["t"].upsert(record, pk=rowid_alias) + table = fresh_db.table("t").upsert(record, pk=rowid_alias) elif method == "insert_replace": - table = fresh_db["t"].insert(record, pk=rowid_alias, replace=True) + table = fresh_db.table("t").insert(record, pk=rowid_alias, replace=True) else: - table = fresh_db["t"].insert(record, pk=rowid_alias, ignore=True) + table = fresh_db.table("t").insert(record, pk=rowid_alias, ignore=True) assert table.last_pk == 1 expected_title = "Hello" if method == "insert_ignore" else "Updated" - assert list(fresh_db["t"].rows) == [{"title": expected_title}] + assert list(fresh_db.table("t").rows) == [{"title": expected_title}] def test_insert_ignore_reports_existing_row_compound_pk(fresh_db): # Compound primary key variant of the ignored-insert lookup - fresh_db["t"].insert_all([{"a": 1, "b": 2, "note": "first"}], pk=("a", "b")) - table = fresh_db["t"].insert( + fresh_db.table("t").insert_all([{"a": 1, "b": 2, "note": "first"}], pk=("a", "b")) + table = fresh_db.table("t").insert( {"a": 1, "b": 2, "note": "second"}, pk=("a", "b"), ignore=True ) assert table.last_pk == (1, 2) - assert list(fresh_db["t"].rows_where("rowid = ?", [table.last_rowid])) == [ + assert list(fresh_db.table("t").rows_where("rowid = ?", [table.last_rowid])) == [ {"a": 1, "b": 2, "note": "first"} ] def test_insert_ignore_reports_existing_row_list_mode(fresh_db): # List-based iteration variant of the ignored-insert lookup - fresh_db["t"].insert_all([["id", "title"], [1, "first"]], pk="id") - table = fresh_db["t"].insert_all( + fresh_db.table("t").insert_all([["id", "title"], [1, "first"]], pk="id") + table = fresh_db.table("t").insert_all( [["id", "title"], [1, "second"]], pk="id", ignore=True ) assert table.last_pk == 1 assert table.last_rowid == 1 - assert list(fresh_db["t"].rows) == [{"id": 1, "title": "first"}] + assert list(fresh_db.table("t").rows) == [{"id": 1, "title": "first"}] def test_insert_ignore_hash_id_reports_pk(fresh_db): # With hash_id the pk is the computed hash; the original record has no id # column to look up so last_rowid is left unset - first = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id") - table = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id", ignore=True) + first = fresh_db.table("dogs").insert({"name": "Cleo"}, hash_id="id") + table = fresh_db.table("dogs").insert({"name": "Cleo"}, hash_id="id", ignore=True) assert table.last_pk == first.last_pk assert table.last_rowid is None - assert fresh_db["dogs"].count == 1 + assert fresh_db.table("dogs").count == 1 def test_insert_ignore_unresolvable_conflict_leaves_pk_unset(fresh_db): @@ -1081,45 +1092,45 @@ def test_insert_ignore_unresolvable_conflict_leaves_pk_unset(fresh_db): # last_rowid are left unset rather than reporting a misleading value # rowid table with a UNIQUE column and no primary key: no pk to look up - fresh_db["u"].db.execute("create table u (title text unique)") - fresh_db["u"].insert({"title": "x"}) - table = fresh_db["u"].insert({"title": "x"}, ignore=True) + fresh_db.table("u").db.execute("create table u (title text unique)") + fresh_db.table("u").insert({"title": "x"}) + table = fresh_db.table("u").insert({"title": "x"}, ignore=True) assert table.last_pk is None assert table.last_rowid is None - assert fresh_db["u"].count == 1 + assert fresh_db.table("u").count == 1 # Conflict on a UNIQUE column other than the primary key: the pk value from # the record does not match the existing row, so the lookup finds nothing - fresh_db["docs"].db.execute( + fresh_db.table("docs").db.execute( "create table docs (id integer primary key, email text unique)" ) - fresh_db["docs"].insert({"id": 1, "email": "a"}, pk="id") - table = fresh_db["docs"].insert({"id": 2, "email": "a"}, ignore=True) + fresh_db.table("docs").insert({"id": 1, "email": "a"}, pk="id") + table = fresh_db.table("docs").insert({"id": 2, "email": "a"}, ignore=True) assert table.last_pk is None assert table.last_rowid is None - assert fresh_db["docs"].count == 1 + assert fresh_db.table("docs").count == 1 def test_insert_ignore_with_pk_after_other_table_insert(fresh_db): # https://github.com/simonw/sqlite-utils/issues/554 user = {"id": "abc", "name": "david"} - fresh_db["users"].insert(user, pk="id") - fresh_db["comments"].insert_all( + fresh_db.table("users").insert(user, pk="id") + fresh_db.table("comments").insert_all( [ {"id": "def", "text": "ok"}, {"id": "ghi", "text": "great"}, ], ) - table = fresh_db["users"].insert(user, pk="id", ignore=True) + table = fresh_db.table("users").insert(user, pk="id", ignore=True) assert table.last_pk == "abc" - assert list(fresh_db["users"].rows) == [user] + assert list(fresh_db.table("users").rows) == [user] def test_insert_hash_id(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") id = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id assert dogs.count == 1 @@ -1137,7 +1148,7 @@ def test_insert_hash_id_columns(fresh_db, use_table_factory): dogs = fresh_db.table("dogs", hash_id_columns=("name", "twitter")) insert_kwargs = {} else: - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") insert_kwargs = {"hash_id_columns": ("name", "twitter")} id = dogs.insert( @@ -1158,26 +1169,26 @@ def test_insert_hash_id_columns(fresh_db, use_table_factory): def test_vacuum(fresh_db): - fresh_db["data"].insert({"foo": "foo", "bar": "bar"}) + fresh_db.table("data").insert({"foo": "foo", "bar": "bar"}) fresh_db.vacuum() def test_works_with_pathlib_path(tmpdir): path = pathlib.Path(tmpdir / "test.db") db = Database(path) - db["demo"].insert_all([{"foo": 1}]) - assert db["demo"].count == 1 + db.table("demo").insert_all([{"foo": 1}]) + assert db.table("demo").count == 1 @pytest.mark.skipif(pd is None, reason="pandas and numpy are not installed") def test_create_table_numpy(fresh_db): df = pd.DataFrame({"col 1": range(3), "col 2": range(3)}) - fresh_db["pandas"].insert_all(df.to_dict(orient="records")) + fresh_db.table("pandas").insert_all(df.to_dict(orient="records")) assert [ {"col 1": 0, "col 2": 0}, {"col 1": 1, "col 2": 1}, {"col 1": 2, "col 2": 2}, - ] == list(fresh_db["pandas"].rows) + ] == list(fresh_db.table("pandas").rows) # Now try all the different types df = pd.DataFrame( { @@ -1222,7 +1233,7 @@ def test_create_table_numpy(fresh_db): "float32", "float64", ] == [str(t) for t in df.dtypes] - fresh_db["types"].insert_all(df.to_dict(orient="records")) + fresh_db.table("types").insert_all(df.to_dict(orient="records")) assert [ { "np.float16": 16.5, @@ -1237,7 +1248,7 @@ def test_create_table_numpy(fresh_db): "np.uint64": 64, "np.uint8": 8, } - ] == list(fresh_db["types"].rows) + ] == list(fresh_db.table("types").rows) def test_cannot_provide_both_filename_and_memory(): @@ -1249,31 +1260,31 @@ def test_cannot_provide_both_filename_and_memory(): def test_creates_id_column(fresh_db): last_pk = fresh_db.table("cats", pk="id").insert({"name": "barry"}).last_pk - assert [{"name": "barry", "id": last_pk}] == list(fresh_db["cats"].rows) + assert [{"name": "barry", "id": last_pk}] == list(fresh_db.table("cats").rows) def test_drop(fresh_db): - fresh_db["t"].insert({"foo": 1}) + fresh_db.table("t").insert({"foo": 1}) assert ["t"] == fresh_db.table_names() - assert None is fresh_db["t"].drop() + assert None is fresh_db.table("t").drop() assert [] == fresh_db.table_names() def test_drop_view(fresh_db): fresh_db.create_view("foo_view", "select 1") assert ["foo_view"] == fresh_db.view_names() - assert None is fresh_db["foo_view"].drop() + assert None is fresh_db.view("foo_view").drop() assert [] == fresh_db.view_names() def test_drop_ignore(fresh_db): with pytest.raises(sqlite3.OperationalError): - fresh_db["does_not_exist"].drop() - fresh_db["does_not_exist"].drop(ignore=True) + fresh_db.table("does_not_exist").drop() + fresh_db.table("does_not_exist").drop(ignore=True) # Testing view is harder, we need to create it in order # to get a View object, then drop it twice fresh_db.create_view("foo_view", "select 1") - view = fresh_db["foo_view"] + view = fresh_db.view("foo_view") assert isinstance(view, View) view.drop() with pytest.raises(sqlite3.OperationalError): @@ -1282,16 +1293,16 @@ def test_drop_ignore(fresh_db): def test_insert_all_empty_list(fresh_db): - fresh_db["t"].insert({"foo": 1}) - assert fresh_db["t"].count == 1 - fresh_db["t"].insert_all([]) - assert fresh_db["t"].count == 1 - fresh_db["t"].insert_all([], replace=True) - assert fresh_db["t"].count == 1 + fresh_db.table("t").insert({"foo": 1}) + assert fresh_db.table("t").count == 1 + fresh_db.table("t").insert_all([]) + assert fresh_db.table("t").count == 1 + fresh_db.table("t").insert_all([], replace=True) + assert fresh_db.table("t").count == 1 def test_insert_all_single_column(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all([{"name": "Cleo"}], pk="name") assert [{"name": "Cleo"}] == list(table.rows) assert table.pks == ["name"] @@ -1299,31 +1310,33 @@ def test_insert_all_single_column(fresh_db): @pytest.mark.parametrize("method_name", ("insert_all", "upsert_all")) def test_insert_all_analyze(fresh_db, method_name): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all([{"id": 1, "name": "Cleo"}], pk="id") assert "sqlite_stat1" not in fresh_db.table_names() table.create_index(["name"], analyze=True) - assert list(fresh_db["sqlite_stat1"].rows) == [ + assert list(fresh_db.table("sqlite_stat1").rows) == [ {"tbl": "table", "idx": "idx_table_name", "stat": "1 1"} ] method = getattr(table, method_name) method([{"id": 2, "name": "Suna"}], pk="id", analyze=True) assert "sqlite_stat1" in fresh_db.table_names() - assert list(fresh_db["sqlite_stat1"].rows) == [ + assert list(fresh_db.table("sqlite_stat1").rows) == [ {"tbl": "table", "idx": "idx_table_name", "stat": "2 1"} ] def test_create_with_a_null_column(fresh_db): record = {"name": "Name", "description": None} - fresh_db["t"].insert(record) - assert [record] == list(fresh_db["t"].rows) + fresh_db.table("t").insert(record) + assert [record] == list(fresh_db.table("t").rows) def test_create_with_nested_bytes(fresh_db): record = {"id": 1, "data": {"foo": b"bytes"}} - fresh_db["t"].insert(record) - assert [{"id": 1, "data": '{"foo": "b\'bytes\'"}'}] == list(fresh_db["t"].rows) + fresh_db.table("t").insert(record) + assert [{"id": 1, "data": '{"foo": "b\'bytes\'"}'}] == list( + fresh_db.table("t").rows + ) @pytest.mark.parametrize( @@ -1361,7 +1374,7 @@ def test_create_table_sql(fresh_db, columns, expected_sql_middle): def test_create(fresh_db): - fresh_db["t"].create( + fresh_db.table("t").create( { "id": int, "text": str, @@ -1374,7 +1387,7 @@ def test_create(fresh_db): not_null=("float", "integer"), defaults={"integer": 0}, ) - assert fresh_db["t"].schema == ( + assert fresh_db.table("t").schema == ( 'CREATE TABLE "t" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "float" REAL NOT NULL,\n' @@ -1386,37 +1399,37 @@ def test_create(fresh_db): def test_create_if_not_exists(fresh_db): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should error with pytest.raises(sqlite3.OperationalError): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should not - fresh_db["t"].create({"id": int}, if_not_exists=True) + fresh_db.table("t").create({"id": int}, if_not_exists=True) def test_create_if_no_columns(fresh_db): with pytest.raises(ValueError) as error: - fresh_db["t"].create({}) + fresh_db.table("t").create({}) assert error.value.args[0] == "Tables must have at least one column" def test_create_ignore(fresh_db): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should error with pytest.raises(sqlite3.OperationalError): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should not - fresh_db["t"].create({"id": int}, ignore=True) + fresh_db.table("t").create({"id": int}, ignore=True) def test_create_replace(fresh_db): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should error with pytest.raises(sqlite3.OperationalError): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should not - fresh_db["t"].create({"name": str}, replace=True) - assert fresh_db["t"].schema == ('CREATE TABLE "t" (\n' ' "name" TEXT\n' ")") + fresh_db.table("t").create({"name": str}, replace=True) + assert fresh_db.table("t").schema == ('CREATE TABLE "t" (\n' ' "name" TEXT\n' ")") @pytest.mark.parametrize( @@ -1484,23 +1497,23 @@ def test_create_replace(fresh_db): ) def test_create_transform(fresh_db, cols, kwargs, expected_schema, should_transform): fresh_db.create_table("demo", {"id": int, "name": str}, pk="id") - fresh_db["demo"].insert({"id": 1, "name": "Cleo"}) + fresh_db.table("demo").insert({"id": 1, "name": "Cleo"}) traces = [] with fresh_db.tracer(lambda sql, parameters: traces.append((sql, parameters))): - fresh_db["demo"].create(cols, **kwargs, transform=True) + fresh_db.table("demo").create(cols, **kwargs, transform=True) at_least_one_create_table = any(sql.startswith("CREATE TABLE") for sql, _ in traces) assert should_transform == at_least_one_create_table - new_schema = fresh_db["demo"].schema + new_schema = fresh_db.table("demo").schema assert new_schema == expected_schema, repr(new_schema) - assert fresh_db["demo"].count == 1 + assert fresh_db.table("demo").count == 1 def test_rename_table(fresh_db): - fresh_db["t"].insert({"foo": "bar"}) + fresh_db.table("t").insert({"foo": "bar"}) assert ["t"] == fresh_db.table_names() fresh_db.rename_table("t", "renamed") assert ["renamed"] == fresh_db.table_names() - assert [{"foo": "bar"}] == list(fresh_db["renamed"].rows) + assert [{"foo": "bar"}] == list(fresh_db.table("renamed").rows) # Should error if table does not exist: with pytest.raises(sqlite3.OperationalError): fresh_db.rename_table("does_not_exist", "renamed") @@ -1527,7 +1540,7 @@ def test_database_strict_override(strict): ) @pytest.mark.parametrize("strict", (False, True)) def test_insert_upsert_strict(fresh_db, method_name, strict): - table = fresh_db["t"] + table = fresh_db.table("t") method = getattr(table, method_name) record = {"id": 1} if method_name.endswith("_all"): @@ -1550,7 +1563,7 @@ def test_create_table_strict(fresh_db, strict): @pytest.mark.parametrize("strict", (False, True)) def test_create_strict(fresh_db, strict): - table = fresh_db["t"] + table = fresh_db.table("t") table.create({"id": int}, strict=strict) assert table.strict == strict or not fresh_db.supports_strict @@ -1575,7 +1588,7 @@ def test_bad_table_and_view_exceptions(fresh_db): def test_pk_persists_after_insert_655(fresh_db): """When pk is passed to insert(), subsequent inserts should use it.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.insert({"id": 1, "name": "Alice"}, pk="id") # Second insert should use pk="id" from _defaults table.insert({"id": 2, "name": "Bob"}) @@ -1586,7 +1599,7 @@ def test_pk_persists_after_insert_655(fresh_db): def test_pk_persists_after_insert_all_655(fresh_db): """When pk is passed to insert_all(), subsequent inserts should use it.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.insert_all([{"id": 1, "name": "Alice"}], pk="id") # Second insert_all should use pk="id" from _defaults table.insert_all([{"id": 2, "name": "Bob"}]) @@ -1596,7 +1609,7 @@ def test_pk_persists_after_insert_all_655(fresh_db): def test_pk_persists_after_create_655(fresh_db): """When pk is passed to create(), it should be stored in _defaults.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.create({"id": int, "name": str}, pk="id") assert table._defaults["pk"] == "id" # Subsequent insert should use the pk @@ -1607,8 +1620,8 @@ def test_pk_persists_after_create_655(fresh_db): def test_foreign_keys_persist_after_create_655(fresh_db): """When foreign_keys is passed to create(), it should be stored in _defaults.""" - fresh_db["authors"].insert({"id": 1, "name": "Alice"}, pk="id") - table = fresh_db["books"] + fresh_db.table("authors").insert({"id": 1, "name": "Alice"}, pk="id") + table = fresh_db.table("books") table.create( {"id": int, "title": str, "author_id": int}, pk="id", @@ -1620,28 +1633,28 @@ def test_foreign_keys_persist_after_create_655(fresh_db): def test_not_null_persists_after_create_655(fresh_db): """When not_null is passed to create(), it should be stored in _defaults.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.create({"id": int, "name": str}, pk="id", not_null=["name"]) assert table._defaults["not_null"] == ["name"] def test_defaults_persist_after_create_655(fresh_db): """When defaults is passed to create(), it should be stored in _defaults.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.create({"id": int, "score": int}, pk="id", defaults={"score": 0}) assert table._defaults["defaults"] == {"score": 0} def test_strict_persists_after_create_655(fresh_db): """When strict is passed to create(), it should be stored in _defaults.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.create({"id": int, "name": str}, pk="id", strict=True) assert table._defaults["strict"] is True def test_upsert_uses_pk_from_prior_insert_655(fresh_db): """After insert with pk, upsert should use the same pk.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.insert({"id": 1, "name": "Alice"}, pk="id") # Upsert should work without specifying pk again table.upsert({"id": 1, "name": "Alice Updated"}) @@ -1651,7 +1664,7 @@ def test_upsert_uses_pk_from_prior_insert_655(fresh_db): def test_upsert_all_uses_pk_from_prior_insert_655(fresh_db): """After insert with pk, upsert_all should use the same pk.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.insert({"id": 1, "name": "Alice"}, pk="id") # Upsert_all should work without specifying pk again table.upsert_all([{"id": 1, "name": "Alice Updated"}, {"id": 2, "name": "Bob"}]) diff --git a/tests/test_default_value.py b/tests/test_default_value.py index 2815180..02b28c3 100644 --- a/tests/test_default_value.py +++ b/tests/test_default_value.py @@ -32,9 +32,9 @@ EXAMPLES = [ @pytest.mark.parametrize("column_def,initial_value,expected_value", EXAMPLES) def test_quote_default_value(fresh_db, column_def, initial_value, expected_value): fresh_db.execute(f"create table foo (col {column_def})") - assert initial_value == fresh_db["foo"].columns[0].default_value + assert initial_value == fresh_db.table("foo").columns[0].default_value assert expected_value == fresh_db.quote_default_value( - fresh_db["foo"].columns[0].default_value + fresh_db.table("foo").columns[0].default_value ) @@ -48,7 +48,7 @@ def test_insert_empty_record_uses_default_values(fresh_db): ) """) - table = fresh_db["has_defaults"] + table = fresh_db.table("has_defaults") table.insert({}) rows = list(table.rows) diff --git a/tests/test_delete.py b/tests/test_delete.py index dffb6bb..a9341b8 100644 --- a/tests/test_delete.py +++ b/tests/test_delete.py @@ -2,7 +2,7 @@ import sqlite_utils def test_delete_rowid_table(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"foo": 1}) rowid = table.insert({"foo": 2}).last_pk table.delete(rowid) @@ -10,7 +10,7 @@ def test_delete_rowid_table(fresh_db): def test_delete_pk_table(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"id": 1}, pk="id") table.insert({"id": 2}, pk="id") table.delete(1) @@ -18,7 +18,7 @@ def test_delete_pk_table(fresh_db): def test_delete_where(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") for i in range(1, 11): table.insert({"id": i}, pk="id") assert table.count == 10 @@ -27,7 +27,7 @@ def test_delete_where(fresh_db): def test_delete_where_all(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") for i in range(1, 11): table.insert({"id": i}, pk="id") assert table.count == 10 @@ -38,27 +38,27 @@ def test_delete_where_all(fresh_db): def test_delete_where_commits(tmpdir): path = str(tmpdir / "test.db") db = sqlite_utils.Database(path) - db["table"].insert_all([{"id": i} for i in range(5)], pk="id") - db["table"].delete_where("id > ?", [2]) + db.table("table").insert_all([{"id": i} for i in range(5)], pk="id") + db.table("table").delete_where("id > ?", [2]) # The connection must not be left inside an open transaction, # otherwise subsequent atomic() blocks never commit either assert not db.conn.in_transaction - db["table"].insert({"id": 100}) + db.table("table").insert({"id": 100}) db.close() db2 = sqlite_utils.Database(path) - assert [r["id"] for r in db2["table"].rows] == [0, 1, 2, 100] + assert [r["id"] for r in db2.table("table").rows] == [0, 1, 2, 100] db2.close() def test_delete_where_analyze(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all(({"id": i, "i": i} for i in range(10)), pk="id") table.create_index(["i"], analyze=True) assert "sqlite_stat1" in fresh_db.table_names() - assert list(fresh_db["sqlite_stat1"].rows) == [ + assert list(fresh_db.table("sqlite_stat1").rows) == [ {"tbl": "table", "idx": "idx_table_i", "stat": "10 1"} ] table.delete_where("id > ?", [5], analyze=True) - assert list(fresh_db["sqlite_stat1"].rows) == [ + assert list(fresh_db.table("sqlite_stat1").rows) == [ {"tbl": "table", "idx": "idx_table_i", "stat": "6 1"} ] diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py index ad853a5..c7a5612 100644 --- a/tests/test_duplicate.py +++ b/tests/test_duplicate.py @@ -22,7 +22,7 @@ def test_duplicate(fresh_db): "bool_col": True, "datetime_col": str(dt), } - table1 = fresh_db["table1"] + table1 = fresh_db.table("table1") row_id = table1.insert(data).last_rowid # Duplicate table: table2 = table1.duplicate("table2") @@ -40,4 +40,4 @@ def test_duplicate(fresh_db): def test_duplicate_fails_if_table_does_not_exist(fresh_db): with pytest.raises(NoTable): - fresh_db["not_a_table"].duplicate("duplicated") + fresh_db.table("not_a_table").duplicate("duplicated") diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index 71a8936..1230b6c 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -5,7 +5,7 @@ from sqlite_utils import Database, cli def test_enable_counts_specific_table(fresh_db): - foo = fresh_db["foo"] + foo = fresh_db.table("foo") assert fresh_db.table_names() == [] for i in range(10): foo.insert({"name": f"item {i}"}) @@ -41,24 +41,24 @@ def test_enable_counts_specific_table(fresh_db): ), } assert fresh_db.table_names() == ["foo", "_counts"] - assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}] + assert list(fresh_db.table("_counts").rows) == [{"count": 10, "table": "foo"}] # Add some items to test the triggers for i in range(5): foo.insert({"name": f"item {10 + i}"}) assert foo.count == 15 - assert list(fresh_db["_counts"].rows) == [{"count": 15, "table": "foo"}] + assert list(fresh_db.table("_counts").rows) == [{"count": 15, "table": "foo"}] # Delete some items foo.delete_where("rowid < 7") assert foo.count == 9 - assert list(fresh_db["_counts"].rows) == [{"count": 9, "table": "foo"}] + assert list(fresh_db.table("_counts").rows) == [{"count": 9, "table": "foo"}] foo.delete_where() assert foo.count == 0 - assert list(fresh_db["_counts"].rows) == [{"count": 0, "table": "foo"}] + assert list(fresh_db.table("_counts").rows) == [{"count": 0, "table": "foo"}] def test_enable_counts_all_tables(fresh_db): - foo = fresh_db["foo"] - bar = fresh_db["bar"] + foo = fresh_db.table("foo") + bar = fresh_db.table("bar") foo.insert({"name": "Cleo"}) bar.insert({"name": "Cleo"}) foo.enable_fts(["name"]) @@ -73,7 +73,7 @@ def test_enable_counts_all_tables(fresh_db): "foo_fts_config", "_counts", } - assert list(fresh_db["_counts"].rows) == [ + assert list(fresh_db.table("_counts").rows) == [ {"count": 1, "table": "foo"}, {"count": 1, "table": "bar"}, {"count": 3, "table": "foo_fts_data"}, @@ -87,10 +87,10 @@ def test_enable_counts_all_tables(fresh_db): def counts_db_path(tmpdir): path = str(tmpdir / "test.db") db = Database(path) - db["foo"].insert({"name": "bar"}) - db["bar"].insert({"name": "bar"}) - db["bar"].insert({"name": "bar"}) - db["baz"].insert({"name": "bar"}) + db.table("foo").insert({"name": "bar"}) + db.table("bar").insert({"name": "bar"}) + db.table("bar").insert({"name": "bar"}) + db.table("baz").insert({"name": "bar"}) return path @@ -163,25 +163,25 @@ def test_uses_counts_after_enable_counts(counts_db_path): def test_reset_counts(counts_db_path): db = Database(counts_db_path) - db["foo"].enable_counts() - db["bar"].enable_counts() + db.table("foo").enable_counts() + db.table("bar").enable_counts() assert db.cached_counts() == {"foo": 1, "bar": 2} # Corrupt the value - db["_counts"].update("foo", {"count": 3}) + db.table("_counts").update("foo", {"count": 3}) assert db.cached_counts() == {"foo": 3, "bar": 2} - assert db["foo"].count == 3 + assert db.table("foo").count == 3 # Reset them db.reset_counts() assert db.cached_counts() == {"foo": 1, "bar": 2} - assert db["foo"].count == 1 + assert db.table("foo").count == 1 def test_reset_counts_cli(counts_db_path): db = Database(counts_db_path) - db["foo"].enable_counts() - db["bar"].enable_counts() + db.table("foo").enable_counts() + db.table("bar").enable_counts() assert db.cached_counts() == {"foo": 1, "bar": 2} - db["_counts"].update("foo", {"count": 3}) + db.table("_counts").update("foo", {"count": 3}) result = CliRunner().invoke(cli.cli, ["reset-counts", counts_db_path]) assert result.exit_code == 0 assert db.cached_counts() == {"foo": 1, "bar": 2} diff --git a/tests/test_extract.py b/tests/test_extract.py index 915e6e1..72579c4 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -11,7 +11,7 @@ def test_extract_single_column(fresh_db, table, fk_column): expected_table = table or "species" expected_fk = fk_column or f"{expected_table}_id" iter_species = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) - fresh_db["tree"].insert_all( + fresh_db.table("tree").insert_all( ( { "id": i, @@ -23,8 +23,8 @@ def test_extract_single_column(fresh_db, table, fk_column): ), pk="id", ) - fresh_db["tree"].extract("species", table=table, fk_column=fk_column) - assert fresh_db["tree"].schema == ( + fresh_db.table("tree").extract("species", table=table, fk_column=fk_column) + assert fresh_db.table("tree").schema == ( 'CREATE TABLE "tree" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT,\n' @@ -32,18 +32,18 @@ def test_extract_single_column(fresh_db, table, fk_column): + ' "end" INTEGER\n' + ")" ) - assert fresh_db[expected_table].schema == ( + assert fresh_db.table(expected_table).schema == ( f'CREATE TABLE "{expected_table}" (\n' + ' "id" INTEGER PRIMARY KEY,\n' ' "species" TEXT\n' ")" ) - assert list(fresh_db[expected_table].rows) == [ + assert list(fresh_db.table(expected_table).rows) == [ {"id": 1, "species": "Palm"}, {"id": 2, "species": "Spruce"}, {"id": 3, "species": "Mangrove"}, {"id": 4, "species": "Oak"}, ] - assert list(itertools.islice(fresh_db["tree"].rows, 0, 4)) == [ + assert list(itertools.islice(fresh_db.table("tree").rows, 0, 4)) == [ {"id": 1, "name": "Tree 1", expected_fk: 1, "end": 1}, {"id": 2, "name": "Tree 2", expected_fk: 2, "end": 1}, {"id": 3, "name": "Tree 3", expected_fk: 3, "end": 1}, @@ -54,7 +54,7 @@ def test_extract_single_column(fresh_db, table, fk_column): def test_extract_multiple_columns_with_rename(fresh_db): iter_common = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) iter_latin = itertools.cycle(["Arecaceae", "Picea", "Rhizophora", "Quercus"]) - fresh_db["tree"].insert_all( + fresh_db.table("tree").insert_all( ( { "id": i, @@ -67,30 +67,30 @@ def test_extract_multiple_columns_with_rename(fresh_db): pk="id", ) - fresh_db["tree"].extract( + fresh_db.table("tree").extract( ["common_name", "latin_name"], rename={"common_name": "name"} ) - assert fresh_db["tree"].schema == ( + assert fresh_db.table("tree").schema == ( 'CREATE TABLE "tree" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT,\n' ' "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n' ")" ) - assert fresh_db["common_name_latin_name"].schema == ( + assert fresh_db.table("common_name_latin_name").schema == ( 'CREATE TABLE "common_name_latin_name" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT,\n' ' "latin_name" TEXT\n' ")" ) - assert list(fresh_db["common_name_latin_name"].rows) == [ + assert list(fresh_db.table("common_name_latin_name").rows) == [ {"name": "Palm", "id": 1, "latin_name": "Arecaceae"}, {"name": "Spruce", "id": 2, "latin_name": "Picea"}, {"name": "Mangrove", "id": 3, "latin_name": "Rhizophora"}, {"name": "Oak", "id": 4, "latin_name": "Quercus"}, ] - assert list(itertools.islice(fresh_db["tree"].rows, 0, 4)) == [ + assert list(itertools.islice(fresh_db.table("tree").rows, 0, 4)) == [ {"id": 1, "name": "Tree 1", "common_name_latin_name_id": 1}, {"id": 2, "name": "Tree 2", "common_name_latin_name_id": 2}, {"id": 3, "name": "Tree 3", "common_name_latin_name_id": 3}, @@ -99,7 +99,7 @@ def test_extract_multiple_columns_with_rename(fresh_db): def test_extract_invalid_columns(fresh_db): - fresh_db["tree"].insert( + fresh_db.table("tree").insert( { "id": 1, "name": "Tree 1", @@ -109,19 +109,19 @@ def test_extract_invalid_columns(fresh_db): pk="id", ) with pytest.raises(InvalidColumns): - fresh_db["tree"].extract(["bad_column"]) + fresh_db.table("tree").extract(["bad_column"]) def test_extract_rowid_table(fresh_db): - fresh_db["tree"].insert( + fresh_db.table("tree").insert( { "name": "Tree 1", "common_name": "Palm", "latin_name": "Arecaceae", } ) - fresh_db["tree"].extract(["common_name", "latin_name"]) - assert fresh_db["tree"].schema == ( + fresh_db.table("tree").extract(["common_name", "latin_name"]) + assert fresh_db.table("tree").schema == ( 'CREATE TABLE "tree" (\n' ' "name" TEXT,\n' ' "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n' @@ -139,68 +139,68 @@ def test_extract_rowid_table(fresh_db): def test_reuse_lookup_table(fresh_db): - fresh_db["species"].insert({"id": 1, "name": "Wolf"}, pk="id") - fresh_db["sightings"].insert({"id": 10, "species": "Wolf"}, pk="id") - fresh_db["individuals"].insert( + fresh_db.table("species").insert({"id": 1, "name": "Wolf"}, pk="id") + fresh_db.table("sightings").insert({"id": 10, "species": "Wolf"}, pk="id") + fresh_db.table("individuals").insert( {"id": 10, "name": "Terriana", "species": "Fox"}, pk="id" ) - fresh_db["sightings"].extract("species", rename={"species": "name"}) - fresh_db["individuals"].extract("species", rename={"species": "name"}) - assert fresh_db["sightings"].schema == ( + fresh_db.table("sightings").extract("species", rename={"species": "name"}) + fresh_db.table("individuals").extract("species", rename={"species": "name"}) + assert fresh_db.table("sightings").schema == ( 'CREATE TABLE "sightings" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "species_id" INTEGER REFERENCES "species"("id")\n' ")" ) - assert fresh_db["individuals"].schema == ( + assert fresh_db.table("individuals").schema == ( 'CREATE TABLE "individuals" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT,\n' ' "species_id" INTEGER REFERENCES "species"("id")\n' ")" ) - assert list(fresh_db["species"].rows) == [ + assert list(fresh_db.table("species").rows) == [ {"id": 1, "name": "Wolf"}, {"id": 2, "name": "Fox"}, ] def test_extract_error_on_incompatible_existing_lookup_table(fresh_db): - fresh_db["species"].insert({"id": 1}) - fresh_db["tree"].insert({"name": "Tree 1", "common_name": "Palm"}) + fresh_db.table("species").insert({"id": 1}) + fresh_db.table("tree").insert({"name": "Tree 1", "common_name": "Palm"}) with pytest.raises(InvalidColumns): - fresh_db["tree"].extract("common_name", table="species") + fresh_db.table("tree").extract("common_name", table="species") # Try again with incompatible existing column type - fresh_db["species2"].insert({"id": 1, "common_name": 3.5}) + fresh_db.table("species2").insert({"id": 1, "common_name": 3.5}) with pytest.raises(InvalidColumns): - fresh_db["tree"].extract("common_name", table="species2") + fresh_db.table("tree").extract("common_name", table="species2") def test_extract_works_with_null_values(fresh_db): - fresh_db["listens"].insert_all( + fresh_db.table("listens").insert_all( [ {"id": 1, "track_title": "foo", "album_title": "bar"}, {"id": 2, "track_title": "baz", "album_title": None}, ], pk="id", ) - fresh_db["listens"].extract( + fresh_db.table("listens").extract( columns=["album_title"], table="albums", fk_column="album_id" ) - assert list(fresh_db["listens"].rows) == [ + assert list(fresh_db.table("listens").rows) == [ {"id": 1, "track_title": "foo", "album_id": 1}, {"id": 2, "track_title": "baz", "album_id": None}, ] - assert list(fresh_db["albums"].rows) == [ + assert list(fresh_db.table("albums").rows) == [ {"id": 1, "album_title": "bar"}, ] def test_extract_null_values_single_column(fresh_db): # https://github.com/simonw/sqlite-utils/issues/186 - fresh_db["species"].insert({"id": 1, "species": "Wolf"}, pk="id") - fresh_db["individuals"].insert_all( + fresh_db.table("species").insert({"id": 1, "species": "Wolf"}, pk="id") + fresh_db.table("individuals").insert_all( [ {"id": 10, "name": "Terriana", "species": "Fox"}, {"id": 11, "name": "Spenidorm", "species": None}, @@ -210,13 +210,13 @@ def test_extract_null_values_single_column(fresh_db): ], pk="id", ) - fresh_db["individuals"].extract("species") + fresh_db.table("individuals").extract("species") # No null row should have been added to species - assert list(fresh_db["species"].rows) == [ + assert list(fresh_db.table("species").rows) == [ {"id": 1, "species": "Wolf"}, {"id": 2, "species": "Fox"}, ] - assert list(fresh_db["individuals"].rows) == [ + assert list(fresh_db.table("individuals").rows) == [ {"id": 10, "name": "Terriana", "species_id": 2}, {"id": 11, "name": "Spenidorm", "species_id": None}, {"id": 12, "name": "Grantheim", "species_id": 1}, @@ -228,7 +228,7 @@ def test_extract_null_values_single_column(fresh_db): def test_extract_null_values_multiple_columns(fresh_db): # A row should be extracted if at least one column is not null - # only rows where ALL extracted columns are null are left alone - fresh_db["circulation"].insert_all( + fresh_db.table("circulation").insert_all( [ {"id": 1, "title": "title one", "creator": "creator one", "year": 2018}, {"id": 2, "title": "title two", "creator": None, "year": 2019}, @@ -237,14 +237,14 @@ def test_extract_null_values_multiple_columns(fresh_db): ], pk="id", ) - fresh_db["circulation"].extract( + fresh_db.table("circulation").extract( ["title", "creator"], table="books", fk_column="book_id" ) - assert list(fresh_db["books"].rows) == [ + assert list(fresh_db.table("books").rows) == [ {"id": 1, "title": "title one", "creator": "creator one"}, {"id": 2, "title": "title two", "creator": None}, ] - assert list(fresh_db["circulation"].rows) == [ + assert list(fresh_db.table("circulation").rows) == [ {"id": 1, "book_id": 1, "year": 2018}, {"id": 2, "book_id": 2, "year": 2019}, {"id": 3, "book_id": None, "year": 2020}, @@ -255,20 +255,20 @@ def test_extract_null_values_multiple_columns(fresh_db): def test_extract_null_values_existing_lookup_table_with_null_row(fresh_db): # Even if the lookup table already contains an all-null row, rows where # every extracted column is null should keep a null foreign key - fresh_db["species"].insert({"id": 1, "species": None}, pk="id") - fresh_db["individuals"].insert_all( + fresh_db.table("species").insert({"id": 1, "species": None}, pk="id") + fresh_db.table("individuals").insert_all( [ {"id": 10, "name": "Terriana", "species": "Fox"}, {"id": 11, "name": "Spenidorm", "species": None}, ], pk="id", ) - fresh_db["individuals"].extract("species") - assert list(fresh_db["species"].rows) == [ + fresh_db.table("individuals").extract("species") + assert list(fresh_db.table("species").rows) == [ {"id": 1, "species": None}, {"id": 2, "species": "Fox"}, ] - assert list(fresh_db["individuals"].rows) == [ + assert list(fresh_db.table("individuals").rows) == [ {"id": 10, "name": "Terriana", "species_id": 2}, {"id": 11, "name": "Spenidorm", "species_id": None}, ] @@ -279,17 +279,19 @@ def test_extract_repeated_into_shared_lookup_with_nulls(fresh_db): # cannot dedupe NULL-containing rows against the existing lookup # table - extracting a second table into the same lookup previously # inserted duplicate rows that nothing pointed to - fresh_db["t1"].insert_all( + fresh_db.table("t1").insert_all( [ {"id": 1, "species": None, "common": "X"}, {"id": 2, "species": "Oak", "common": "Oak"}, ], pk="id", ) - fresh_db["t2"].insert_all([{"id": 1, "species": None, "common": "X"}], pk="id") - fresh_db["t1"].extract(["species", "common"], table="lk") - fresh_db["t2"].extract(["species", "common"], table="lk") - assert fresh_db["lk"].count == 2 + fresh_db.table("t2").insert_all( + [{"id": 1, "species": None, "common": "X"}], pk="id" + ) + fresh_db.table("t1").extract(["species", "common"], table="lk") + fresh_db.table("t2").extract(["species", "common"], table="lk") + assert fresh_db.table("lk").count == 2 # Both tables point at the same lookup row t1_fk = fresh_db.execute("select lk_id from t1 where id = 1").fetchone()[0] t2_fk = fresh_db.execute("select lk_id from t2 where id = 1").fetchone()[0] @@ -298,8 +300,8 @@ def test_extract_repeated_into_shared_lookup_with_nulls(fresh_db): def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db): # Non-NULL rows were already deduped by the unique index - keep it so - fresh_db["t1"].insert_all([{"id": 1, "species": "Oak"}], pk="id") - fresh_db["t2"].insert_all([{"id": 1, "species": "Oak"}], pk="id") - fresh_db["t1"].extract(["species"], table="lk") - fresh_db["t2"].extract(["species"], table="lk") - assert fresh_db["lk"].count == 1 + fresh_db.table("t1").insert_all([{"id": 1, "species": "Oak"}], pk="id") + fresh_db.table("t2").insert_all([{"id": 1, "species": "Oak"}], pk="id") + fresh_db.table("t1").extract(["species"], table="lk") + fresh_db.table("t2").extract(["species"], table="lk") + assert fresh_db.table("lk").count == 1 diff --git a/tests/test_extracts.py b/tests/test_extracts.py index 9519b91..4e7cf39 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -32,15 +32,15 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): assert {expected_table, "Trees"} == set(fresh_db.table_names()) assert ( f'CREATE TABLE "{expected_table}" (\n "id" INTEGER PRIMARY KEY,\n "value" TEXT\n)' - == fresh_db[expected_table].schema + == fresh_db.table(expected_table).schema ) assert ( f'CREATE TABLE "Trees" (\n "id" INTEGER,\n "species_id" INTEGER REFERENCES "{expected_table}"("id")\n)' - == fresh_db["Trees"].schema + == fresh_db.table("Trees").schema ) # Should have a foreign key reference - assert len(fresh_db["Trees"].foreign_keys) == 1 - fk = fresh_db["Trees"].foreign_keys[0] + assert len(fresh_db.table("Trees").foreign_keys) == 1 + fk = fresh_db.table("Trees").foreign_keys[0] assert fk.table == "Trees" assert fk.column == "species_id" @@ -54,22 +54,22 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): partial=0, columns=["value"], ) - ] == fresh_db[expected_table].indexes + ] == fresh_db.table(expected_table).indexes # Finally, check the rows assert [{"id": 1, "value": "Oak"}, {"id": 2, "value": "Palm"}] == list( - fresh_db[expected_table].rows + fresh_db.table(expected_table).rows ) assert [ {"id": 1, "species_id": 1}, {"id": 2, "species_id": 1}, {"id": 3, "species_id": 2}, - ] == list(fresh_db["Trees"].rows) + ] == list(fresh_db.table("Trees").rows) def test_extracts_null_values(fresh_db): # https://github.com/simonw/sqlite-utils/issues/186 # Null values should stay null, not be extracted into the lookup table - fresh_db["Trees"].insert_all( + fresh_db.table("Trees").insert_all( [ {"id": 1, "species_id": "Oak"}, {"id": 2, "species_id": None}, @@ -78,11 +78,11 @@ def test_extracts_null_values(fresh_db): ], extracts={"species_id": "Species"}, ) - assert list(fresh_db["Species"].rows) == [ + assert list(fresh_db.table("Species").rows) == [ {"id": 1, "value": "Oak"}, {"id": 2, "value": "Palm"}, ] - assert list(fresh_db["Trees"].rows) == [ + assert list(fresh_db.table("Trees").rows) == [ {"id": 1, "species_id": 1}, {"id": 2, "species_id": None}, {"id": 3, "species_id": 2}, @@ -92,7 +92,7 @@ def test_extracts_null_values(fresh_db): def test_extracts_null_values_list_mode(fresh_db): # Same as test_extracts_null_values but for list-based records - fresh_db["Trees"].insert_all( + fresh_db.table("Trees").insert_all( [ ["id", "species_id"], [1, "Oak"], @@ -102,11 +102,11 @@ def test_extracts_null_values_list_mode(fresh_db): ], extracts={"species_id": "Species"}, ) - assert list(fresh_db["Species"].rows) == [ + assert list(fresh_db.table("Species").rows) == [ {"id": 1, "value": "Oak"}, {"id": 2, "value": "Palm"}, ] - assert list(fresh_db["Trees"].rows) == [ + assert list(fresh_db.table("Trees").rows) == [ {"id": 1, "species_id": 1}, {"id": 2, "species_id": None}, {"id": 3, "species_id": 2}, diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 45f4f35..271125c 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -32,7 +32,7 @@ def compound_db(): def test_compound_foreign_key(compound_db): - fks = compound_db["courses"].foreign_keys + fks = compound_db.table("courses").foreign_keys assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True @@ -46,10 +46,10 @@ def test_compound_foreign_key(compound_db): def test_single_foreign_key_gets_columns_fields(fresh_db): - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) - fresh_db["books"].add_foreign_key("author_id", "authors", "id") - fk = fresh_db["books"].foreign_keys[0] + fresh_db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db.table("books").insert({"title": "Hedgehogs", "author_id": 1}) + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") + fk = fresh_db.table("books").foreign_keys[0] assert fk.is_compound is False assert fk.column == "author_id" assert fk.other_column == "id" @@ -60,10 +60,10 @@ def test_single_foreign_key_gets_columns_fields(fresh_db): def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): # Clean break in 4.0: ForeignKey is a dataclass, not a namedtuple, so the # old tuple unpacking and indexing patterns now fail hard. - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) - fresh_db["books"].add_foreign_key("author_id", "authors", "id") - fk = fresh_db["books"].foreign_keys[0] + fresh_db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db.table("books").insert({"title": "Hedgehogs", "author_id": 1}) + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") + fk = fresh_db.table("books").foreign_keys[0] with pytest.raises(TypeError): _table, _column, _other_table, _other_column = fk with pytest.raises(TypeError): @@ -71,16 +71,18 @@ def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): def test_foreign_keys_are_sortable(fresh_db): - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["categories"].insert({"id": 1, "name": "Wildlife"}, pk="id") - fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1, "category_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db.table("categories").insert({"id": 1, "name": "Wildlife"}, pk="id") + fresh_db.table("books").insert( + {"title": "Hedgehogs", "author_id": 1, "category_id": 1} + ) fresh_db.add_foreign_keys( [ ("books", "author_id", "authors", "id"), ("books", "category_id", "categories", "id"), ] ) - fks = sorted(fresh_db["books"].foreign_keys) + fks = sorted(fresh_db.table("books").foreign_keys) assert fks[0].column == "author_id" assert fks[1].column == "category_id" @@ -105,7 +107,7 @@ def test_mixed_compound_and_single_foreign_keys_are_sortable(): REFERENCES departments(campus_name, dept_code) ); """) - fks = db["courses"].foreign_keys + fks = db.table("courses").foreign_keys assert len(fks) == 2 assert {fk.is_compound for fk in fks} == {True, False} fks_sorted = sorted(fks) @@ -163,8 +165,8 @@ def test_create_table_with_compound_foreign_key(departments_db, foreign_keys): pk="course_code", foreign_keys=foreign_keys, ) - assert departments_db["courses"].schema == EXPECTED_COURSES_SCHEMA - fks = departments_db["courses"].foreign_keys + assert departments_db.table("courses").schema == EXPECTED_COURSES_SCHEMA + fks = departments_db.table("courses").foreign_keys assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True @@ -181,10 +183,10 @@ def test_create_table_compound_foreign_key_enforced(departments_db): pk="course_code", foreign_keys=[(("campus_name", "dept_code"), "departments")], ) - departments_db["departments"].insert( + departments_db.table("departments").insert( {"campus_name": "Berkeley", "dept_code": "CS", "dept_name": "Computer Science"} ) - departments_db["courses"].insert( + departments_db.table("courses").insert( {"course_code": "CS101", "campus_name": "Berkeley", "dept_code": "CS"} ) with pytest.raises(sqlite3.IntegrityError): @@ -207,8 +209,8 @@ def test_create_table_compound_foreign_key_missing_other_column(departments_db): def test_transform_preserves_compound_foreign_key(compound_db): - compound_db["courses"].transform(rename={"course_name": "title"}) - fks = compound_db["courses"].foreign_keys + compound_db.table("courses").transform(rename={"course_name": "title"}) + fks = compound_db.table("courses").foreign_keys assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True @@ -218,8 +220,8 @@ def test_transform_preserves_compound_foreign_key(compound_db): def test_transform_rename_member_column_updates_compound_foreign_key(compound_db): - compound_db["courses"].transform(rename={"campus_name": "campus"}) - fks = compound_db["courses"].foreign_keys + compound_db.table("courses").transform(rename={"campus_name": "campus"}) + fks = compound_db.table("courses").foreign_keys assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True @@ -231,9 +233,9 @@ def test_transform_rename_member_column_updates_compound_foreign_key(compound_db def test_transform_drop_member_column_drops_compound_foreign_key(compound_db): # Matches single-column behavior: dropping the column silently # drops the foreign key that used it - compound_db["courses"].transform(drop={"dept_code"}) - assert compound_db["courses"].foreign_keys == [] - assert "FOREIGN KEY" not in compound_db["courses"].schema + compound_db.table("courses").transform(drop={"dept_code"}) + assert compound_db.table("courses").foreign_keys == [] + assert "FOREIGN KEY" not in compound_db.table("courses").schema @pytest.mark.parametrize( @@ -246,11 +248,11 @@ def test_transform_drop_member_column_drops_compound_foreign_key(compound_db): ), ) def test_transform_drop_compound_foreign_key(compound_db, drop_foreign_keys): - compound_db["courses"].transform(drop_foreign_keys=drop_foreign_keys) - assert compound_db["courses"].foreign_keys == [] + compound_db.table("courses").transform(drop_foreign_keys=drop_foreign_keys) + assert compound_db.table("courses").foreign_keys == [] # The columns themselves survive assert {"campus_name", "dept_code"} <= set( - compound_db["courses"].columns_dict.keys() + compound_db.table("courses").columns_dict.keys() ) @@ -265,12 +267,12 @@ def courses_db(departments_db): def test_add_compound_foreign_key(courses_db): - t = courses_db["courses"].add_foreign_key( + t = courses_db.table("courses").add_foreign_key( ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code") ) # Returns self assert t.name == "courses" - fks = courses_db["courses"].foreign_keys + fks = courses_db.table("courses").foreign_keys assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True @@ -281,27 +283,33 @@ def test_add_compound_foreign_key(courses_db): def test_add_compound_foreign_key_guesses_other_columns(courses_db): # Lists work here too, though tuples are the documented form - courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") - fk = courses_db["courses"].foreign_keys[0] + courses_db.table("courses").add_foreign_key( + ["campus_name", "dept_code"], "departments" + ) + fk = courses_db.table("courses").foreign_keys[0] assert fk.other_columns == ("campus_name", "dept_code") def test_add_compound_foreign_key_error_if_already_exists(courses_db): - courses_db["courses"].add_foreign_key(("campus_name", "dept_code"), "departments") + courses_db.table("courses").add_foreign_key( + ("campus_name", "dept_code"), "departments" + ) with pytest.raises(AlterError) as ex: - courses_db["courses"].add_foreign_key( + courses_db.table("courses").add_foreign_key( ("campus_name", "dept_code"), "departments" ) assert "already exists" in ex.value.args[0] # ignore=True should not raise - courses_db["courses"].add_foreign_key( + courses_db.table("courses").add_foreign_key( ("campus_name", "dept_code"), "departments", ignore=True ) def test_add_compound_foreign_key_error_if_column_missing(courses_db): with pytest.raises(AlterError): - courses_db["courses"].add_foreign_key(("campus_name", "nope"), "departments") + courses_db.table("courses").add_foreign_key( + ("campus_name", "nope"), "departments" + ) def test_db_add_foreign_keys_compound(courses_db): @@ -315,14 +323,14 @@ def test_db_add_foreign_keys_compound(courses_db): ) ] ) - fk = courses_db["courses"].foreign_keys[0] + fk = courses_db.table("courses").foreign_keys[0] assert fk.is_compound is True assert fk.columns == ("campus_name", "dept_code") def test_index_foreign_keys_compound_creates_composite_index(compound_db): compound_db.index_foreign_keys() - index_columns = [i.columns for i in compound_db["courses"].indexes] + index_columns = [i.columns for i in compound_db.table("courses").indexes] assert ["campus_name", "dept_code"] in index_columns # No separate single-column indexes for the members assert ["campus_name"] not in index_columns @@ -339,22 +347,22 @@ def test_foreign_key_captures_on_delete_and_on_update(): ON DELETE CASCADE ON UPDATE RESTRICT ); """) - fk = db["books"].foreign_keys[0] + fk = db.table("books").foreign_keys[0] assert fk.on_delete == "CASCADE" assert fk.on_update == "RESTRICT" def test_foreign_key_on_delete_defaults_to_no_action(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id") - fk = fresh_db["books"].foreign_keys[0] + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("books").insert({"id": 1, "author_id": 1}, pk="id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") + fk = fresh_db.table("books").foreign_keys[0] assert fk.on_delete == "NO ACTION" assert fk.on_update == "NO ACTION" def test_create_table_foreign_key_with_on_delete(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db.table("authors").insert({"id": 1}, pk="id") fresh_db.create_table( "books", {"id": int, "author_id": int}, @@ -369,8 +377,8 @@ def test_create_table_foreign_key_with_on_delete(fresh_db): ) ], ) - assert "ON DELETE CASCADE" in fresh_db["books"].schema - assert fresh_db["books"].foreign_keys[0].on_delete == "CASCADE" + assert "ON DELETE CASCADE" in fresh_db.table("books").schema + assert fresh_db.table("books").foreign_keys[0].on_delete == "CASCADE" def test_transform_preserves_on_delete_cascade(): @@ -383,11 +391,11 @@ def test_transform_preserves_on_delete_cascade(): author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE ); """) - db["books"].transform(rename={"title": "book_title"}) - fk = db["books"].foreign_keys[0] + db.table("books").transform(rename={"title": "book_title"}) + fk = db.table("books").foreign_keys[0] assert fk.on_delete == "CASCADE" assert fk.on_update == "NO ACTION" - assert "ON DELETE CASCADE" in db["books"].schema + assert "ON DELETE CASCADE" in db.table("books").schema def test_transform_preserves_compound_foreign_key_on_delete(): @@ -406,11 +414,11 @@ def test_transform_preserves_compound_foreign_key_on_delete(): REFERENCES departments(campus_name, dept_code) ON DELETE CASCADE ); """) - db["courses"].transform(rename={"course_code": "code"}) - fk = db["courses"].foreign_keys[0] + db.table("courses").transform(rename={"course_code": "code"}) + fk = db.table("courses").foreign_keys[0] assert fk.is_compound is True assert fk.on_delete == "CASCADE" - assert "ON DELETE CASCADE" in db["courses"].schema + assert "ON DELETE CASCADE" in db.table("courses").schema def test_implicit_primary_key_reference_is_resolved(): @@ -424,7 +432,7 @@ def test_implicit_primary_key_reference_is_resolved(): author_id INTEGER REFERENCES authors ); """) - fk = db["books"].foreign_keys[0] + fk = db.table("books").foreign_keys[0] assert fk.is_compound is False assert fk.other_column == "author_id" assert fk.other_columns == ("author_id",) @@ -445,7 +453,7 @@ def test_implicit_compound_primary_key_reference_is_resolved(): FOREIGN KEY (campus_name, dept_code) REFERENCES departments ); """) - fk = db["courses"].foreign_keys[0] + fk = db.table("courses").foreign_keys[0] assert fk.is_compound is True assert fk.other_columns == ("campus_name", "dept_code") @@ -470,14 +478,14 @@ def test_add_foreign_keys_preserves_actions(fresh_db): # https://github.com/simonw/sqlite-utils/issues/594 review finding: # ForeignKey objects passed to db.add_foreign_keys() were flattened # to plain tuples, losing on_delete/on_update - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("books").insert({"id": 1, "author_id": 1}, pk="id") fresh_db.add_foreign_keys( [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] ) - fk = fresh_db["books"].foreign_keys[0] + fk = fresh_db.table("books").foreign_keys[0] assert fk.on_delete == "CASCADE" - assert "ON DELETE CASCADE" in fresh_db["books"].schema + assert "ON DELETE CASCADE" in fresh_db.table("books").schema def test_add_foreign_keys_preserves_actions_compound(courses_db): @@ -495,36 +503,36 @@ def test_add_foreign_keys_preserves_actions_compound(courses_db): ) ] ) - fk = courses_db["courses"].foreign_keys[0] + fk = courses_db.table("courses").foreign_keys[0] assert fk.is_compound is True assert fk.on_delete == "CASCADE" - assert "ON DELETE CASCADE" in courses_db["courses"].schema + assert "ON DELETE CASCADE" in courses_db.table("courses").schema def test_add_foreign_key_on_delete_on_update(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") - fresh_db["books"].add_foreign_key( + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("books").insert({"id": 1, "author_id": 1}, pk="id") + fresh_db.table("books").add_foreign_key( "author_id", "authors", "id", on_delete="CASCADE", on_update="RESTRICT" ) - fk = fresh_db["books"].foreign_keys[0] + fk = fresh_db.table("books").foreign_keys[0] assert fk.on_delete == "CASCADE" assert fk.on_update == "RESTRICT" - assert "ON UPDATE RESTRICT ON DELETE CASCADE" in fresh_db["books"].schema + assert "ON UPDATE RESTRICT ON DELETE CASCADE" in fresh_db.table("books").schema # The cascade should actually fire fresh_db.execute("PRAGMA foreign_keys = ON") fresh_db.execute("delete from authors where id = 1") - assert fresh_db["books"].count == 0 + assert fresh_db.table("books").count == 0 def test_add_compound_foreign_key_on_delete(courses_db): - courses_db["courses"].add_foreign_key( + courses_db.table("courses").add_foreign_key( ("campus_name", "dept_code"), "departments", on_delete="SET NULL" ) - fk = courses_db["courses"].foreign_keys[0] + fk = courses_db.table("courses").foreign_keys[0] assert fk.is_compound is True assert fk.on_delete == "SET NULL" - assert "ON DELETE SET NULL" in courses_db["courses"].schema + assert "ON DELETE SET NULL" in courses_db.table("courses").schema def test_implicit_compound_foreign_key_resolves_pk_declaration_order(fresh_db): @@ -536,7 +544,7 @@ def test_implicit_compound_foreign_key_resolves_pk_declaration_order(fresh_db): fresh_db.execute( "create table child (x text, y text, foreign key (x, y) references other)" ) - fk = fresh_db["child"].foreign_keys[0] + fk = fresh_db.table("child").foreign_keys[0] assert fk.other_columns == ("a", "b") @@ -549,46 +557,46 @@ def test_transform_implicit_compound_foreign_key_stays_valid(fresh_db): "create table child (x text, y text, foreign key (x, y) references other)" ) fresh_db.execute("PRAGMA foreign_keys = ON") - fresh_db["other"].insert({"a": "A", "b": "B"}) - fresh_db["child"].insert({"x": "A", "y": "B"}) - fresh_db["child"].transform(types={"x": str}) - assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + fresh_db.table("other").insert({"a": "A", "b": "B"}) + fresh_db.table("child").insert({"x": "A", "y": "B"}) + fresh_db.table("child").transform(types={"x": str}) + assert fresh_db.table("child").foreign_keys[0].other_columns == ("a", "b") # The constraint still points the right way around - fresh_db["child"].insert({"x": "A", "y": "B"}) + fresh_db.table("child").insert({"x": "A", "y": "B"}) with pytest.raises(sqlite3.IntegrityError): - fresh_db["child"].insert({"x": "B", "y": "A"}) + fresh_db.table("child").insert({"x": "B", "y": "A"}) def test_create_compound_foreign_key_guesses_pk_declaration_order(fresh_db): fresh_db.execute("create table other (b text, a text, primary key (a, b))") - fresh_db["other"].insert({"a": "A", "b": "B"}) - fresh_db["child"].create( + fresh_db.table("other").insert({"a": "A", "b": "B"}) + fresh_db.table("child").create( {"id": int, "x": str, "y": str}, pk="id", foreign_keys=[(("x", "y"), "other")], ) - assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + assert fresh_db.table("child").foreign_keys[0].other_columns == ("a", "b") fresh_db.execute("PRAGMA foreign_keys = ON") - fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}) + fresh_db.table("child").insert({"id": 1, "x": "A", "y": "B"}) with pytest.raises(sqlite3.IntegrityError): - fresh_db["child"].insert({"id": 2, "x": "B", "y": "A"}) + fresh_db.table("child").insert({"id": 2, "x": "B", "y": "A"}) def test_add_compound_foreign_key_guesses_pk_declaration_order(fresh_db): fresh_db.execute("create table other (b text, a text, primary key (a, b))") - fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}, pk="id") - fresh_db["child"].add_foreign_key(("x", "y"), "other") - assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + fresh_db.table("child").insert({"id": 1, "x": "A", "y": "B"}, pk="id") + fresh_db.table("child").add_foreign_key(("x", "y"), "other") + assert fresh_db.table("child").foreign_keys[0].other_columns == ("a", "b") def test_foreign_keys_are_hashable(fresh_db): # set() over foreign_keys worked with the 3.x namedtuple and must # keep working with the dataclass - fresh_db["p"].insert({"id": 1}, pk="id") - fresh_db["c"].insert( + fresh_db.table("p").insert({"id": 1}, pk="id") + fresh_db.table("c").insert( {"id": 1, "pid": 1}, pk="id", foreign_keys=[("pid", "p", "id")] ) - fks = set(fresh_db["c"].foreign_keys) + fks = set(fresh_db.table("c").foreign_keys) assert len(fks) == 1 assert ForeignKey("c", "pid", "p", "id") in fks # Usable as dict keys too @@ -617,9 +625,9 @@ def test_create_table_mixed_foreign_keys_list(fresh_db): # 3.x accepted a mix of ForeignKey objects, tuples and bare column # strings in foreign_keys= (ForeignKey was a namedtuple, so it passed # the tuple check) - keep accepting the mix - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["publishers"].insert({"id": 1}, pk="id") - fresh_db["books"].create( + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("publishers").insert({"id": 1}, pk="id") + fresh_db.table("books").create( {"id": int, "author_id": int, "publisher_id": int}, pk="id", foreign_keys=[ @@ -627,14 +635,14 @@ def test_create_table_mixed_foreign_keys_list(fresh_db): ("publisher_id", "publishers", "id"), ], ) - fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} + fks = {fk.column: fk.other_table for fk in fresh_db.table("books").foreign_keys} assert fks == {"author_id": "authors", "publisher_id": "publishers"} def test_create_table_mixed_foreign_keys_with_string(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["publishers"].insert({"id": 1}, pk="id") - fresh_db["books"].create( + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("publishers").insert({"id": 1}, pk="id") + fresh_db.table("books").create( {"id": int, "author_id": int, "publisher_id": int}, pk="id", foreign_keys=[ @@ -642,15 +650,15 @@ def test_create_table_mixed_foreign_keys_with_string(fresh_db): ("publisher_id", "publishers", "id"), ], ) - fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} + fks = {fk.column: fk.other_table for fk in fresh_db.table("books").foreign_keys} assert fks == {"author_id": "authors", "publisher_id": "publishers"} def test_add_foreign_keys_existing_with_different_actions_errors(fresh_db): # Requesting an existing foreign key with different ON DELETE/ON UPDATE # actions was silently skipped, dropping the requested change - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert( + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("books").insert( {"id": 1, "author_id": 1}, pk="id", foreign_keys=[("author_id", "authors", "id")], @@ -660,19 +668,21 @@ def test_add_foreign_keys_existing_with_different_actions_errors(fresh_db): [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] ) assert "ON DELETE" in str(ex.value) - assert fresh_db["books"].foreign_keys[0].on_delete == "NO ACTION" + assert fresh_db.table("books").foreign_keys[0].on_delete == "NO ACTION" def test_add_foreign_keys_identical_existing_is_noop(fresh_db): # An exact match, including actions, is silently skipped so repeated # calls stay idempotent - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id", on_delete="CASCADE") + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("books").insert({"id": 1, "author_id": 1}, pk="id") + fresh_db.table("books").add_foreign_key( + "author_id", "authors", "id", on_delete="CASCADE" + ) fresh_db.add_foreign_keys( [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] ) - fks = fresh_db["books"].foreign_keys + fks = fresh_db.table("books").foreign_keys assert len(fks) == 1 assert fks[0].on_delete == "CASCADE" @@ -680,13 +690,13 @@ def test_add_foreign_keys_identical_existing_is_noop(fresh_db): def test_add_foreign_keys_compound_column_count_mismatch_errors(fresh_db): # Previously the extra other-column was silently discarded, creating # a single-column foreign key to just ("id") - fresh_db["departments"].insert( + fresh_db.table("departments").insert( {"campus": "north", "code": "cs"}, pk=("campus", "code") ) - fresh_db["courses"].insert({"id": 1, "campus": "north"}, pk="id") + fresh_db.table("courses").insert({"id": 1, "campus": "north"}, pk="id") with pytest.raises(ValueError) as ex: fresh_db.add_foreign_keys( [("courses", ("campus",), "departments", ("campus", "code"))] ) assert "same number of columns" in str(ex.value) - assert fresh_db["courses"].foreign_keys == [] + assert fresh_db.table("courses").foreign_keys == [] diff --git a/tests/test_fts.py b/tests/test_fts.py index 79af042..312b032 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -20,7 +20,7 @@ search_records = [ def test_enable_fts(fresh_db): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert_all(search_records) assert ["searchable"] == fresh_db.table_names() table.enable_fts(["text", "country"], fts_version="FTS4") @@ -54,7 +54,7 @@ def test_enable_fts(fresh_db): def test_enable_fts_escape_table_names(fresh_db): # Table names with restricted chars are handled correctly. # colons and dots are restricted characters for table names. - table = fresh_db["http://example.com"] + table = fresh_db.table("http://example.com") table.insert_all(search_records) assert ["http://example.com"] == fresh_db.table_names() table.enable_fts(["text", "country"], fts_version="FTS4") @@ -87,7 +87,7 @@ def test_enable_fts_escape_table_names(fresh_db): def test_search_duplicate_columns_are_deduped(fresh_db): # https://github.com/simonw/sqlite-utils/issues/624 - table = fresh_db["t"] + table = fresh_db.table("t") table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version="FTS4") rows = list(table.search("tanuki", columns=["text", "text"])) @@ -100,7 +100,7 @@ def test_search_duplicate_columns_are_deduped(fresh_db): def test_search_limit_offset(fresh_db): - table = fresh_db["t"] + table = fresh_db.table("t") table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version="FTS4") assert len(list(table.search("are"))) == 2 @@ -113,7 +113,7 @@ def test_search_limit_offset(fresh_db): def test_search_offset_without_limit(fresh_db): - table = fresh_db["t"] + table = fresh_db.table("t") table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version="FTS4") assert [row["rowid"] for row in table.search("are", order_by="rowid")] == [1, 2] @@ -125,7 +125,7 @@ def test_search_offset_without_limit(fresh_db): @pytest.mark.parametrize("fts_version", ("FTS4", "FTS5")) def test_search_where(fresh_db, fts_version): - table = fresh_db["t"] + table = fresh_db.table("t") table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version=fts_version) results = list( @@ -142,7 +142,7 @@ def test_search_where(fresh_db, fts_version): def test_search_where_args_disallows_query(fresh_db): - table = fresh_db["t"] + table = fresh_db.table("t") with pytest.raises(ValueError) as ex: list( table.search( @@ -156,7 +156,7 @@ def test_search_where_args_disallows_query(fresh_db): def test_search_include_rank(fresh_db): - table = fresh_db["t"] + table = fresh_db.table("t") table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version="FTS5") results = list(table.search("are", include_rank=True)) @@ -182,7 +182,7 @@ def test_search_include_rank(fresh_db): def test_enable_fts_table_names_containing_spaces(fresh_db): - table = fresh_db["test"] + table = fresh_db.table("test") table.insert({"column with spaces": "in its name"}) table.enable_fts(["column with spaces"]) assert [ @@ -196,7 +196,7 @@ def test_enable_fts_table_names_containing_spaces(fresh_db): def test_populate_fts(fresh_db): - table = fresh_db["populatable"] + table = fresh_db.table("populatable") table.insert(search_records[0]) table.enable_fts(["text", "country"], fts_version="FTS4") assert [] == list(table.search("trash pandas")) @@ -217,7 +217,7 @@ def test_populate_fts(fresh_db): def test_populate_fts_escape_table_names(fresh_db): # Restricted characters such as colon and dots should be escaped. - table = fresh_db["http://example.com"] + table = fresh_db.table("http://example.com") table.insert(search_records[0]) table.enable_fts(["text", "country"], fts_version="FTS4") assert [] == list(table.search("trash pandas")) @@ -238,7 +238,7 @@ def test_populate_fts_escape_table_names(fresh_db): @pytest.mark.parametrize("fts_version", ("4", "5")) def test_fts_tokenize(fresh_db, fts_version): table_name = f"searchable_{fts_version}" - table = fresh_db[table_name] + table = fresh_db.table(table_name) table.insert_all(search_records) # Test without porter stemming table.enable_fts( @@ -266,7 +266,7 @@ def test_fts_tokenize(fresh_db, fts_version): def test_fts_tokenize_escaped(fresh_db): # A malicious tokenize value must not be able to break out of the # string literal in the CREATE VIRTUAL TABLE statement. - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert_all(search_records) malicious = "porter'); CREATE TABLE injected(x); --" with pytest.raises(Exception): @@ -278,7 +278,7 @@ def test_fts_tokenize_escaped(fresh_db): def test_optimize_fts(fresh_db): for fts_version in ("4", "5"): table_name = f"searchable_{fts_version}" - table = fresh_db[table_name] + table = fresh_db.table(table_name) table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version=f"FTS{fts_version}") # You can call optimize successfully against the tables OR their _fts equivalents: @@ -288,11 +288,11 @@ def test_optimize_fts(fresh_db): "searchable_4_fts", "searchable_5_fts", ): - fresh_db[table_name].optimize() + fresh_db.table(table_name).optimize() def test_enable_fts_with_triggers(fresh_db): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert(search_records[0]) table.enable_fts(["text", "country"], fts_version="FTS4", create_triggers=True) rows1 = list(table.search("tanuki")) @@ -321,7 +321,7 @@ def test_enable_fts_with_triggers(fresh_db): @pytest.mark.parametrize("create_triggers", [True, False]) def test_disable_fts(fresh_db, create_triggers): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert(search_records[0]) table.enable_fts(["text", "country"], create_triggers=create_triggers) assert { @@ -354,7 +354,7 @@ def test_disable_fts(fresh_db, create_triggers): def test_rebuild_fts(fresh_db): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert(search_records[0]) table.enable_fts(["text", "country"]) # Run a search @@ -380,7 +380,7 @@ def test_rebuild_fts(fresh_db): def test_optimize_and_rebuild_fts_commit(tmpdir, method): path = str(tmpdir / "test.db") db = Database(path) - table = db["searchable"] + table = db.table("searchable") table.insert(search_records[0]) table.enable_fts(["text", "country"]) getattr(table, method)() @@ -390,16 +390,16 @@ def test_optimize_and_rebuild_fts_commit(tmpdir, method): table.insert(search_records[1]) db.close() db2 = Database(path) - assert db2["searchable"].count == 2 + assert db2.table("searchable").count == 2 db2.close() @pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"]) def test_rebuild_fts_invalid(fresh_db, invalid_table): - fresh_db["not_searchable"].insert({"foo": "bar"}) + fresh_db.table("not_searchable").insert({"foo": "bar"}) # Raise OperationalError on invalid table with pytest.raises(sqlite3.OperationalError): - fresh_db[invalid_table].rebuild_fts() + fresh_db.table(invalid_table).rebuild_fts() @pytest.mark.parametrize("fts_version", ["FTS4", "FTS5"]) @@ -408,15 +408,17 @@ def test_rebuild_removes_junk_docsize_rows(tmpdir, fts_version): path = tmpdir / "test.db" db = Database(str(path), recursive_triggers=False) licenses = [{"key": "apache2", "name": "Apache 2"}, {"key": "bsd", "name": "BSD"}] - db["licenses"].insert_all(licenses, pk="key", replace=True) - db["licenses"].enable_fts(["name"], create_triggers=True, fts_version=fts_version) - assert db["licenses_fts_docsize"].count == 2 + db.table("licenses").insert_all(licenses, pk="key", replace=True) + db.table("licenses").enable_fts( + ["name"], create_triggers=True, fts_version=fts_version + ) + assert db.table("licenses_fts_docsize").count == 2 # Bug: insert with replace increases the number of rows in _docsize: - db["licenses"].insert_all(licenses, pk="key", replace=True) - assert db["licenses_fts_docsize"].count == 4 + db.table("licenses").insert_all(licenses, pk="key", replace=True) + assert db.table("licenses_fts_docsize").count == 4 # rebuild should fix this: - db["licenses_fts"].rebuild_fts() - assert db["licenses_fts_docsize"].count == 2 + db.table("licenses_fts").rebuild_fts() + assert db.table("licenses_fts_docsize").count == 2 @pytest.mark.parametrize( @@ -430,7 +432,7 @@ def test_rebuild_removes_junk_docsize_rows(tmpdir, fts_version): ) def test_enable_fts_replace(kwargs): db = Database(memory=True) - db["books"].insert( + db.table("books").insert( { "id": 1, "title": "Habits of Australian Marsupials", @@ -438,31 +440,31 @@ def test_enable_fts_replace(kwargs): }, pk="id", ) - db["books"].enable_fts(["title", "author"]) - assert not db["books"].triggers - assert db["books_fts"].columns_dict.keys() == {"title", "author"} - assert "FTS5" in db["books_fts"].schema - assert "porter" not in db["books_fts"].schema + db.table("books").enable_fts(["title", "author"]) + assert not db.table("books").triggers + assert db.table("books_fts").columns_dict.keys() == {"title", "author"} + assert "FTS5" in db.table("books_fts").schema + assert "porter" not in db.table("books_fts").schema # Now modify the FTS configuration should_have_changed_columns = "columns" in kwargs if "columns" not in kwargs: kwargs["columns"] = ["title", "author"] - db["books"].enable_fts(**kwargs, replace=True) + db.table("books").enable_fts(**kwargs, replace=True) # Check that the new configuration is correct if should_have_changed_columns: - assert db["books_fts"].columns_dict.keys() == {"title"} + assert db.table("books_fts").columns_dict.keys() == {"title"} if "create_triggers" in kwargs: - assert db["books"].triggers + assert db.table("books").triggers if "fts_version" in kwargs: - assert "FTS4" in db["books_fts"].schema + assert "FTS4" in db.table("books_fts").schema if "tokenize" in kwargs: - assert "porter" in db["books_fts"].schema + assert "porter" in db.table("books_fts").schema def test_enable_fts_replace_does_nothing_if_args_the_same(): queries = [] db = Database(memory=True, tracer=lambda sql, params: queries.append((sql, params))) - db["books"].insert( + db.table("books").insert( { "id": 1, "title": "Habits of Australian Marsupials", @@ -470,17 +472,19 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): }, pk="id", ) - db["books"].enable_fts(["title", "author"], create_triggers=True) + db.table("books").enable_fts(["title", "author"], create_triggers=True) queries.clear() # Running that again shouldn't run much SQL: - db["books"].enable_fts(["title", "author"], create_triggers=True, replace=True) + db.table("books").enable_fts( + ["title", "author"], create_triggers=True, replace=True + ) # The only SQL that executed should be select statements assert all(q[0].startswith("select ") for q in queries) def test_enable_fts_replace_handles_legacy_bracket_quoted_content_table(): db = Database(memory=True) - db["books"].insert( + db.table("books").insert( { "id": 1, "title": "Habits of Australian Marsupials", @@ -495,10 +499,10 @@ def test_enable_fts_replace_handles_legacy_bracket_quoted_content_table(): ); """) - db["books"].enable_fts(["title", "author"], replace=True) + db.table("books").enable_fts(["title", "author"], replace=True) - assert db["books_fts"].columns_dict.keys() == {"title", "author"} - assert 'content="books"' in db["books_fts"].schema + assert db.table("books_fts").columns_dict.keys() == {"title", "author"} + assert 'content="books"' in db.table("books_fts").schema def test_view_has_no_enable_fts(): @@ -506,7 +510,7 @@ def test_view_has_no_enable_fts(): db.create_view("hello", "select 1 + 1") # Views deliberately do not have an enable_fts() method with pytest.raises(AttributeError): - db["hello"].enable_fts() # type: ignore[union-attr] + db.view("hello").enable_fts() # type: ignore[union-attr] @pytest.mark.parametrize( @@ -712,14 +716,14 @@ def test_view_has_no_enable_fts(): ) def test_search_sql(kwargs, fts, expected): db = Database(memory=True) - db["books"].insert( + db.table("books").insert( { "title": "Habits of Australian Marsupials", "author": "Marlee Hawkins", } ) - db["books"].enable_fts(["title", "author"], fts_version=fts) - sql = db["books"].search_sql(**kwargs) + db.table("books").enable_fts(["title", "author"], fts_version=fts) + sql = db.table("books").search_sql(**kwargs) assert sql == expected @@ -740,7 +744,7 @@ def test_search_sql(kwargs, fts, expected): ), ) def test_quote_fts_query(fresh_db, input, expected): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert_all(search_records) table.enable_fts(["text", "country"]) quoted = fresh_db.quote_fts(input) @@ -750,7 +754,7 @@ def test_quote_fts_query(fresh_db, input, expected): def test_search_quote(fresh_db): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert_all(search_records) table.enable_fts(["text", "country"]) query = "cat's" @@ -763,7 +767,7 @@ def test_search_quote(fresh_db): def test_enable_fts_cli_on_view_errors(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["t"].insert({"text": "hello"}) + db.table("t").insert({"text": "hello"}) db.create_view("v", "select * from t") db.close() from click.testing import CliRunner diff --git a/tests/test_get.py b/tests/test_get.py index 3cdaed8..5e29506 100644 --- a/tests/test_get.py +++ b/tests/test_get.py @@ -4,14 +4,14 @@ from sqlite_utils.db import NotFoundError def test_get_rowid(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") cleo = {"name": "Cleo", "age": 4} row_id = dogs.insert(cleo).last_rowid assert cleo == dogs.get(row_id) def test_get_primary_key(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") cleo = {"name": "Cleo", "age": 4, "id": 5} last_pk = dogs.insert(cleo, pk="id").last_pk assert 5 == last_pk @@ -23,10 +23,10 @@ def test_get_primary_key(fresh_db): [(100, None), (None, None), ((1, 2), "Need 1 primary key value"), ("2", None)], ) def test_get_not_found(argument, expected_msg, fresh_db): - fresh_db["dogs"].insert( + fresh_db.table("dogs").insert( {"id": 1, "name": "Cleo", "age": 4, "is_good": True}, pk="id" ) with pytest.raises(NotFoundError) as excinfo: - fresh_db["dogs"].get(argument) + fresh_db.table("dogs").get(argument) if expected_msg is not None: assert expected_msg == excinfo.value.args[0] diff --git a/tests/test_gis.py b/tests/test_gis.py index 8b41d22..592af4c 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -45,7 +45,7 @@ def test_add_geometry_column(): coord_dimension="XY", ) - assert db["geometry_columns"].get(["locations", "geometry"]) == { + assert db.table("geometry_columns").get(["locations", "geometry"]) == { "f_table_name": "locations", "f_geometry_column": "geometry", "geometry_type": 1, # point @@ -133,7 +133,7 @@ def test_cli_add_geometry_column(tmpdir): db = Database(str(db_path)) db.init_spatialite() - table = db["locations"].create({"name": str}) + table = db.table("locations").create({"name": str}) result = CliRunner().invoke( cli, @@ -149,7 +149,7 @@ def test_cli_add_geometry_column(tmpdir): assert result.exit_code == 0 - assert db["geometry_columns"].get(["locations", "geometry"]) == { + assert db.table("geometry_columns").get(["locations", "geometry"]) == { "f_table_name": "locations", "f_geometry_column": "geometry", "geometry_type": 1, # point @@ -164,7 +164,7 @@ def test_cli_add_geometry_column_options(tmpdir): db_path = tmpdir / "spatial.db" db = Database(str(db_path)) db.init_spatialite() - table = db["locations"].create({"name": str}) + table = db.table("locations").create({"name": str}) result = CliRunner().invoke( cli, @@ -183,7 +183,7 @@ def test_cli_add_geometry_column_options(tmpdir): assert result.exit_code == 0 - assert db["geometry_columns"].get(["locations", "geometry"]) == { + assert db.table("geometry_columns").get(["locations", "geometry"]) == { "f_table_name": "locations", "f_geometry_column": "geometry", "geometry_type": 3, # polygon @@ -202,7 +202,7 @@ def test_cli_add_geometry_column_invalid_type(tmpdir): db = Database(str(db_path)) db.init_spatialite() - table = db["locations"].create({"name": str}) + table = db.table("locations").create({"name": str}) result = CliRunner().invoke( cli, @@ -225,7 +225,7 @@ def test_cli_create_spatial_index(tmpdir): db = Database(str(db_path)) db.init_spatialite() - table = db["locations"].create({"name": str}) + table = db.table("locations").create({"name": str}) table.add_geometry_column("geometry", "POINT") result = CliRunner().invoke( diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index ab652c7..d017f1f 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -11,8 +11,8 @@ def test_roundtrip_integers(integer): row = { "integer": integer, } - db["test"].insert(row) - assert list(db["test"].rows) == [row] + db.table("test").insert(row) + assert list(db.table("test").rows) == [row] @given(st.text()) @@ -21,8 +21,8 @@ def test_roundtrip_text(text): row = { "text": text, } - db["test"].insert(row) - assert list(db["test"].rows) == [row] + db.table("test").insert(row) + assert list(db.table("test").rows) == [row] @given(st.binary(max_size=1024 * 1024)) @@ -31,8 +31,8 @@ def test_roundtrip_binary(binary): row = { "binary": binary, } - db["test"].insert(row) - assert list(db["test"].rows) == [row] + db.table("test").insert(row) + assert list(db.table("test").rows) == [row] @given(st.floats(allow_nan=False)) @@ -41,5 +41,5 @@ def test_roundtrip_floats(floats): row = { "floats": floats, } - db["test"].insert(row) - assert list(db["test"].rows) == [row] + db.table("test").insert(row) + assert list(db.table("test").rows) == [row] diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 1724d2d..93c4daf 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -57,7 +57,7 @@ def test_insert_files(silent, pk_args, expected_pks): ) assert result.exit_code == 0, result.stdout db = Database(db_path) - rows_by_path = {r["path"]: r for r in db["files"].rows} + rows_by_path = {r["path"]: r for r in db.table("files").rows} one, two, three = ( rows_by_path["one.txt"], rows_by_path["two.txt"], @@ -114,7 +114,7 @@ def test_insert_files(silent, pk_args, expected_pks): for colname, expected_type in expected_types.items(): for row in (one, two, three): assert isinstance(row[colname], expected_type) - assert set(db["files"].pks) == set(expected_pks) + assert set(db.table("files").pks) == set(expected_pks) @pytest.mark.parametrize( @@ -144,7 +144,7 @@ def test_insert_files_stdin(use_text, encoding, input, expected): ) assert result.exit_code == 0, result.stdout db = Database(db_path) - row = next(iter(db["files"].rows)) + row = next(iter(db.table("files").rows)) key = "content" if use_text: key = "content_text" diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 2a8d579..b0953f1 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,6 +1,6 @@ import pytest -from sqlite_utils.db import Check, Database, Index, View, XIndex, XIndexColumn +from sqlite_utils.db import Check, Database, Index, Table, View, XIndex, XIndexColumn def _check_supports_strict(): @@ -21,10 +21,10 @@ def test_view_names(fresh_db): def test_table_names_fts4(existing_db): - existing_db["woo"].insert({"title": "Hello"}).enable_fts( + existing_db.table("woo").insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" ) - existing_db["woo2"].insert({"title": "Hello"}).enable_fts( + existing_db.table("woo2").insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS5" ) assert ["woo_fts"] == existing_db.table_names(fts4=True) @@ -32,17 +32,17 @@ def test_table_names_fts4(existing_db): def test_detect_fts(existing_db): - existing_db["woo"].insert({"title": "Hello"}).enable_fts( + existing_db.table("woo").insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" ) - existing_db["woo2"].insert({"title": "Hello"}).enable_fts( + existing_db.table("woo2").insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS5" ) - assert "woo_fts" == existing_db["woo"].detect_fts() - assert "woo_fts" == existing_db["woo_fts"].detect_fts() - assert "woo2_fts" == existing_db["woo2"].detect_fts() - assert "woo2_fts" == existing_db["woo2_fts"].detect_fts() - assert existing_db["foo"].detect_fts() is None + assert "woo_fts" == existing_db.table("woo").detect_fts() + assert "woo_fts" == existing_db.table("woo_fts").detect_fts() + assert "woo2_fts" == existing_db.table("woo2").detect_fts() + assert "woo2_fts" == existing_db.table("woo2_fts").detect_fts() + assert existing_db.table("foo").detect_fts() is None @pytest.mark.parametrize("reverse_order", (True, False)) @@ -52,14 +52,14 @@ def test_detect_fts_similar_tables(fresh_db, reverse_order): if reverse_order: table1, table2 = table2, table1 - fresh_db[table1].insert({"title": "Hello"}).enable_fts( + fresh_db.table(table1).insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" ) - fresh_db[table2].insert({"title": "Hello"}).enable_fts( + fresh_db.table(table2).insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" ) - assert fresh_db[table1].detect_fts() == f"{table1}_fts" - assert fresh_db[table2].detect_fts() == f"{table2}_fts" + assert fresh_db.table(table1).detect_fts() == f"{table1}_fts" + assert fresh_db.table(table2).detect_fts() == f"{table2}_fts" def test_tables(existing_db): @@ -77,26 +77,34 @@ def test_views(fresh_db): assert view.columns_dict == {"1": str} +def test_getitem_returns_table_or_view(fresh_db): + fresh_db.table("items").insert({"id": 1}, pk="id") + fresh_db.create_view("item_ids", "select id from items") + + assert isinstance(fresh_db["items"], Table) + assert isinstance(fresh_db["item_ids"], View) + + def test_count(existing_db): - assert existing_db["foo"].count == 3 - assert existing_db["foo"].count_where() == 3 - assert existing_db["foo"].execute_count() == 3 + assert existing_db.table("foo").count == 3 + assert existing_db.table("foo").count_where() == 3 + assert existing_db.table("foo").execute_count() == 3 def test_count_where(existing_db): - assert existing_db["foo"].count_where("text != ?", ["two"]) == 2 - assert existing_db["foo"].count_where("text != :t", {"t": "two"}) == 2 + assert existing_db.table("foo").count_where("text != ?", ["two"]) == 2 + assert existing_db.table("foo").count_where("text != :t", {"t": "two"}) == 2 def test_columns(existing_db): - table = existing_db["foo"] + table = existing_db.table("foo") assert [{"name": "text", "type": "TEXT"}] == [ {"name": col.name, "type": col.type} for col in table.columns ] def test_table_schema(existing_db): - assert existing_db["foo"].schema == "CREATE TABLE foo (text TEXT)" + assert existing_db.table("foo").schema == "CREATE TABLE foo (text TEXT)" def test_database_schema(existing_db): @@ -104,9 +112,9 @@ def test_database_schema(existing_db): def test_table_repr(fresh_db): - table = fresh_db["dogs"].insert({"name": "Cleo", "age": 4}) + table = fresh_db.table("dogs").insert({"name": "Cleo", "age": 4}) assert "
" == repr(table) - assert "
" == repr(fresh_db["cats"]) + assert "
" == repr(fresh_db.table("cats")) def test_indexes(fresh_db): @@ -125,7 +133,7 @@ def test_indexes(fresh_db): columns=["c2", "c3"], ), Index(seq=1, name="Gosh_c1", unique=0, origin="c", partial=0, columns=["c1"]), - ] == fresh_db["Gosh"].indexes + ] == fresh_db.table("Gosh").indexes def test_xindexes(fresh_db): @@ -134,7 +142,7 @@ def test_xindexes(fresh_db): create index Gosh_c1 on Gosh(c1); create index Gosh_c2c3 on Gosh(c2, c3 desc); """) - assert fresh_db["Gosh"].xindexes == [ + assert fresh_db.table("Gosh").xindexes == [ XIndex( name="Gosh_c2c3", columns=[ @@ -166,15 +174,15 @@ def test_xindexes(fresh_db): def test_guess_foreign_table(fresh_db, column, expected_table_guess): fresh_db.create_table("authors", {"name": str}) fresh_db.create_table("genre", {"name": str}) - assert expected_table_guess == fresh_db["books"].guess_foreign_table(column) + assert expected_table_guess == fresh_db.table("books").guess_foreign_table(column) @pytest.mark.parametrize( "pk,expected", ((None, ["rowid"]), ("id", ["id"]), (["id", "id2"], ["id", "id2"])) ) def test_pks(fresh_db, pk, expected): - fresh_db["foo"].insert_all([{"id": 1, "id2": 2}], pk=pk) - assert expected == fresh_db["foo"].pks + fresh_db.table("foo").insert_all([{"id": 1, "id2": 2}], pk=pk) + assert expected == fresh_db.table("foo").pks def test_checks(fresh_db): @@ -185,7 +193,7 @@ def test_checks(fresh_db): CONSTRAINT within_maximum CHECK(score <= maximum) ) """) - scores = fresh_db["scores"] + scores = fresh_db.table("scores") expected_column = Check("score > 0", name="positive", column="score") expected_table = Check("score <= maximum", name="within_maximum") assert scores.checks == [expected_column, expected_table] @@ -195,26 +203,26 @@ def test_checks(fresh_db): def test_checks_nonexistent_and_virtual_tables(fresh_db): - assert fresh_db["does_not_exist"].checks == [] - fresh_db["searchable"].insert({"text": "hello"}).enable_fts( + assert fresh_db.table("does_not_exist").checks == [] + fresh_db.table("searchable").insert({"text": "hello"}).enable_fts( ["text"], fts_version="FTS5" ) - assert fresh_db["searchable_fts"].checks == [] + assert fresh_db.table("searchable_fts").checks == [] def test_triggers_and_triggers_dict(fresh_db): assert [] == fresh_db.triggers - authors = fresh_db["authors"] + authors = fresh_db.table("authors") authors.insert_all( [ {"name": "Frank Herbert", "famous_works": "Dune"}, {"name": "Neal Stephenson", "famous_works": "Cryptonomicon"}, ] ) - fresh_db["other"].insert({"foo": "bar"}) + fresh_db.table("other").insert({"foo": "bar"}) assert authors.triggers == [] assert authors.triggers_dict == {} - assert fresh_db["other"].triggers == [] + assert fresh_db.table("other").triggers == [] assert fresh_db.triggers_dict == {} authors.enable_fts( ["name", "famous_works"], fts_version="FTS4", create_triggers=True @@ -226,7 +234,7 @@ def test_triggers_and_triggers_dict(fresh_db): } assert expected_triggers == {(t.name, t.table) for t in fresh_db.triggers} assert expected_triggers == { - (t.name, t.table) for t in fresh_db["authors"].triggers + (t.name, t.table) for t in fresh_db.table("authors").triggers } expected_triggers = { "authors_ai": ( @@ -246,13 +254,13 @@ def test_triggers_and_triggers_dict(fresh_db): ), } assert authors.triggers_dict == expected_triggers - assert fresh_db["other"].triggers == [] - assert fresh_db["other"].triggers_dict == {} + assert fresh_db.table("other").triggers == [] + assert fresh_db.table("other").triggers_dict == {} assert fresh_db.triggers_dict == expected_triggers def test_has_counts_triggers(fresh_db): - authors = fresh_db["authors"] + authors = fresh_db.table("authors") authors.insert({"name": "Frank Herbert"}) assert not authors.has_counts_triggers authors.enable_counts() @@ -301,14 +309,14 @@ def test_has_counts_triggers(fresh_db): ) def test_virtual_table_using(fresh_db, sql, expected_name, expected_using): fresh_db.execute(sql) - assert fresh_db[expected_name].virtual_table_using == expected_using + assert fresh_db.table(expected_name).virtual_table_using == expected_using def test_use_rowid(fresh_db): - fresh_db["rowid_table"].insert({"name": "Cleo"}) - fresh_db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id") - assert fresh_db["rowid_table"].use_rowid - assert not fresh_db["regular_table"].use_rowid + fresh_db.table("rowid_table").insert({"name": "Cleo"}) + fresh_db.table("regular_table").insert({"id": 1, "name": "Cleo"}, pk="id") + assert fresh_db.table("rowid_table").use_rowid + assert not fresh_db.table("regular_table").use_rowid @pytest.mark.skipif( @@ -327,7 +335,7 @@ def test_use_rowid(fresh_db): ) def test_table_strict(fresh_db, create_table, expected_strict): fresh_db.execute(create_table) - table = fresh_db["t"] + table = fresh_db.table("t") assert table.strict == expected_strict @@ -343,10 +351,10 @@ def test_table_strict(fresh_db, create_table, expected_strict): ), ) def test_table_default_values(fresh_db, value): - fresh_db["default_values"].insert( + fresh_db.table("default_values").insert( {"nodefault": 1, "value": value}, defaults={"value": value} ) - default_values = fresh_db["default_values"].default_values + default_values = fresh_db.table("default_values").default_values assert default_values == {"value": value} @@ -356,8 +364,8 @@ def test_table_default_values_escaped_quotes(fresh_db): fresh_db.execute( "create table t (id integer primary key, name text default 'O''Brien')" ) - assert "default 'O''Brien'" in fresh_db["t"].schema - assert fresh_db["t"].default_values == {"name": "O'Brien"} + assert "default 'O''Brien'" in fresh_db.table("t").schema + assert fresh_db.table("t").default_values == {"name": "O'Brien"} def test_pks_use_primary_key_declaration_order(fresh_db): @@ -365,11 +373,11 @@ def test_pks_use_primary_key_declaration_order(fresh_db): # pks must follow the declaration order, which is what SQLite uses to # resolve implicit foreign key references and compound pk lookups fresh_db.execute("create table t (b text, a text, primary key (a, b))") - assert fresh_db["t"].pks == ["a", "b"] + assert fresh_db.table("t").pks == ["a", "b"] def test_transform_preserves_compound_pk_declaration_order(fresh_db): fresh_db.execute("create table t (a text, b text, c text, primary key (b, a))") - fresh_db["t"].transform(drop={"c"}) - assert fresh_db["t"].pks == ["b", "a"] - assert 'PRIMARY KEY ("b", "a")' in fresh_db["t"].schema + fresh_db.table("t").transform(drop={"c"}) + assert fresh_db.table("t").pks == ["b", "a"] + assert 'PRIMARY KEY ("b", "a")' in fresh_db.table("t").schema diff --git a/tests/test_list_mode.py b/tests/test_list_mode.py index 646098e..b9ab812 100644 --- a/tests/test_list_mode.py +++ b/tests/test_list_mode.py @@ -19,9 +19,9 @@ def test_insert_all_list_mode_basic(): yield [2, "Bob", 25] yield [3, "Charlie", 35] - db["people"].insert_all(data_generator()) + db.table("people").insert_all(data_generator()) - rows = list(db["people"].rows) + rows = list(db.table("people").rows) assert len(rows) == 3 assert rows[0] == {"id": 1, "name": "Alice", "age": 30} assert rows[1] == {"id": 2, "name": "Bob", "age": 25} @@ -37,10 +37,10 @@ def test_insert_all_list_mode_with_pk(): yield [1, "Alice", 95] yield [2, "Bob", 87] - db["scores"].insert_all(data_generator(), pk="id") + db.table("scores").insert_all(data_generator(), pk="id") - assert db["scores"].pks == ["id"] - rows = list(db["scores"].rows) + assert db.table("scores").pks == ["id"] + rows = list(db.table("scores").rows) assert len(rows) == 2 @@ -54,7 +54,7 @@ def test_upsert_all_list_mode(): yield [1, "Alice", 100] yield [2, "Bob", 200] - db["data"].insert_all(initial_data(), pk="id") + db.table("data").insert_all(initial_data(), pk="id") # Upsert with some updates and new records def upsert_data(): @@ -62,9 +62,9 @@ def test_upsert_all_list_mode(): yield [1, "Alice", 150] # Update existing yield [3, "Charlie", 300] # Insert new - db["data"].upsert_all(upsert_data(), pk="id") + db.table("data").upsert_all(upsert_data(), pk="id") - rows = list(db["data"].rows_where(order_by="id")) + rows = list(db.table("data").rows_where(order_by="id")) assert len(rows) == 3 assert rows[0] == {"id": 1, "name": "Alice", "value": 150} assert rows[1] == {"id": 2, "name": "Bob", "value": 200} @@ -81,9 +81,9 @@ def test_list_mode_with_various_types(): yield [2, "Bob", 87.3, False] yield [3, "Charlie", None, True] - db["mixed"].insert_all(data_generator()) + db.table("mixed").insert_all(data_generator()) - rows = list(db["mixed"].rows) + rows = list(db.table("mixed").rows) assert len(rows) == 3 assert rows[0]["score"] == 95.5 assert rows[1]["active"] == 0 # SQLite stores boolean as int @@ -99,7 +99,7 @@ def test_list_mode_error_non_string_columns(): yield ["a", "b", "c"] with pytest.raises(ValueError, match="must be a list of column name strings"): - db["bad"].insert_all(bad_data()) + db.table("bad").insert_all(bad_data()) def test_list_mode_error_mixed_types(): @@ -111,7 +111,7 @@ def test_list_mode_error_mixed_types(): yield {"id": 1, "name": "Alice"} # Should be a list, not dict with pytest.raises(ValueError, match="must also be lists"): - db["bad"].insert_all(bad_data()) + db.table("bad").insert_all(bad_data()) def test_list_mode_empty_after_headers(): @@ -122,9 +122,9 @@ def test_list_mode_empty_after_headers(): yield ["id", "name", "age"] # No data rows - result = db["people"].insert_all(data_generator()) + result = db.table("people").insert_all(data_generator()) assert result is not None - assert not db["people"].exists() + assert not db.table("people").exists() def test_list_mode_batch_processing(): @@ -136,7 +136,7 @@ def test_list_mode_batch_processing(): for i in range(1000): yield [i, f"value_{i}"] - db["large"].insert_all(large_data(), batch_size=100) + db.table("large").insert_all(large_data(), batch_size=100) count = db.execute("SELECT COUNT(*) as c FROM large").fetchone()[0] assert count == 1000 @@ -152,9 +152,9 @@ def test_list_mode_shorter_rows(): yield [2, "Bob"] # Missing age and city yield [3, "Charlie", 35] # Missing city - db["people"].insert_all(data_generator()) + db.table("people").insert_all(data_generator()) - rows = list(db["people"].rows_where(order_by="id")) + rows = list(db.table("people").rows_where(order_by="id")) assert rows[0] == {"id": 1, "name": "Alice", "age": 30, "city": "NYC"} assert rows[1] == {"id": 2, "name": "Bob", "age": None, "city": None} assert rows[2] == {"id": 3, "name": "Charlie", "age": 35, "city": None} @@ -170,9 +170,9 @@ def test_backwards_compatibility_dict_mode(): {"id": 2, "name": "Bob", "age": 25}, ] - db["people"].insert_all(data) + db.table("people").insert_all(data) - rows = list(db["people"].rows) + rows = list(db.table("people").rows) assert len(rows) == 2 assert rows[0] == {"id": 1, "name": "Alice", "age": 30} @@ -189,9 +189,9 @@ def test_insert_all_tuple_mode_basic(): yield (2, "Bob", 25) yield (3, "Charlie", 35) - db["people"].insert_all(data_generator()) + db.table("people").insert_all(data_generator()) - rows = list(db["people"].rows) + rows = list(db.table("people").rows) assert len(rows) == 3 assert rows[0] == {"id": 1, "name": "Alice", "age": 30} assert rows[1] == {"id": 2, "name": "Bob", "age": 25} @@ -211,9 +211,9 @@ def test_insert_all_mixed_list_tuple(): yield [3, "Charlie", 35] yield (4, "Diana", 40) - db["people"].insert_all(data_generator()) + db.table("people").insert_all(data_generator()) - rows = list(db["people"].rows) + rows = list(db.table("people").rows) assert len(rows) == 4 assert rows[0] == {"id": 1, "name": "Alice", "age": 30} assert rows[1] == {"id": 2, "name": "Bob", "age": 25} @@ -231,7 +231,7 @@ def test_upsert_all_tuple_mode(): yield (1, "Alice", 100) yield (2, "Bob", 200) - db["data"].insert_all(initial_data(), pk="id") + db.table("data").insert_all(initial_data(), pk="id") # Upsert with tuples def upsert_data(): @@ -239,9 +239,9 @@ def test_upsert_all_tuple_mode(): yield (1, "Alice", 150) # Update existing yield (3, "Charlie", 300) # Insert new - db["data"].upsert_all(upsert_data(), pk="id") + db.table("data").upsert_all(upsert_data(), pk="id") - rows = list(db["data"].rows_where(order_by="id")) + rows = list(db.table("data").rows_where(order_by="id")) assert len(rows) == 3 assert rows[0] == {"id": 1, "name": "Alice", "value": 150} assert rows[1] == {"id": 2, "name": "Bob", "value": 200} @@ -258,9 +258,9 @@ def test_tuple_mode_shorter_rows(): yield 2, "Bob" # Missing age and city yield 3, "Charlie", 35 # Missing city - db["people"].insert_all(data_generator()) + db.table("people").insert_all(data_generator()) - rows = list(db["people"].rows_where(order_by="id")) + rows = list(db.table("people").rows_where(order_by="id")) assert rows[0] == {"id": 1, "name": "Alice", "age": 30, "city": "NYC"} assert rows[1] == {"id": 2, "name": "Bob", "age": None, "city": None} assert rows[2] == {"id": 3, "name": "Charlie", "age": 35, "city": None} @@ -271,18 +271,18 @@ def test_list_mode_single_record_upsert_last_pk(): db = Database(memory=True) # Create table first - db["data"].insert({"id": 1, "name": "Alice", "value": 100}, pk="id") + db.table("data").insert({"id": 1, "name": "Alice", "value": 100}, pk="id") # Now upsert a single record using list mode def upsert_data(): yield ["id", "name", "value"] yield [1, "Alice", 150] # Update existing - table = db["data"] + table = db.table("data") table.upsert_all(upsert_data(), pk="id") # Verify the data was updated - rows = list(db["data"].rows) + rows = list(db.table("data").rows) assert rows == [{"id": 1, "name": "Alice", "value": 150}] # Verify last_pk is populated correctly diff --git a/tests/test_lookup.py b/tests/test_lookup.py index c93d1ed..f96cfef 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -4,7 +4,7 @@ from sqlite_utils.db import Index def test_lookup_new_table(fresh_db): - species = fresh_db["species"] + species = fresh_db.table("species") palm_id = species.lookup({"name": "Palm"}) oak_id = species.lookup({"name": "Oak"}) cherry_id = species.lookup({"name": "Cherry"}) @@ -26,7 +26,7 @@ def test_lookup_new_table(fresh_db): def test_lookup_new_table_compound_key(fresh_db): - species = fresh_db["species"] + species = fresh_db.table("species") palm_id = species.lookup({"name": "Palm", "type": "Tree"}) oak_id = species.lookup({"name": "Oak", "type": "Tree"}) assert palm_id == species.lookup({"name": "Palm", "type": "Tree"}) @@ -70,7 +70,7 @@ def test_lookup_fails_if_constraint_cannot_be_added(fresh_db): def test_lookup_with_extra_values(fresh_db): - species = fresh_db["species"] + species = fresh_db.table("species") id = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2020-01-01"}) assert species.get(id) == { "id": 1, @@ -90,9 +90,9 @@ def test_lookup_with_extra_values(fresh_db): def test_lookup_with_extra_insert_parameters(fresh_db): - other_table = fresh_db["other_table"] + other_table = fresh_db.table("other_table") other_table.insert({"id": 1, "name": "Name"}, pk="id") - species = fresh_db["species"] + species = fresh_db.table("species") id = species.lookup( {"name": "Palm", "type": "Tree"}, { @@ -156,15 +156,15 @@ def test_lookup_with_extra_insert_parameters(fresh_db): @pytest.mark.parametrize("strict", (False, True)) def test_lookup_new_table_strict(fresh_db, strict): - fresh_db["species"].lookup({"name": "Palm"}, strict=strict) - assert fresh_db["species"].strict == strict or not fresh_db.supports_strict + fresh_db.table("species").lookup({"name": "Palm"}, strict=strict) + assert fresh_db.table("species").strict == strict or not fresh_db.supports_strict def test_lookup_null_value_idempotent(fresh_db): # https://github.com/simonw/sqlite-utils/issues/186 # Repeated lookups of a null value should return the same row, # not insert a duplicate row each time - species = fresh_db["species"] + species = fresh_db.table("species") first_id = species.lookup({"name": None}) second_id = species.lookup({"name": None}) assert first_id == second_id @@ -172,7 +172,7 @@ def test_lookup_null_value_idempotent(fresh_db): def test_lookup_compound_key_with_null_idempotent(fresh_db): - species = fresh_db["species"] + species = fresh_db.table("species") palm_id = species.lookup({"name": "Palm", "type": None}) oak_id = species.lookup({"name": "Oak", "type": "Tree"}) assert palm_id == species.lookup({"name": "Palm", "type": None}) diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 4fca918..4dde7e4 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -4,45 +4,45 @@ from sqlite_utils.db import ForeignKey, NoObviousTable def test_insert_m2m_single(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( "humans", {"id": 1, "name": "Natalie D"}, pk="id" ) assert {"dogs_humans", "humans", "dogs"} == set(fresh_db.table_names()) - humans = fresh_db["humans"] - dogs_humans = fresh_db["dogs_humans"] + humans = fresh_db.table("humans") + dogs_humans = fresh_db.table("dogs_humans") assert [{"id": 1, "name": "Natalie D"}] == list(humans.rows) assert [{"humans_id": 1, "dogs_id": 1}] == list(dogs_humans.rows) def test_insert_m2m_alter(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( "humans", {"id": 1, "name": "Natalie D"}, pk="id" ) dogs.update(1).m2m( "humans", {"id": 2, "name": "Simon W", "nerd": True}, pk="id", alter=True ) - assert list(fresh_db["humans"].rows) == [ + assert list(fresh_db.table("humans").rows) == [ {"id": 1, "name": "Natalie D", "nerd": None}, {"id": 2, "name": "Simon W", "nerd": 1}, ] - assert list(fresh_db["dogs_humans"].rows) == [ + assert list(fresh_db.table("dogs_humans").rows) == [ {"humans_id": 1, "dogs_id": 1}, {"humans_id": 2, "dogs_id": 1}, ] def test_insert_m2m_list(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( "humans", [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}], pk="id", ) assert {"dogs", "humans", "dogs_humans"} == set(fresh_db.table_names()) - humans = fresh_db["humans"] - dogs_humans = fresh_db["dogs_humans"] + humans = fresh_db.table("humans") + dogs_humans = fresh_db.table("dogs_humans") assert [{"humans_id": 1, "dogs_id": 1}, {"humans_id": 2, "dogs_id": 1}] == list( dogs_humans.rows ) @@ -68,7 +68,7 @@ def test_insert_m2m_iterable(fresh_db): def iterable(): yield from iterable_records - platypuses = fresh_db["platypuses"] + platypuses = fresh_db.table("platypuses") platypuses.insert({"id": 1, "name": "Perry"}, pk="id").m2m( "humans", iterable(), @@ -76,8 +76,8 @@ def test_insert_m2m_iterable(fresh_db): ) assert {"platypuses", "humans", "humans_platypuses"} == set(fresh_db.table_names()) - humans = fresh_db["humans"] - humans_platypuses = fresh_db["humans_platypuses"] + humans = fresh_db.table("humans") + humans_platypuses = fresh_db.table("humans_platypuses") assert [ {"humans_id": 1, "platypuses_id": 1}, {"humans_id": 2, "platypuses_id": 1}, @@ -111,14 +111,14 @@ def test_m2m_with_table_objects(fresh_db): assert expected_tables == set(fresh_db.table_names()) assert dogs.count == 1 assert humans.count == 2 - assert fresh_db["dogs_humans"].count == 2 + assert fresh_db.table("dogs_humans").count == 2 def test_m2m_lookup(fresh_db): people = fresh_db.table("people", pk="id") people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) - people_tags = fresh_db["people_tags"] - tags = fresh_db["tags"] + people_tags = fresh_db.table("people_tags") + tags = fresh_db.table("tags") assert people_tags.exists() assert tags.exists() assert [ @@ -150,9 +150,9 @@ def test_m2m_explicit_table_name_argument(fresh_db): people.insert({"name": "Wahyu"}).m2m( "tags", lookup={"tag": "Coworker"}, m2m_table="tagged" ) - assert fresh_db["tags"].exists - assert fresh_db["tagged"].exists - assert not fresh_db["people_tags"].exists() + assert fresh_db.table("tags").exists + assert fresh_db.table("tagged").exists + assert not fresh_db.table("people_tags").exists() def test_m2m_table_candidates(fresh_db): @@ -181,25 +181,25 @@ def test_uses_existing_m2m_table_if_exists(fresh_db): # Code should look for an existing table with fks to both tables # and use that if it exists. people = fresh_db.create_table("people", {"id": int, "name": str}, pk="id") - fresh_db["tags"].lookup({"tag": "Coworker"}) + fresh_db.table("tags").lookup({"tag": "Coworker"}) fresh_db.create_table( "tagged", {"people_id": int, "tags_id": int}, foreign_keys=["people_id", "tags_id"], ) people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) - assert fresh_db["tags"].exists() - assert fresh_db["tagged"].exists() - assert not fresh_db["people_tags"].exists() - assert not fresh_db["tags_people"].exists() - assert [{"people_id": 1, "tags_id": 1}] == list(fresh_db["tagged"].rows) + assert fresh_db.table("tags").exists() + assert fresh_db.table("tagged").exists() + assert not fresh_db.table("people_tags").exists() + assert not fresh_db.table("tags_people").exists() + assert [{"people_id": 1, "tags_id": 1}] == list(fresh_db.table("tagged").rows) def test_requires_explicit_m2m_table_if_multiple_options(fresh_db): # If the code scans for m2m tables and finds more than one candidate # it should require that the m2m_table=x argument is used people = fresh_db.create_table("people", {"id": int, "name": str}, pk="id") - fresh_db["tags"].lookup({"tag": "Coworker"}) + fresh_db.table("tags").lookup({"tag": "Coworker"}) fresh_db.create_table( "tagged", {"people_id": int, "tags_id": int}, diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 3f3dfea..fa419ec 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -10,11 +10,11 @@ def migrations(): @migrations() def m001(db): - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) @migrations() def m002(db): - db["cats"].create({"name": str}) + db.table("cats").create({"name": str}) db.execute("insert into dogs (name) values ('Pancakes')") return migrations @@ -28,11 +28,11 @@ def migrations_not_ordered_alphabetically(): @migrations() def m002(db): - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) @migrations() def m001(db): - db["cats"].create({"name": str}) + db.table("cats").create({"name": str}) db.execute("insert into dogs (name) values ('Pancakes')") return migrations @@ -44,7 +44,7 @@ def migrations2(): @migrations() def m001(db): - db["dogs2"].insert({"name": "Cleo"}) + db.table("dogs2").insert({"name": "Cleo"}) return migrations @@ -96,7 +96,7 @@ def test_applied_at_is_a_string(migrations): def test_failing_migration_rolls_back(migrations): @migrations() def m003(db): - db["birds"].create({"name": str}) + db.table("birds").create({"name": str}) db.execute("insert into dogs (name) values ('Dozer')") raise ValueError("boom") @@ -105,7 +105,7 @@ def test_failing_migration_rolls_back(migrations): migrations.apply(db) # m001 and m002 committed before the failure and stay applied assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"} - assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"] + assert [r["name"] for r in db.table("dogs").rows] == ["Cleo", "Pancakes"] assert [m.name for m in migrations.applied(db)] == ["m001", "m002"] # Everything m003 did was rolled back and it is still pending assert [m.name for m in migrations.pending(db)] == ["m003"] @@ -117,11 +117,11 @@ def test_rerun_after_failure_applies_each_migration_once(): @migrations() def m001(db): - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) @migrations() def m002(db): - db["dogs"].insert({"name": "Pancakes"}) + db.table("dogs").insert({"name": "Pancakes"}) if state["fail"]: raise ValueError("boom") @@ -131,7 +131,7 @@ def test_rerun_after_failure_applies_each_migration_once(): state["fail"] = False migrations.apply(db) # m001 must not have been re-applied, m002 applied exactly once - assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"] + assert [r["name"] for r in db.table("dogs").rows] == ["Cleo", "Pancakes"] assert [m.name for m in migrations.applied(db)] == ["m001", "m002"] @@ -142,7 +142,7 @@ def test_non_transactional_migration_allows_vacuum(tmpdir): @migrations() def m001(db): - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) @migrations(transactional=False) def m002(db): @@ -185,11 +185,13 @@ def test_apply_composes_inside_outer_transaction(migrations): ) def test_upgrades_sqlite_migrations(migrations, create_table, pk): db = sqlite_utils.Database(memory=True) - db["_sqlite_migrations"].create(create_table, pk=pk) + db.table("_sqlite_migrations").create(create_table, pk=pk) assert db.table_names() == ["_sqlite_migrations"] - assert db["_sqlite_migrations"].pks == ([pk] if isinstance(pk, str) else list(pk)) + assert db.table("_sqlite_migrations").pks == ( + [pk] if isinstance(pk, str) else list(pk) + ) migrations.apply(db) - assert db["_sqlite_migrations"].pks == ["id"] + assert db.table("_sqlite_migrations").pks == ["id"] def test_pending_and_applied_are_read_only(migrations): @@ -227,7 +229,7 @@ def test_stop_before_applied_migration_errors(migrations): assert "m001" in str(ex.value) assert "already been applied" in str(ex.value) # Nothing else was applied - assert not db["cats"].exists() + assert not db.table("cats").exists() def test_stop_before_applied_migration_errors_before_any_apply(migrations): @@ -238,9 +240,9 @@ def test_stop_before_applied_migration_errors_before_any_apply(migrations): @only_second() def m002(db): - db["cats"].create({"name": str}) + db.table("cats").create({"name": str}) only_second.apply(db) # m002 applied, m001 still pending with pytest.raises(ValueError): migrations.apply(db, stop_before="m002") - assert not db["dogs"].exists() + assert not db.table("dogs").exists() diff --git a/tests/test_mutator_transactions.py b/tests/test_mutator_transactions.py index 37ae1b6..3f13c6b 100644 --- a/tests/test_mutator_transactions.py +++ b/tests/test_mutator_transactions.py @@ -112,7 +112,7 @@ def test_mutator_commits_by_default(tmp_path, mutate, expected_rows): db = seed_database(path) assert not db.conn.in_transaction - mutate(db["items"]) + mutate(db.table("items")) assert current_rows(db) == expected_rows assert not db.conn.in_transaction @@ -127,7 +127,7 @@ def test_mutator_commits_with_outer_atomic(tmp_path, mutate, expected_rows): with db.atomic(): assert db.conn.in_transaction - mutate(db["items"]) + mutate(db.table("items")) assert current_rows(db) == expected_rows assert db.conn.in_transaction @@ -143,7 +143,7 @@ def test_mutator_rolls_back_outer_atomic(tmp_path, mutate, expected_rows): db = seed_database(path) with pytest.raises(RollbackTest), db.atomic(): - mutate(db["items"]) + mutate(db.table("items")) assert current_rows(db) == expected_rows assert db.conn.in_transaction raise RollbackTest diff --git a/tests/test_query.py b/tests/test_query.py index 9d79755..ac0d924 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -6,7 +6,7 @@ from sqlite_utils.utils import sqlite3 def test_query(fresh_db): - fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) + fresh_db.table("dogs").insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) results = fresh_db.query("select * from dogs order by name desc") assert isinstance(results, types.GeneratorType) assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}] @@ -20,13 +20,13 @@ def test_query_executes_eagerly(fresh_db): def test_query_rejects_statements_that_return_no_rows(fresh_db): - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) with pytest.raises(ValueError) as ex: fresh_db.query("update dogs set name = 'Cleopaws'") assert "execute()" in str(ex.value) # The rejected update was rolled back, and no transaction is left open assert not fresh_db.conn.in_transaction - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + assert [row["name"] for row in fresh_db.table("dogs").rows] == ["Cleo"] def test_query_rejected_ddl_is_rolled_back(fresh_db): @@ -37,7 +37,7 @@ def test_query_rejected_ddl_is_rolled_back(fresh_db): def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db): - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) fresh_db.begin() fresh_db.execute("insert into dogs (name) values ('Pancakes')") with pytest.raises(ValueError): @@ -45,7 +45,7 @@ def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db): # The transaction is still open and the earlier insert is intact assert fresh_db.conn.in_transaction fresh_db.commit() - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo", "Pancakes"] + assert [row["name"] for row in fresh_db.table("dogs").rows] == ["Cleo", "Pancakes"] @pytest.mark.parametrize( @@ -77,7 +77,7 @@ def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db): # A COMMIT hidden behind a leading comment must not slip past the # keyword check - previously it committed the caller's open # transaction before the ValueError was raised - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) fresh_db.begin() fresh_db.execute("insert into dogs (name) values ('Pancakes')") with pytest.raises(ValueError): @@ -85,7 +85,7 @@ def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db): # The explicit transaction is still open and can still be rolled back assert fresh_db.conn.in_transaction fresh_db.rollback() - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + assert [row["name"] for row in fresh_db.table("dogs").rows] == ["Cleo"] @pytest.mark.parametrize("sql", ["; COMMIT", "\ufeffCOMMIT"]) @@ -94,7 +94,7 @@ def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql): # real token, so the keyword scanner must skip them too - previously # '; COMMIT' slipped past the check and committed the caller's open # transaction before raising OperationalError - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) fresh_db.begin() fresh_db.execute("insert into dogs (name) values ('Pancakes')") with pytest.raises(ValueError): @@ -102,7 +102,7 @@ def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql): # The explicit transaction is still open and can still be rolled back assert fresh_db.conn.in_transaction fresh_db.rollback() - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + assert [row["name"] for row in fresh_db.table("dogs").rows] == ["Cleo"] def test_query_error_leaves_no_transaction_open(fresh_db): @@ -190,12 +190,12 @@ def test_first_keyword(sql, expected): reason="RETURNING requires SQLite 3.35.0 or higher", ) def test_query_insert_returning(fresh_db): - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) rows = list( fresh_db.query("insert into dogs (name) values ('Pancakes') returning name") ) assert rows == [{"name": "Pancakes"}] - assert fresh_db["dogs"].count == 2 + assert fresh_db.table("dogs").count == 2 @pytest.mark.skipif( @@ -207,7 +207,7 @@ def test_query_insert_returning_commits_without_iteration(tmpdir): path = str(tmpdir / "test.db") db = Database(path) - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) # Never iterate over the results db.query("insert into dogs (name) values ('Pancakes') returning name") assert not db.conn.in_transaction @@ -227,7 +227,7 @@ def test_query_insert_returning_partial_iteration_still_commits(tmpdir): path = str(tmpdir / "test.db") db = Database(path) - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) row = next( db.query( "insert into dogs (name) values ('Pancakes'), ('Marnie') returning name" @@ -246,7 +246,7 @@ def test_query_insert_returning_partial_iteration_still_commits(tmpdir): reason="RETURNING requires SQLite 3.35.0 or higher", ) def test_query_insert_returning_respects_explicit_transaction(fresh_db): - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) fresh_db.begin() rows = list( fresh_db.query("insert into dogs (name) values ('Pancakes') returning name") @@ -255,13 +255,13 @@ def test_query_insert_returning_respects_explicit_transaction(fresh_db): # Still inside the explicit transaction - not committed assert fresh_db.conn.in_transaction fresh_db.rollback() - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + assert [row["name"] for row in fresh_db.table("dogs").rows] == ["Cleo"] def test_query_duplicate_column_names_are_deduped(fresh_db): # https://github.com/simonw/sqlite-utils/issues/624 - fresh_db["one"].insert({"id": 1, "value": "left"}) - fresh_db["two"].insert({"id": 2, "value": "right"}) + fresh_db.table("one").insert({"id": 1, "value": "left"}) + fresh_db.table("two").insert({"id": 2, "value": "right"}) rows = list( fresh_db.query("select one.id, two.id, one.value, two.value from one, two") ) @@ -277,7 +277,7 @@ def test_query_deduped_column_avoids_existing_names(fresh_db): def test_execute_returning_dicts(fresh_db): # Like db.query() but returns a list, included for backwards compatibility # see https://github.com/simonw/sqlite-utils/issues/290 - fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") + fresh_db.table("test").insert({"id": 1, "bar": 2}, pk="id") assert fresh_db.execute_returning_dicts("select * from test") == [ {"id": 1, "bar": 2} ] diff --git a/tests/test_recipes.py b/tests/test_recipes.py index c6222a3..c6a548c 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -8,7 +8,7 @@ from sqlite_utils.utils import sqlite3 @pytest.fixture def dates_db(fresh_db): - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "dt": "5th October 2019 12:04"}, {"id": 2, "dt": "6th October 2019 00:05:06"}, @@ -21,8 +21,8 @@ def dates_db(fresh_db): def test_parsedate(dates_db): - dates_db["example"].convert("dt", recipes.parsedate) - assert list(dates_db["example"].rows) == [ + dates_db.table("example").convert("dt", recipes.parsedate) + assert list(dates_db.table("example").rows) == [ {"id": 1, "dt": "2019-10-05"}, {"id": 2, "dt": "2019-10-06"}, {"id": 3, "dt": ""}, @@ -31,8 +31,8 @@ def test_parsedate(dates_db): def test_parsedatetime(dates_db): - dates_db["example"].convert("dt", recipes.parsedatetime) - assert list(dates_db["example"].rows) == [ + dates_db.table("example").convert("dt", recipes.parsedatetime) + assert list(dates_db.table("example").rows) == [ {"id": 1, "dt": "2019-10-05T12:04:00"}, {"id": 2, "dt": "2019-10-06T00:05:06"}, {"id": 3, "dt": ""}, @@ -50,16 +50,16 @@ def test_parsedatetime(dates_db): ), ) def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected): - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "dt": "03/04/05"}, ], pk="id", ) - fresh_db["example"].convert( + fresh_db.table("example").convert( "dt", lambda value: getattr(recipes, recipe)(value, **kwargs) ) - assert list(fresh_db["example"].rows) == [ + assert list(fresh_db.table("example").rows) == [ {"id": 1, "dt": expected}, ] @@ -68,7 +68,7 @@ def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected): @pytest.mark.parametrize("fn", ("parsedate", "parsedatetime")) def test_dateparse_errors_raises(fresh_db, fn): """Test that invalid dates raise errors when errors=None""" - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "dt": "invalid"}, ], @@ -76,30 +76,32 @@ def test_dateparse_errors_raises(fresh_db, fn): ) # Exception in SQLite callback surfaces as OperationalError with pytest.raises(sqlite3.OperationalError): - fresh_db["example"].convert("dt", lambda value: getattr(recipes, fn)(value)) + fresh_db.table("example").convert( + "dt", lambda value: getattr(recipes, fn)(value) + ) @pytest.mark.parametrize("fn", ("parsedate", "parsedatetime")) @pytest.mark.parametrize("errors", (recipes.SET_NULL, recipes.IGNORE)) def test_dateparse_errors_handled(fresh_db, fn, errors): """Test error handling modes for invalid dates""" - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "dt": "invalid"}, ], pk="id", ) - fresh_db["example"].convert( + fresh_db.table("example").convert( "dt", lambda value: getattr(recipes, fn)(value, errors=errors) ) - rows = list(fresh_db["example"].rows) + rows = list(fresh_db.table("example").rows) expected = [{"id": 1, "dt": None if errors is recipes.SET_NULL else "invalid"}] assert rows == expected @pytest.mark.parametrize("delimiter", [None, ";", "-"]) def test_jsonsplit(fresh_db, delimiter): - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])}, {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])}, @@ -114,8 +116,8 @@ def test_jsonsplit(fresh_db, delimiter): else: fn = recipes.jsonsplit - fresh_db["example"].convert("tags", fn) - assert list(fresh_db["example"].rows) == [ + fresh_db.table("example").convert("tags", fn) + assert list(fresh_db.table("example").rows) == [ {"id": 1, "tags": '["foo", "bar"]'}, {"id": 2, "tags": '["bar", "baz"]'}, ] @@ -130,7 +132,7 @@ def test_jsonsplit(fresh_db, delimiter): ), ) def test_jsonsplit_type(fresh_db, type, expected): - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "records": "1,2,3"}, ], @@ -144,5 +146,5 @@ def test_jsonsplit_type(fresh_db, type, expected): else: fn = recipes.jsonsplit - fresh_db["example"].convert("records", fn) - assert json.loads(fresh_db["example"].get(1)["records"]) == expected + fresh_db.table("example").convert("records", fn) + assert json.loads(fresh_db.table("example").get(1)["records"]) == expected diff --git a/tests/test_recreate.py b/tests/test_recreate.py index 09e237e..d8b846e 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -33,8 +33,8 @@ def test_recreate(tmp_path, use_path, create_file_first): filepath = pathlib.Path(filepath) if create_file_first: db = Database(filepath) - db["t1"].insert({"foo": "bar"}) + db.table("t1").insert({"foo": "bar"}) assert ["t1"] == db.table_names() db.close() - Database(filepath, recreate=True)["t2"].insert({"foo": "bar"}) + Database(filepath, recreate=True).table("t2").insert({"foo": "bar"}) assert ["t2"] == Database(filepath).table_names() diff --git a/tests/test_rows.py b/tests/test_rows.py index dccb6ad..476569e 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -3,7 +3,7 @@ import pytest def test_rows(existing_db): assert [{"text": "one"}, {"text": "two"}, {"text": "three"}] == list( - existing_db["foo"].rows + existing_db.table("foo").rows ) @@ -18,7 +18,7 @@ def test_rows(existing_db): ], ) def test_rows_where(where, where_args, expected_ids, fresh_db): - table = fresh_db["dogs"] + table = fresh_db.table("dogs") table.insert_all( [ {"id": 1, "name": "Cleo", "age": 4, "is_good": True}, @@ -41,7 +41,7 @@ def test_rows_where(where, where_args, expected_ids, fresh_db): ], ) def test_rows_where_order_by(where, order_by, expected_ids, fresh_db): - table = fresh_db["dogs"] + table = fresh_db.table("dogs") table.insert_all( [ {"id": 1, "name": "Cleo", "age": 4}, @@ -65,7 +65,7 @@ def test_rows_where_order_by(where, order_by, expected_ids, fresh_db): ], ) def test_rows_where_offset_limit(fresh_db, offset, limit, expected): - table = fresh_db["rows"] + table = fresh_db.table("rows") table.insert_all([{"id": id} for id in range(1, 101)], pk="id") assert table.count == 100 assert expected == [ @@ -74,13 +74,13 @@ def test_rows_where_offset_limit(fresh_db, offset, limit, expected): def test_pks_and_rows_where_offset_without_limit(fresh_db): - table = fresh_db["rows"] + table = fresh_db.table("rows") table.insert_all([{"id": id} for id in range(1, 6)], pk="id") assert [pk for pk, _ in table.pks_and_rows_where(offset=3, order_by="id")] == [4, 5] def test_pks_and_rows_where_rowid(fresh_db): - table = fresh_db["rowid_table"] + table = fresh_db.table("rowid_table") table.insert_all({"number": i + 10} for i in range(3)) pks_and_rows = list(table.pks_and_rows_where()) assert pks_and_rows == [ @@ -91,7 +91,7 @@ def test_pks_and_rows_where_rowid(fresh_db): def test_pks_and_rows_where_simple_pk(fresh_db): - table = fresh_db["simple_pk_table"] + table = fresh_db.table("simple_pk_table") table.insert_all(({"id": i + 10} for i in range(3)), pk="id") pks_and_rows = list(table.pks_and_rows_where()) assert pks_and_rows == [ @@ -102,7 +102,7 @@ def test_pks_and_rows_where_simple_pk(fresh_db): def test_pks_and_rows_where_compound_pk(fresh_db): - table = fresh_db["compound_pk_table"] + table = fresh_db.table("compound_pk_table") table.insert_all( ({"type": "number", "number": i, "plusone": i + 1} for i in range(3)), pk=("type", "number"), @@ -117,8 +117,8 @@ def test_pks_and_rows_where_compound_pk(fresh_db): def test_rows_where_duplicate_select_columns_are_deduped(fresh_db): # https://github.com/simonw/sqlite-utils/issues/624 - fresh_db["t"].insert({"id": 1, "name": "Cleo"}) - rows = list(fresh_db["t"].rows_where(select="id, id, name")) + fresh_db.table("t").insert({"id": 1, "name": "Cleo"}) + rows = list(fresh_db.table("t").rows_where(select="id, id, name")) assert rows == [{"id": 1, "id_2": 1, "name": "Cleo"}] @@ -130,10 +130,10 @@ def test_pks_and_rows_where_view(fresh_db): # an AttributeError from View lacking Table-only properties from sqlite_utils.utils import sqlite3 - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.create_view("dog_names", "select name from dogs") try: - result = list(fresh_db["dog_names"].pks_and_rows_where()) + result = list(fresh_db.view("dog_names").pks_and_rows_where()) except sqlite3.OperationalError: pass # SQLite 3.36+: no such column: rowid else: @@ -144,6 +144,6 @@ def test_pks_and_rows_where_view(fresh_db): def test_pks_and_rows_where_compound_pk_declaration_order(fresh_db): # Compound pks are returned in PRIMARY KEY declaration order fresh_db.execute("create table t (b text, a text, primary key (a, b))") - fresh_db["t"].insert({"a": "A", "b": "B"}) - pks_and_rows = list(fresh_db["t"].pks_and_rows_where()) + fresh_db.table("t").insert({"a": "A", "b": "B"}) + pks_and_rows = list(fresh_db.table("t").pks_and_rows_where()) assert pks_and_rows == [(("A", "B"), {"b": "B", "a": "A"})] diff --git a/tests/test_sniff.py b/tests/test_sniff.py index 7149978..029a7fc 100644 --- a/tests/test_sniff.py +++ b/tests/test_sniff.py @@ -19,7 +19,7 @@ def test_sniff(tmpdir, filepath): ) assert result.exit_code == 0, result.stdout db = Database(db_path) - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"id": "1", "species": "dog", "name": "Cleo", "age": "5"}, {"id": "2", "species": "dog", "name": "Pancakes", "age": "4"}, {"id": "3", "species": "cat", "name": "Mozie", "age": "8"}, diff --git a/tests/test_transform.py b/tests/test_transform.py index 980ee9d..28fa4d7 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -128,7 +128,7 @@ def test_transform_sql_table_with_primary_key( def tracer(sql, params): return captured.append((sql, params)) - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") @@ -209,7 +209,7 @@ def test_transform_sql_table_with_no_primary_key( def tracer(sql, params): return captured.append((sql, params)) - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) @@ -229,7 +229,7 @@ def test_transform_sql_table_with_no_primary_key( def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) assert ( dogs.schema @@ -244,7 +244,7 @@ def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db): def test_transform_rename_pk(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") dogs.transform(rename={"id": "pk"}) assert ( @@ -265,7 +265,7 @@ def test_transform_preserves_keyword_literal_defaults(fresh_db): " note TEXT DEFAULT NULL" ")" ) - table = fresh_db["t"] + table = fresh_db.table("t") table.insert({"id": 1}) before = fresh_db.execute("SELECT is_active, flag, note FROM t").fetchone() assert before == (1, 0, None) @@ -288,7 +288,7 @@ def test_transform_preserves_keyword_literal_defaults(fresh_db): def test_transform_not_null(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") dogs.transform(not_null={"name"}) assert ( @@ -298,7 +298,7 @@ def test_transform_not_null(fresh_db): def test_transform_remove_a_not_null(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, not_null={"age"}, pk="id") dogs.transform(not_null={"name": True, "age": False}) assert ( @@ -309,7 +309,7 @@ def test_transform_remove_a_not_null(fresh_db): @pytest.mark.parametrize("not_null", [{"age"}, {"age": True}]) def test_transform_add_not_null_with_rename(fresh_db, not_null): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") dogs.transform(not_null=not_null, rename={"age": "dog_age"}) assert ( @@ -319,7 +319,7 @@ def test_transform_add_not_null_with_rename(fresh_db, not_null): def test_transform_defaults(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id") dogs.transform(defaults={"age": 1}) assert ( @@ -329,7 +329,7 @@ def test_transform_defaults(fresh_db): def test_transform_defaults_and_rename_column(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id") dogs.transform(rename={"age": "dog_age"}, defaults={"age": 1}) assert ( @@ -339,7 +339,7 @@ def test_transform_defaults_and_rename_column(fresh_db): def test_remove_defaults(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5}, defaults={"age": 1}, pk="id") dogs.transform(defaults={"age": None}) assert ( @@ -350,8 +350,8 @@ def test_remove_defaults(fresh_db): @pytest.fixture def authors_db(fresh_db): - books = fresh_db["books"] - authors = fresh_db["authors"] + books = fresh_db.table("books") + authors = fresh_db.table("authors") authors.insert({"id": 5, "name": "Jane McGonical"}, pk="id") books.insert( {"id": 2, "title": "Reality is Broken", "author_id": 5}, @@ -362,13 +362,13 @@ def authors_db(fresh_db): def test_transform_foreign_keys_persist(authors_db): - assert authors_db["books"].foreign_keys == [ + assert authors_db.table("books").foreign_keys == [ ForeignKey( table="books", column="author_id", other_table="authors", other_column="id" ) ] - authors_db["books"].transform(rename={"title": "book_title"}) - assert authors_db["books"].foreign_keys == [ + authors_db.table("books").transform(rename={"title": "book_title"}) + assert authors_db.table("books").foreign_keys == [ ForeignKey( table="books", column="author_id", other_table="authors", other_column="id" ) @@ -381,8 +381,8 @@ def test_transform_foreign_keys_survive_renamed_column( ): if use_pragma_foreign_keys: authors_db.conn.execute("PRAGMA foreign_keys=ON") - authors_db["books"].transform(rename={"author_id": "author_id_2"}) - assert authors_db["books"].foreign_keys == [ + authors_db.table("books").transform(rename={"author_id": "author_id_2"}) + assert authors_db.table("books").foreign_keys == [ ForeignKey( table="books", column="author_id_2", @@ -393,9 +393,9 @@ def test_transform_foreign_keys_survive_renamed_column( def _add_country_city_continent(db): - db["country"].insert({"id": 1, "name": "France"}, pk="id") - db["continent"].insert({"id": 2, "name": "Europe"}, pk="id") - db["city"].insert({"id": 24, "name": "Paris"}, pk="id") + db.table("country").insert({"id": 1, "name": "France"}, pk="id") + db.table("continent").insert({"id": 2, "name": "Europe"}, pk="id") + db.table("city").insert({"id": 24, "name": "Paris"}, pk="id") _CAVEAU = { @@ -413,11 +413,11 @@ def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys): fresh_db.conn.execute("PRAGMA foreign_keys=ON") # Create table with three foreign keys so we can drop two of them _add_country_city_continent(fresh_db) - fresh_db["places"].insert( + fresh_db.table("places").insert( _CAVEAU, foreign_keys=("country", "continent", "city"), ) - assert fresh_db["places"].foreign_keys == [ + assert fresh_db.table("places").foreign_keys == [ ForeignKey( table="places", column="city", other_table="city", other_column="id" ), @@ -432,9 +432,9 @@ def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys): ), ] # Drop two of those foreign keys - fresh_db["places"].transform(drop_foreign_keys=("country", "continent")) + fresh_db.table("places").transform(drop_foreign_keys=("country", "continent")) # Should be only one foreign key now - assert fresh_db["places"].foreign_keys == [ + assert fresh_db.table("places").foreign_keys == [ ForeignKey(table="places", column="city", other_table="city", other_column="id") ] if use_pragma_foreign_keys: @@ -443,17 +443,17 @@ def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys): def test_transform_verify_foreign_keys(fresh_db): fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db["authors"].insert({"id": 3, "name": "Tina"}, pk="id") - fresh_db["books"].insert( + fresh_db.table("authors").insert({"id": 3, "name": "Tina"}, pk="id") + fresh_db.table("books").insert( {"id": 1, "title": "Book", "author_id": 3}, pk="id", foreign_keys={"author_id"} ) # Renaming the id column on authors should break everything with pytest.raises(OperationalError) as e: - fresh_db["authors"].transform(rename={"id": "id2"}) + fresh_db.table("authors").transform(rename={"id": "id2"}) assert e.value.args[0] == 'foreign key mismatch - "books" referencing "authors"' # This should have rolled us back assert ( - fresh_db["authors"].schema + fresh_db.table("authors").schema == 'CREATE TABLE "authors" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n)' ) assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] @@ -476,20 +476,22 @@ def test_transform_on_delete_cascade_does_not_delete_records( author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE ); """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db.table("books").insert( + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ) # Transform the table on the other end of the cascading foreign key - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["authors"].rows) == [ + fresh_db.table("authors").transform(rename={"name": "author_name"}) + assert list(fresh_db.table("authors").rows) == [ {"id": 1, "author_name": "Ursula K. Le Guin"} ] - assert list(fresh_db["books"].rows) == [ + assert list(fresh_db.table("books").rows) == [ {"id": 1, "title": "The Dispossessed", "author_id": 1} ] # Transforming the table with the cascading foreign key should not # delete its records either - fresh_db["books"].transform(rename={"title": "book_title"}) - assert list(fresh_db["books"].rows) == [ + fresh_db.table("books").transform(rename={"title": "book_title"}) + assert list(fresh_db.table("books").rows) == [ {"id": 1, "book_title": "The Dispossessed", "author_id": 1} ] if use_pragma_foreign_keys: @@ -511,17 +513,19 @@ def test_transform_in_transaction_refuses_destructive_on_delete(fresh_db, on_del author_id INTEGER REFERENCES authors(id) ON DELETE {on_delete} ); """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - previous_schema = fresh_db["authors"].schema + fresh_db.table("authors").insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db.table("books").insert( + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ) + previous_schema = fresh_db.table("authors").schema with fresh_db.atomic(), pytest.raises(TransactionError) as excinfo: - fresh_db["authors"].transform(rename={"name": "author_name"}) + fresh_db.table("authors").transform(rename={"name": "author_name"}) message = str(excinfo.value) assert "books" in message assert f"ON DELETE {on_delete.upper()}" in message # Nothing should have changed - assert fresh_db["authors"].schema == previous_schema - assert list(fresh_db["books"].rows) == [ + assert fresh_db.table("authors").schema == previous_schema + assert list(fresh_db.table("books").rows) == [ {"id": 1, "title": "The Dispossessed", "author_id": 1} ] assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] @@ -538,16 +542,16 @@ def test_transform_in_transaction_refuses_self_referential_cascade(fresh_db): parent_id INTEGER REFERENCES categories(id) ON DELETE CASCADE ); """) - fresh_db["categories"].insert_all( + fresh_db.table("categories").insert_all( [ {"id": 1, "name": "Fiction", "parent_id": None}, {"id": 2, "name": "Science Fiction", "parent_id": 1}, ] ) with fresh_db.atomic(), pytest.raises(TransactionError) as excinfo: - fresh_db["categories"].transform(rename={"name": "title"}) + fresh_db.table("categories").transform(rename={"name": "title"}) assert "categories" in str(excinfo.value) - assert fresh_db["categories"].count == 2 + assert fresh_db.table("categories").count == 2 def test_transform_in_transaction_allowed_with_no_action_foreign_key(fresh_db): @@ -562,14 +566,16 @@ def test_transform_in_transaction_allowed_with_no_action_foreign_key(fresh_db): author_id INTEGER REFERENCES authors(id) ); """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db.table("books").insert( + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ) with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["authors"].rows) == [ + fresh_db.table("authors").transform(rename={"name": "author_name"}) + assert list(fresh_db.table("authors").rows) == [ {"id": 1, "author_name": "Ursula K. Le Guin"} ] - assert list(fresh_db["books"].rows) == [ + assert list(fresh_db.table("books").rows) == [ {"id": 1, "title": "The Dispossessed", "author_id": 1} ] assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] @@ -587,11 +593,13 @@ def test_transform_in_transaction_allowed_for_child_table(fresh_db): author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE ); """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db.table("books").insert( + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ) with fresh_db.atomic(): - fresh_db["books"].transform(rename={"title": "book_title"}) - assert list(fresh_db["books"].rows) == [ + fresh_db.table("books").transform(rename={"title": "book_title"}) + assert list(fresh_db.table("books").rows) == [ {"id": 1, "book_title": "The Dispossessed", "author_id": 1} ] @@ -607,24 +615,28 @@ def test_transform_in_transaction_allowed_with_foreign_keys_off(fresh_db): author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE ); """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db.table("books").insert( + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ) with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["books"].rows) == [ + fresh_db.table("authors").transform(rename={"name": "author_name"}) + assert list(fresh_db.table("books").rows) == [ {"id": 1, "title": "The Dispossessed", "author_id": 1} ] def test_transform_add_foreign_keys_from_scratch(fresh_db): _add_country_city_continent(fresh_db) - fresh_db["places"].insert(_CAVEAU) + fresh_db.table("places").insert(_CAVEAU) # Should have no foreign keys - assert fresh_db["places"].foreign_keys == [] + assert fresh_db.table("places").foreign_keys == [] # Now add them using .transform() - fresh_db["places"].transform(add_foreign_keys=("country", "continent", "city")) + fresh_db.table("places").transform( + add_foreign_keys=("country", "continent", "city") + ) # Should now have all three: - assert fresh_db["places"].foreign_keys == [ + assert fresh_db.table("places").foreign_keys == [ ForeignKey( table="places", column="city", other_table="city", other_column="id" ), @@ -638,7 +650,7 @@ def test_transform_add_foreign_keys_from_scratch(fresh_db): table="places", column="country", other_table="country", other_column="id" ), ] - assert fresh_db["places"].schema == ( + assert fresh_db.table("places").schema == ( 'CREATE TABLE "places" (\n' ' "id" INTEGER,\n' ' "name" TEXT,\n' @@ -662,18 +674,18 @@ def test_transform_add_foreign_keys_from_scratch(fresh_db): ) def test_transform_add_foreign_keys_from_partial(fresh_db, add_foreign_keys): _add_country_city_continent(fresh_db) - fresh_db["places"].insert( + fresh_db.table("places").insert( _CAVEAU, foreign_keys=("city",), ) # Should have one foreign keys - assert fresh_db["places"].foreign_keys == [ + assert fresh_db.table("places").foreign_keys == [ ForeignKey(table="places", column="city", other_table="city", other_column="id") ] # Now add three more using .transform() - fresh_db["places"].transform(add_foreign_keys=add_foreign_keys) + fresh_db.table("places").transform(add_foreign_keys=add_foreign_keys) # Should now have all three: - assert fresh_db["places"].foreign_keys == [ + assert fresh_db.table("places").foreign_keys == [ ForeignKey( table="places", column="city", other_table="city", other_column="id" ), @@ -702,14 +714,14 @@ def test_transform_add_foreign_keys_from_partial(fresh_db, add_foreign_keys): ) def test_transform_replace_foreign_keys(fresh_db, foreign_keys): _add_country_city_continent(fresh_db) - fresh_db["places"].insert( + fresh_db.table("places").insert( _CAVEAU, foreign_keys=("city",), ) - assert len(fresh_db["places"].foreign_keys) == 1 + assert len(fresh_db.table("places").foreign_keys) == 1 # Replace with two different ones - fresh_db["places"].transform(foreign_keys=foreign_keys) - assert fresh_db["places"].schema == ( + fresh_db.table("places").transform(foreign_keys=foreign_keys) + assert fresh_db.table("places").schema == ( 'CREATE TABLE "places" (\n' ' "id" INTEGER,\n' ' "name" TEXT,\n' @@ -729,7 +741,7 @@ def test_transform_preserves_rowids(fresh_db, table_type): pk = ("id", "name") elif table_type == "rowid": pk = None - fresh_db["places"].insert_all( + fresh_db.table("places").insert_all( [ {"id": "1", "name": "Paris", "country": "France"}, {"id": "2", "name": "London", "country": "UK"}, @@ -738,13 +750,13 @@ def test_transform_preserves_rowids(fresh_db, table_type): pk=pk, ) # Now delete and insert a row to mix up the `rowid` sequence - fresh_db["places"].delete_where("id = ?", ["2"]) - fresh_db["places"].insert({"id": "4", "name": "London", "country": "UK"}) + fresh_db.table("places").delete_where("id = ?", ["2"]) + fresh_db.table("places").insert({"id": "4", "name": "London", "country": "UK"}) previous_rows = [ tuple(row) for row in fresh_db.execute("select rowid, id, name from places") ] # Transform it - fresh_db["places"].transform(column_order=("country", "name")) + fresh_db.table("places").transform(column_order=("country", "name")) # Should be the same next_rows = [ tuple(row) for row in fresh_db.execute("select rowid, id, name from places") @@ -774,7 +786,7 @@ def test_transform_strict(fresh_db, initial_strict, transform_strict, expected_s def test_transform_to_strict_with_invalid_data(fresh_db): if not fresh_db.supports_strict: pytest.skip("SQLite version does not support strict tables") - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.create({"id": int}) dogs.insert({"id": "not-an-integer"}) @@ -801,7 +813,7 @@ def test_transform_strict_updates_default(fresh_db): @pytest.mark.parametrize("method_name", ("transform", "transform_sql")) def test_transform_to_strict_not_supported(fresh_db, method_name): - table = fresh_db["items"] + table = fresh_db.table("items") table.create({"id": int}) fresh_db._supports_strict = False @@ -823,7 +835,7 @@ def test_transform_to_strict_not_supported(fresh_db, method_name): def test_transform_indexes(fresh_db, indexes, transform_params): # https://github.com/simonw/sqlite-utils/issues/633 # New table should have same indexes as old table after transformation - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5, "breed": "Labrador"}, pk="id") for index in indexes: @@ -849,13 +861,13 @@ def test_transform_indexes(fresh_db, indexes, transform_params): if "keep_table" in transform_params: assert all( index.origin == "pk" - for index in fresh_db[transform_params["keep_table"]].indexes + for index in fresh_db.table(transform_params["keep_table"]).indexes ) def test_transform_retains_indexes_with_foreign_keys(fresh_db): - dogs = fresh_db["dogs"] - owners = fresh_db["owners"] + dogs = fresh_db.table("dogs") + owners = fresh_db.table("owners") dogs.insert({"id": 1, "name": "Cleo", "owner_id": 1}, pk="id") owners.insert({"id": 1, "name": "Alice"}, pk="id") @@ -890,7 +902,7 @@ def test_transform_retains_indexes_with_foreign_keys(fresh_db): ) def test_transform_with_indexes_errors(fresh_db, transform_params): # Should error with a compound (name, age) index if age is renamed or dropped - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id") dogs.create_index(["name", "age"]) @@ -906,7 +918,7 @@ def test_transform_with_indexes_errors(fresh_db, transform_params): def test_transform_with_unique_constraint_implicit_index(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") # Create a table with a UNIQUE constraint on 'name', which creates an implicit index fresh_db.execute(""" CREATE TABLE dogs ( @@ -933,7 +945,7 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db): def test_transform_preserves_view(fresh_db): # https://github.com/simonw/sqlite-utils/issues/831 - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view dogs_view as select id, name from dogs") view_sql_before = fresh_db.execute( @@ -958,8 +970,8 @@ def test_transform_preserves_view(fresh_db): def test_transform_variants_preserve_view(fresh_db, transform_params): # Covers retyping, changing primary key and foreign key modifications, # with a view whose columns are untouched by the transform - fresh_db["other"].insert({"id": 1}, pk="id") - dogs = fresh_db["dogs"] + fresh_db.table("other").insert({"id": 1}, pk="id") + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "other_id": 1}, pk="id") if "drop_foreign_keys" in transform_params: dogs.transform(add_foreign_keys=[("other_id", "other", "id")]) @@ -972,13 +984,13 @@ def test_transform_variants_preserve_view(fresh_db, transform_params): "select sql from sqlite_master where name = 'dogs_view'" ).fetchone()[0] assert view_sql_before == view_sql_after - assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}] + assert list(fresh_db.view("dogs_view").rows) == [{"id": 1, "name": "Cleo"}] def test_transform_view_referencing_renamed_column(fresh_db): # The view survives but querying it raises "no such column" - inherent # to SQLite views, whose SQL is stored as text - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view dogs_view as select id, name from dogs") dogs.transform(rename={"name": "title"}) @@ -987,7 +999,7 @@ def test_transform_view_referencing_renamed_column(fresh_db): def test_transform_view_on_view(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view v1 as select id, name from dogs") fresh_db.execute("create view v2 as select name from v1") @@ -999,13 +1011,13 @@ def test_transform_view_on_view(fresh_db): "select sql from sqlite_master where type = 'view' order by name" ).fetchall() assert sqls_before == sqls_after - assert list(fresh_db["v2"].rows) == [{"name": "Cleo"}] + assert list(fresh_db.view("v2").rows) == [{"name": "Cleo"}] def test_transform_keep_table_does_not_repoint_view(fresh_db): # Without legacy_alter_table the ALTER TABLE dogs RENAME TO dogs_backup # step would rewrite the view to select from "dogs_backup" - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view dogs_view as select id, name from dogs") dogs.transform(types={"name": str}, keep_table="dogs_backup") @@ -1015,7 +1027,7 @@ def test_transform_keep_table_does_not_repoint_view(fresh_db): assert "dogs_backup" not in view_sql # View reads from the live table, not the frozen backup dogs.insert({"id": 2, "name": "Pancakes"}) - assert list(fresh_db["dogs_view"].rows) == [ + assert list(fresh_db.view("dogs_view").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 2, "name": "Pancakes"}, ] @@ -1024,7 +1036,7 @@ def test_transform_keep_table_does_not_repoint_view(fresh_db): def test_transform_sql_standalone_statements_work_with_view(fresh_db): # The documented "run these statements yourself" workflow should be # standalone-correct, so the pragmas must come from transform_sql() - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view dogs_view as select id, name from dogs") sqls = dogs.transform_sql(types={"name": str}, tmp_suffix="suffix") @@ -1033,12 +1045,12 @@ def test_transform_sql_standalone_statements_work_with_view(fresh_db): assert sqls[-1] == "PRAGMA legacy_alter_table=OFF;" for sql in sqls: fresh_db.execute(sql) - assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}] + assert list(fresh_db.view("dogs_view").rows) == [{"id": 1, "name": "Cleo"}] def test_transform_with_view_in_open_transaction(fresh_db): fresh_db.conn.execute("PRAGMA foreign_keys=ON") - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view dogs_view as select id, name from dogs") with fresh_db.conn: @@ -1054,7 +1066,7 @@ def test_transform_with_view_in_open_transaction(fresh_db): def test_transform_restores_legacy_alter_table_setting(fresh_db): if sqlite3.sqlite_version_info < (3, 25, 0): pytest.skip("legacy_alter_table pragma requires SQLite 3.25 or higher") - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") # Default is OFF, reset to OFF afterwards dogs.transform(types={"name": str}) @@ -1075,7 +1087,7 @@ def test_transform_preserves_check_constraints(fresh_db): CONSTRAINT nonzero_id CHECK(id != 0) ) """) - scores = fresh_db["scores"] + scores = fresh_db.table("scores") scores.insert({"id": 1, "score": 50}) scores.transform() assert scores.checks == [ @@ -1095,7 +1107,7 @@ def test_transform_preserves_check_ending_in_line_comment(fresh_db): ) ) """) - inventory = fresh_db["inventory"] + inventory = fresh_db.table("inventory") inventory.transform(types={"quantity": float}) assert inventory.checks == [Check("quantity >= 0 -- Quantity cannot be negative")] with pytest.raises(sqlite3.IntegrityError, match="CHECK constraint failed"): @@ -1113,7 +1125,7 @@ def test_transform_preserves_comments_owned_by_columns(fresh_db): age INTEGER -- May be NULL ) """) - people = fresh_db["people"] + people = fresh_db.table("people") people.insert({"id": 1, "name": "Cleo", "age": 5}) people.transform( rename={"name": "display_name"}, @@ -1143,8 +1155,8 @@ def test_transform_drops_comments_owned_by_dropped_column(fresh_db): obsolete TEXT /* Drop this too */ ) """) - fresh_db["t"].transform(drop={"obsolete"}) - schema = fresh_db["t"].schema + fresh_db.table("t").transform(drop={"obsolete"}) + schema = fresh_db.table("t").schema assert "Keep this explanation" in schema assert "Drop this explanation" not in schema assert "Drop this too" not in schema @@ -1159,7 +1171,7 @@ def test_transform_renames_columns_inside_check_constraints(fresh_db): CONSTRAINT within_maximum CHECK(quantity <= maximum) ) """) - inventory = fresh_db["inventory"] + inventory = fresh_db.table("inventory") inventory.insert({"quantity": 2, "maximum": 3}) inventory.transform(rename={"quantity": "amount"}) assert inventory.checks == [ @@ -1182,7 +1194,7 @@ def test_transform_check_rewrite_preserves_functions_and_quotes(fresh_db): CHECK(length("old name") > 0 AND length != '') ) """) - items = fresh_db["items"] + items = fresh_db.table("items") items.insert({"length": "label", "old name": "hello"}) items.transform(rename={"length": "description", "old name": "new name"}) assert items.checks == [Check("length(\"new name\") > 0 AND description != ''")] @@ -1190,9 +1202,9 @@ def test_transform_check_rewrite_preserves_functions_and_quotes(fresh_db): def test_transform_check_rewrite_quotes_keyword_column(fresh_db): fresh_db.execute("CREATE TABLE t(old_name TEXT CHECK(old_name != ''))") - fresh_db["t"].insert({"old_name": "value"}) - fresh_db["t"].transform(rename={"old_name": "select"}) - assert fresh_db["t"].checks == [Check("\"select\" != ''", column="select")] + fresh_db.table("t").insert({"old_name": "value"}) + fresh_db.table("t").transform(rename={"old_name": "select"}) + assert fresh_db.table("t").checks == [Check("\"select\" != ''", column="select")] def test_transform_check_rewrite_does_not_rename_collations_or_cast_types(fresh_db): @@ -1209,9 +1221,9 @@ def test_transform_check_rewrite_does_not_rename_collations_or_cast_types(fresh_ ) ) """) - fresh_db["t"].insert({"nocase": "n", "kind": "k", "other": "o"}) - fresh_db["t"].transform(rename={"nocase": "label", "kind": "category"}) - check = fresh_db["t"].checks[0].check + fresh_db.table("t").insert({"nocase": "n", "kind": "k", "other": "o"}) + fresh_db.table("t").transform(rename={"nocase": "label", "kind": "category"}) + check = fresh_db.table("t").checks[0].check assert "COLLATE nocase" in check assert "AS kind" in check assert "AND label != ''" in check @@ -1226,9 +1238,9 @@ def test_transform_drops_check_owned_by_dropped_column(fresh_db): CHECK(id > 0) ) """) - fresh_db["t"].insert({"id": 1, "obsolete": 2}) - fresh_db["t"].transform(drop={"obsolete"}) - assert fresh_db["t"].checks == [Check("id > 0")] + fresh_db.table("t").insert({"id": 1, "obsolete": 2}) + fresh_db.table("t").transform(drop={"obsolete"}) + assert fresh_db.table("t").checks == [Check("id > 0")] def test_transform_refuses_to_drop_column_used_by_remaining_check(fresh_db): @@ -1239,7 +1251,7 @@ def test_transform_refuses_to_drop_column_used_by_remaining_check(fresh_db): CHECK(minimum <= maximum) ) """) - ranges = fresh_db["ranges"] + ranges = fresh_db.table("ranges") ranges.insert({"minimum": 1, "maximum": 2}) schema_before = ranges.schema with pytest.raises( diff --git a/tests/test_update.py b/tests/test_update.py index e6ae7d8..44cc098 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -7,14 +7,14 @@ from sqlite_utils.db import NotFoundError def test_update_rowid_table(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") rowid = table.insert({"foo": "bar"}).last_pk table.update(rowid, {"foo": "baz"}) assert [{"foo": "baz"}] == list(table.rows) def test_update_pk_table(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") pk = table.insert({"foo": "bar", "id": 5}, pk="id").last_pk assert 5 == pk table.update(pk, {"foo": "baz"}) @@ -22,7 +22,7 @@ def test_update_pk_table(fresh_db): def test_update_compound_pk_table(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") pk = table.insert({"id1": 5, "id2": 3, "v": 1}, pk=("id1", "id2")).last_pk assert (5, 3) == pk table.update(pk, {"v": 2}) @@ -42,14 +42,14 @@ def test_update_compound_pk_table(fresh_db): ), ) def test_update_invalid_pk(fresh_db, pk, update_pk): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"id1": 5, "id2": 3, "v": 1}, pk=pk) with pytest.raises(NotFoundError): table.update(update_pk, {"v": 2}) def test_update_alter(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") rowid = table.insert({"foo": "bar"}).last_pk table.update(rowid, {"new_col": 1.2}, alter=True) assert [{"foo": "bar", "new_col": 1.2}] == list(table.rows) @@ -72,7 +72,7 @@ def test_update_alter(fresh_db): def test_update_alter_with_special_column_characters(fresh_db): # With double-quote escaping, columns with special characters are now valid - table = fresh_db["table"] + table = fresh_db.table("table") rowid = table.insert({"foo": "bar"}).last_pk table.update(rowid, {"new_col[abc]": 1.2}, alter=True) assert list(table.rows) == [{"foo": "bar", "new_col[abc]": 1.2}] @@ -106,8 +106,8 @@ def test_update_with_no_values_sets_last_pk(fresh_db): ), ) def test_update_dictionaries_and_lists_as_json(fresh_db, data_structure): - fresh_db["test"].insert({"id": 1, "data": ""}, pk="id") - fresh_db["test"].update(1, {"data": data_structure}) + fresh_db.table("test").insert({"id": 1, "data": ""}, pk="id") + fresh_db.table("test").update(1, {"data": data_structure}) row = fresh_db.execute("select id, data from test").fetchone() assert row[0] == 1 assert data_structure == json.loads(row[1]) diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 0eaae9b..0f44cc7 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -7,7 +7,7 @@ from sqlite_utils.db import PrimaryKeyRequired @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_upsert(use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) - table = db["table"] + table = db.table("table") table.insert({"id": 1, "name": "Cleo"}, pk="id") table.upsert({"id": 1, "age": 5}, pk="id", alter=True) assert list(table.rows) == [{"id": 1, "name": "Cleo", "age": 5}] @@ -15,7 +15,7 @@ def test_upsert(use_old_upsert): def test_upsert_all(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Nixie"}], pk="id") table.upsert_all([{"id": 1, "age": 5}, {"id": 2, "age": 5}], pk="id", alter=True) assert list(table.rows) == [ @@ -26,7 +26,7 @@ def test_upsert_all(fresh_db): def test_upsert_all_single_column(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert_all([{"name": "Cleo"}], pk="name") assert list(table.rows) == [{"name": "Cleo"}] assert table.pks == ["name"] @@ -34,16 +34,16 @@ def test_upsert_all_single_column(fresh_db): def test_upsert_all_not_null(fresh_db): # https://github.com/simonw/sqlite-utils/issues/538 - fresh_db["comments"].upsert_all( + fresh_db.table("comments").upsert_all( [{"id": 1, "name": "Cleo"}], pk="id", not_null=["name"], ) - assert list(fresh_db["comments"].rows) == [{"id": 1, "name": "Cleo"}] + assert list(fresh_db.table("comments").rows) == [{"id": 1, "name": "Cleo"}] def test_upsert_error_if_no_pk(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") with pytest.raises(PrimaryKeyRequired): table.upsert_all([{"id": 1, "name": "Cleo"}]) with pytest.raises(PrimaryKeyRequired): @@ -53,7 +53,7 @@ def test_upsert_error_if_no_pk(fresh_db): @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_upsert_empty_record_errors(use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) - table = db["table"] + table = db.table("table") table.insert({"id": 1, "name": "Cleo"}, pk="id") with pytest.raises(PrimaryKeyRequired): table.upsert({}, pk="id") @@ -66,7 +66,7 @@ def test_upsert_empty_record_errors(use_old_upsert): @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_upsert_missing_pk_value_errors(use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) - table = db["table"] + table = db.table("table") table.insert({"id": 1, "name": "Cleo"}, pk="id") # Records that omit the pk column entirely with pytest.raises(PrimaryKeyRequired): @@ -78,7 +78,7 @@ def test_upsert_missing_pk_value_errors(use_old_upsert): def test_upsert_missing_compound_pk_value_errors(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"a": "x", "b": "y", "v": 1}, pk=("a", "b")) # Missing one component of the detected compound primary key with pytest.raises(PrimaryKeyRequired): @@ -105,7 +105,7 @@ def test_upsert_uses_compound_pk_from_existing_table(use_old_upsert): primary key (Source, Object, Category) ) """) - table = db["summary"] + table = db.table("summary") table.upsert( { "Source": "Client A", @@ -134,7 +134,7 @@ def test_upsert_uses_compound_pk_from_existing_table(use_old_upsert): def test_upsert_with_hash_id(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert({"foo": "bar"}, hash_id="pk") assert [{"pk": "a5e744d0164540d33b1d7ea616c28f2fa97e754a", "foo": "bar"}] == list( table.rows @@ -144,7 +144,7 @@ def test_upsert_with_hash_id(fresh_db): @pytest.mark.parametrize("hash_id", (None, "custom_id")) def test_upsert_with_hash_id_columns(fresh_db, hash_id): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert({"a": 1, "b": 2, "c": 3}, hash_id=hash_id, hash_id_columns=("a", "b")) assert list(table.rows) == [ { @@ -167,7 +167,7 @@ def test_upsert_with_hash_id_columns(fresh_db, hash_id): def test_upsert_compound_primary_key(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert_all( [ {"species": "dog", "id": 1, "name": "Cleo", "age": 4}, diff --git a/tests/test_wal.py b/tests/test_wal.py index 35318f8..0e8f332 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -18,7 +18,7 @@ def test_enable_disable_wal(db_path_tmpdir): assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()] db.enable_wal() assert "wal" == db.journal_mode - db["test"].insert({"foo": "bar"}) + db.table("test").insert({"foo": "bar"}) assert "test.db-wal" in [f.basename for f in tmpdir.listdir()] db.disable_wal() assert "delete" == db.journal_mode @@ -27,25 +27,25 @@ def test_enable_disable_wal(db_path_tmpdir): def test_enable_wal_inside_transaction_raises(db_path_tmpdir): db, _path, _tmpdir = db_path_tmpdir - db["test"].insert({"id": 1}, pk="id") + db.table("test").insert({"id": 1}, pk="id") with pytest.raises(TransactionError), db.atomic(): - db["test"].insert({"id": 2}, pk="id") + db.table("test").insert({"id": 2}, pk="id") db.enable_wal() # The atomic() block must have rolled back cleanly and the # journal mode must be unchanged assert db.journal_mode == "delete" - assert [r["id"] for r in db["test"].rows] == [1] + assert [r["id"] for r in db.table("test").rows] == [1] def test_disable_wal_inside_transaction_raises(db_path_tmpdir): db, _path, _tmpdir = db_path_tmpdir db.enable_wal() - db["test"].insert({"id": 1}, pk="id") + db.table("test").insert({"id": 1}, pk="id") with pytest.raises(TransactionError), db.atomic(): - db["test"].insert({"id": 2}, pk="id") + db.table("test").insert({"id": 2}, pk="id") db.disable_wal() assert db.journal_mode == "wal" - assert [r["id"] for r in db["test"].rows] == [1] + assert [r["id"] for r in db.table("test").rows] == [1] def test_ensure_autocommit_on(db_path_tmpdir): @@ -65,9 +65,9 @@ def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): db, _path, _tmpdir = db_path_tmpdir db.enable_wal() with db.atomic(): - db["test"].insert({"id": 1}, pk="id") + db.table("test").insert({"id": 1}, pk="id") db.enable_wal() - assert [r["id"] for r in db["test"].rows] == [1] + assert [r["id"] for r in db.table("test").rows] == [1] def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): @@ -75,7 +75,7 @@ def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): # effect, silently breaking the caller's rollback guarantee - so # entering autocommit mode with a transaction open is an error db, _path, _tmpdir = db_path_tmpdir - db["test"].insert({"id": 1}, pk="id") + db.table("test").insert({"id": 1}, pk="id") db.begin() db.execute("insert into test (id) values (2)") with pytest.raises(TransactionError), db.ensure_autocommit_on(): @@ -83,4 +83,4 @@ def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): # The transaction is still open and can still be rolled back assert db.conn.in_transaction db.rollback() - assert [r["id"] for r in db["test"].rows] == [1] + assert [r["id"] for r in db.table("test").rows] == [1] From ebb04a97de765ce5f0b6d1149c992062fa25629a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 13:41:33 -0700 Subject: [PATCH 19/33] Fixes for Pyright, closes #833 --- .github/workflows/test.yml | 3 + Justfile | 3 +- pyproject.toml | 1 + sqlite_utils/cli.py | 17 ++-- sqlite_utils/db.py | 135 ++++++++++++++++++-------------- sqlite_utils/utils.py | 11 ++- tests/test_cli.py | 2 +- tests/test_cli_bulk.py | 1 + tests/test_cli_insert.py | 1 + tests/test_constructor.py | 3 +- tests/test_create.py | 1 + tests/test_foreign_keys.py | 2 +- tests/test_fts.py | 2 +- tests/test_list_mode.py | 4 +- tests/test_register_function.py | 6 +- tests/test_upsert.py | 2 +- 16 files changed, 109 insertions(+), 85 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 923de2e..6c720a1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,6 +43,9 @@ jobs: run: pytest --sqlite-autocommit - name: run mypy run: mypy sqlite_utils tests + - name: run pyright regression checks + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' + run: pyright sqlite_utils tests - name: run flake8 run: flake8 - name: run ty diff --git a/Justfile b/Justfile index be41523..e93075f 100644 --- a/Justfile +++ b/Justfile @@ -8,11 +8,12 @@ @run *options: uv run -- {{options}} -# Run linters: black, flake8, mypy, ty, cog +# Run linters: black, flake8, mypy, pyright, ty, cog @lint: just run black . --check uv run flake8 uv run mypy sqlite_utils tests + uv run pyright sqlite_utils tests uv run ty check sqlite_utils uv run cog --check README.md docs/*.rst uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt diff --git a/pyproject.toml b/pyproject.toml index 6bc0a64..9b4d6f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dev = [ # flake8 "flake8", "flake8-pyproject", + "pyright>=1.1.411", "ty>=0.0.37", # For stable cog: "tabulate>=0.10.0", diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index d9c7728..c90c137 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1093,7 +1093,7 @@ def insert_upsert_implementation( column_type_overrides = {column: ctype.upper() for column, ctype in (types or [])} def _insert_docs(docs, tracker=None): - extra_kwargs = { + extra_kwargs: dict[str, Any] = { "ignore": ignore, "replace": replace, "truncate": truncate, @@ -3275,15 +3275,10 @@ def convert( raise click.ClickException(str(e)) if dry_run: # Pull first 20 values for first column and preview them - if multi: - - def preview(v): + def preview(v): + if multi: return json.dumps(fn(v), default=repr, ensure_ascii=False) if v else v - - else: - - def preview(v): - return fn(v) if v else v + return fn(v) if v else v db.conn.create_function("preview_transform", 1, preview) sql = """ @@ -3788,12 +3783,12 @@ def _rows_from_code(code): code = pathlib.Path(code).read_text() except FileNotFoundError: raise click.ClickException(f"File not found: {code}") - namespace = {} + namespace: dict[str, Any] = {} try: exec(code, namespace) # noqa: S102 except SyntaxError as ex: raise click.ClickException(f"Error in --code: {ex}") - rows = namespace.get("rows") + rows: Any = namespace.get("rows") if callable(rows): rows = rows() if isinstance(rows, dict): diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 82a95c1..66dc700 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -18,6 +18,7 @@ from dataclasses import dataclass, field from types import TracebackType from typing import ( Any, + TypeVar, Union, cast, ) @@ -256,8 +257,8 @@ class ForeignKey: column: str | None = field(compare=False) other_table: str other_column: str | None = field(compare=False) - columns: tuple[str, ...] = () - other_columns: tuple[str, ...] = () + columns: tuple[str, ...] | list[str] = () + other_columns: tuple[str, ...] | list[str] = () is_compound: bool = False on_delete: str = "NO ACTION" on_update: str = "NO ACTION" @@ -320,6 +321,8 @@ ForeignKeyIndicator = ( ForeignKeysType = Iterable[ForeignKeyIndicator] | list[ForeignKeyIndicator] +PrimaryKey = str | tuple[str, ...] | list[str] + class Default: pass @@ -327,6 +330,8 @@ class Default: DEFAULT = Default() +T = TypeVar("T") + Tracer = Callable[[str, Sequence[Any] | dict[str, Any] | None], None] @@ -1853,8 +1858,8 @@ class Database: fk_object = self._resolve_foreign_key_casing( fk_object, table_obj.columns_dict ) - columns = fk_object.columns - other_columns = fk_object.other_columns + columns = tuple(fk_object.columns) + other_columns = tuple(fk_object.other_columns) for column in columns: if column not in table_obj.columns_dict: raise AlterError(f"No such column: {column} in {table}") @@ -1914,9 +1919,10 @@ class Database: existing_indexes = {tuple(i.columns) for i in table.indexes} for fk in table.foreign_keys: # A compound foreign key gets a single composite index - if fk.columns not in existing_indexes: + fk_columns = tuple(fk.columns) + if fk_columns not in existing_indexes: table.create_index(fk.columns, find_unique_name=True) - existing_indexes.add(fk.columns) + existing_indexes.add(fk_columns) def vacuum(self) -> None: "Run a SQLite ``VACUUM`` against the database." @@ -2453,7 +2459,7 @@ class Table(Queryable): replace: bool = False, ignore: bool = False, transform: bool = False, - strict: bool | Default = DEFAULT, + strict: bool | Default | None = DEFAULT, ) -> "Table": """ Create a table with the specified columns. @@ -2524,7 +2530,7 @@ class Table(Queryable): replace=replace, ignore=ignore, transform=transform, - strict=strict, # type: ignore[arg-type] + strict=cast(bool, strict), ) return self @@ -2860,7 +2866,7 @@ class Table(Queryable): for name, type_ in current_column_pairs: type_ = types.get(name) or type_ if name in drop: - del [copy_from_to[name]] + del copy_from_to[name] continue new_name = rename.get(name) or name new_column_pairs.append((new_name, type_)) @@ -3343,7 +3349,10 @@ class Table(Queryable): :param on_update: ``ON UPDATE`` action for the foreign key. """ columns = (column,) if isinstance(column, str) else tuple(column) + if not columns: + raise ValueError("column must contain at least one column name") columns = tuple(resolve_casing(c, self.columns_dict) for c in columns) + assert columns # Ensure columns exist for col in columns: if col not in self.columns_dict: @@ -3354,7 +3363,7 @@ class Table(Queryable): raise ValueError( "other_table must be specified for a compound foreign key" ) - other_table = self.guess_foreign_table(columns[0]) + other_table = self.guess_foreign_table(next(iter(columns))) # If other_column is not specified, detect the primary key on other_table if other_column is None: if len(columns) > 1: @@ -3801,8 +3810,10 @@ class Table(Queryable): for row in cursor: yield dict(zip(columns, row)) - def value_or_default(self, key: str, value: Any) -> Any: - return self._defaults[key] if value is DEFAULT else value + def value_or_default(self, key: str, value: T | Default) -> T: + if value is DEFAULT: + return cast(T, self._defaults[key]) + return cast(T, value) def delete(self, pk_values: list | tuple | str | float) -> "Table": """ @@ -3987,7 +3998,7 @@ class Table(Queryable): def _convert_multi( self, column, fn, drop, show_progress, where=None, where_args=None - ): + ) -> "Table": # First we execute the function pk_to_values = {} new_column_types: dict[str, set[type]] = {} @@ -4033,6 +4044,7 @@ class Table(Queryable): bar.update(1) if drop: self.transform(drop=(column,)) + return self def build_insert_queries_and_params( self, @@ -4336,8 +4348,8 @@ class Table(Queryable): def insert( self, record: dict[str, Any], - pk=DEFAULT, - foreign_keys=DEFAULT, + pk: PrimaryKey | Default | None = DEFAULT, + foreign_keys: ForeignKeysType | Default | None = DEFAULT, column_order: list[str] | Default | None = DEFAULT, not_null: Iterable[str] | Default | None = DEFAULT, defaults: dict[str, Any] | Default | None = DEFAULT, @@ -4405,24 +4417,24 @@ class Table(Queryable): def insert_all( self, records: Iterable[dict[str, Any]] | Iterable[Sequence[Any]], - pk=DEFAULT, - foreign_keys=DEFAULT, - column_order=DEFAULT, - not_null=DEFAULT, - defaults=DEFAULT, - batch_size=DEFAULT, - hash_id=DEFAULT, - hash_id_columns=DEFAULT, - alter=DEFAULT, - ignore=DEFAULT, - replace=DEFAULT, - truncate=False, - extracts=DEFAULT, - conversions=DEFAULT, - columns=DEFAULT, - upsert=False, - analyze=False, - strict=DEFAULT, + pk: PrimaryKey | Default | None = DEFAULT, + foreign_keys: ForeignKeysType | Default | None = DEFAULT, + column_order: list[str] | Default | None = DEFAULT, + not_null: Iterable[str] | Default | None = DEFAULT, + defaults: dict[str, Any] | Default | None = DEFAULT, + batch_size: int | Default = DEFAULT, + hash_id: str | Default | None = DEFAULT, + hash_id_columns: Iterable[str] | Default | None = DEFAULT, + alter: bool | Default | None = DEFAULT, + ignore: bool | Default | None = DEFAULT, + replace: bool | Default | None = DEFAULT, + truncate: bool = False, + extracts: dict[str, str] | list[str] | Default | None = DEFAULT, + conversions: dict[str, str] | Default | None = DEFAULT, + columns: dict[str, Any] | Default | None = DEFAULT, + upsert: bool = False, + analyze: bool = False, + strict: bool | Default | None = DEFAULT, ) -> "Table": """ Like ``.insert()`` but takes a list of records and ensures that the table @@ -4715,6 +4727,7 @@ class Table(Queryable): elif isinstance(pk, str): self.last_pk = row[resolve_casing(pk, row)] else: + assert pk is not None self.last_pk = tuple( row[resolve_casing(p, row)] for p in pk ) @@ -4732,6 +4745,7 @@ class Table(Queryable): pk_index = column_names.index(resolve_casing(pk, column_names)) self.last_pk = first_record_list[pk_index] else: + assert pk is not None self.last_pk = tuple( first_record_list[ column_names.index(resolve_casing(p, column_names)) @@ -4743,6 +4757,7 @@ class Table(Queryable): if hash_id: self.last_pk = hash_record(first_record_dict, hash_id_columns) else: + assert pk is not None self.last_pk = ( first_record_dict[resolve_casing(pk, first_record_dict)] if isinstance(pk, str) @@ -4759,19 +4774,19 @@ class Table(Queryable): def upsert( self, - record, - pk=DEFAULT, - foreign_keys=DEFAULT, - column_order=DEFAULT, - not_null=DEFAULT, - defaults=DEFAULT, - hash_id=DEFAULT, - hash_id_columns=DEFAULT, - alter=DEFAULT, - extracts=DEFAULT, - conversions=DEFAULT, - columns=DEFAULT, - strict=DEFAULT, + record: dict[str, Any], + pk: PrimaryKey | Default | None = DEFAULT, + foreign_keys: ForeignKeysType | Default | None = DEFAULT, + column_order: list[str] | Default | None = DEFAULT, + not_null: Iterable[str] | Default | None = DEFAULT, + defaults: dict[str, Any] | Default | None = DEFAULT, + hash_id: str | Default | None = DEFAULT, + hash_id_columns: Iterable[str] | Default | None = DEFAULT, + alter: bool | Default | None = DEFAULT, + extracts: dict[str, str] | list[str] | Default | None = DEFAULT, + conversions: dict[str, str] | Default | None = DEFAULT, + columns: dict[str, Any] | Default | None = DEFAULT, + strict: bool | Default | None = DEFAULT, ) -> "Table": """ Like ``.insert()`` but performs an ``UPSERT``, where records are inserted if they do @@ -4798,20 +4813,20 @@ class Table(Queryable): def upsert_all( self, records: Iterable[dict[str, Any]] | Iterable[Sequence[Any]], - pk=DEFAULT, - foreign_keys=DEFAULT, - column_order=DEFAULT, - not_null=DEFAULT, - defaults=DEFAULT, - batch_size=DEFAULT, - hash_id=DEFAULT, - hash_id_columns=DEFAULT, - alter=DEFAULT, - extracts=DEFAULT, - conversions=DEFAULT, - columns=DEFAULT, - analyze=False, - strict=DEFAULT, + pk: PrimaryKey | Default | None = DEFAULT, + foreign_keys: ForeignKeysType | Default | None = DEFAULT, + column_order: list[str] | Default | None = DEFAULT, + not_null: Iterable[str] | Default | None = DEFAULT, + defaults: dict[str, Any] | Default | None = DEFAULT, + batch_size: int | Default = DEFAULT, + hash_id: str | Default | None = DEFAULT, + hash_id_columns: Iterable[str] | Default | None = DEFAULT, + alter: bool | Default | None = DEFAULT, + extracts: dict[str, str] | list[str] | Default | None = DEFAULT, + conversions: dict[str, str] | Default | None = DEFAULT, + columns: dict[str, Any] | Default | None = DEFAULT, + analyze: bool = False, + strict: bool | Default | None = DEFAULT, ) -> "Table": """ Like ``.upsert()`` but can be applied to a list of records. diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index ed5a558..3145a6f 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -14,6 +14,7 @@ from typing import ( TYPE_CHECKING, Any, BinaryIO, + Generic, TypeVar, Union, cast, @@ -344,7 +345,11 @@ def rows_from_file( reader = csv.DictReader(decoded_fp, dialect=dialect) else: reader = csv.DictReader(decoded_fp) - rows = _extra_key_strategy(reader, ignore_extras, extras_key) + rows = _extra_key_strategy( + cast(Iterable[dict[str | None, object]], reader), + ignore_extras, + extras_key, + ) return _CloseableIterator(iter(rows), decoded_fp), Format.CSV elif format == Format.TSV: rows, _ = rows_from_file( @@ -487,12 +492,12 @@ class ValueTracker: del self.couldbe[key] -class NullProgressBar: +class NullProgressBar(Generic[T]): def __init__(self, *args: Iterable[T]) -> None: self.args = args def __iter__(self) -> Iterator[T]: - yield from self.args[0] # type: ignore + yield from self.args[0] def update(self, value: int) -> None: pass diff --git a/tests/test_cli.py b/tests/test_cli.py index 012900c..f60c7d5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1220,7 +1220,7 @@ def test_rows(db_path, args, expected): {"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}, ], - column_order=("id", "name", "age"), + column_order=["id", "name", "age"], ) result = CliRunner().invoke(cli.cli, ["rows", db_path, "dogs"] + args) assert expected == result.output.strip() diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py index 24889b3..c5c9dcd 100644 --- a/tests/test_cli_bulk.py +++ b/tests/test_cli_bulk.py @@ -91,6 +91,7 @@ def test_cli_bulk_batch_size(test_db_and_path): stdin=subprocess.PIPE, stdout=sys.stdout, ) + assert proc.stdin is not None # Writing one record should not commit proc.stdin.write(b'{"id": 3, "name": "Three"}\n\n') proc.stdin.flush() diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 01e7e94..0117862 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -577,6 +577,7 @@ def test_insert_streaming_batch_size_1(db_path): stdin=subprocess.PIPE, stdout=sys.stdout, ) + assert proc.stdin is not None proc.stdin.write(b'{"name": "Azi"}\n') proc.stdin.flush() diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 2d0a298..412b66c 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -83,7 +83,8 @@ def test_autocommit_connections_are_rejected(tmpdir, autocommit): ) def test_legacy_transaction_control_connection_is_accepted(tmpdir): conn = sqlite3.connect( - str(tmpdir / "test.db"), autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL + str(tmpdir / "test.db"), + autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL, # type: ignore[arg-type] ) db = Database(conn) db.table("t").insert({"id": 1}, pk="id") diff --git a/tests/test_create.py b/tests/test_create.py index 0af68a6..83ce403 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1182,6 +1182,7 @@ def test_works_with_pathlib_path(tmpdir): @pytest.mark.skipif(pd is None, reason="pandas and numpy are not installed") def test_create_table_numpy(fresh_db): + assert pd is not None df = pd.DataFrame({"col 1": range(3), "col 2": range(3)}) fresh_db.table("pandas").insert_all(df.to_dict(orient="records")) assert [ diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 271125c..f916c65 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -608,7 +608,7 @@ def test_foreign_key_is_immutable(): fk = ForeignKey("c", "pid", "p", "id") with pytest.raises(dataclasses.FrozenInstanceError): - fk.table = "other" + setattr(fk, "table", "other") def test_foreign_key_equality_and_hash_include_actions(): diff --git a/tests/test_fts.py b/tests/test_fts.py index 312b032..04b5bc3 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -510,7 +510,7 @@ def test_view_has_no_enable_fts(): db.create_view("hello", "select 1 + 1") # Views deliberately do not have an enable_fts() method with pytest.raises(AttributeError): - db.view("hello").enable_fts() # type: ignore[union-attr] + db.view("hello").enable_fts() # type: ignore[attr-defined] @pytest.mark.parametrize( diff --git a/tests/test_list_mode.py b/tests/test_list_mode.py index b9ab812..75f5a76 100644 --- a/tests/test_list_mode.py +++ b/tests/test_list_mode.py @@ -99,7 +99,7 @@ def test_list_mode_error_non_string_columns(): yield ["a", "b", "c"] with pytest.raises(ValueError, match="must be a list of column name strings"): - db.table("bad").insert_all(bad_data()) + db.table("bad").insert_all(bad_data()) # type: ignore[arg-type] def test_list_mode_error_mixed_types(): @@ -111,7 +111,7 @@ def test_list_mode_error_mixed_types(): yield {"id": 1, "name": "Alice"} # Should be a list, not dict with pytest.raises(ValueError, match="must also be lists"): - db.table("bad").insert_all(bad_data()) + db.table("bad").insert_all(bad_data()) # type: ignore[arg-type] def test_list_mode_empty_after_headers(): diff --git a/tests/test_register_function.py b/tests/test_register_function.py index 618bf1e..63f0570 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -86,21 +86,21 @@ def test_register_function_deterministic_tries_again_if_exception_raised(fresh_d def test_register_function_replace(fresh_db): @fresh_db.register_function() - def one(): + def one(): # pyright: ignore[reportRedeclaration] return "one" assert "one" == fresh_db.execute("select one()").fetchone()[0] # This will silently fail to replaec the function @fresh_db.register_function() - def one(): # noqa + def one(): # pyright: ignore[reportRedeclaration] return "two" assert "one" == fresh_db.execute("select one()").fetchone()[0] # This will replace it @fresh_db.register_function(replace=True) - def one(): # noqa + def one(): # pyright: ignore[reportRedeclaration] return "two" assert "two" == fresh_db.execute("select one()").fetchone()[0] diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 0f44cc7..8274557 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -8,7 +8,7 @@ from sqlite_utils.db import PrimaryKeyRequired def test_upsert(use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) table = db.table("table") - table.insert({"id": 1, "name": "Cleo"}, pk="id") + table.insert_all([{"id": 1, "name": "Cleo"}], pk="id", replace=True) table.upsert({"id": 1, "age": 5}, pk="id", alter=True) assert list(table.rows) == [{"id": 1, "name": "Cleo", "age": 5}] assert table.last_pk == 1 From 25c632fbbc286b6e5d622a975a2231c12ce6837c Mon Sep 17 00:00:00 2001 From: Rami Abdelrazzaq Date: Wed, 12 Aug 2026 16:05:09 -0500 Subject: [PATCH 20/33] Handle empty input in rows_from_file Closes #808 --- sqlite_utils/utils.py | 2 ++ tests/test_rows_from_file.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 3145a6f..06404eb 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -373,6 +373,8 @@ def rows_from_file( raise TypeError( "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO" ) + if not first_bytes: + return (), Format.CSV if first_bytes.startswith((b"[", b"{")): # TODO: Detect newline-JSON return rows_from_file(buffered, format=Format.JSON) diff --git a/tests/test_rows_from_file.py b/tests/test_rows_from_file.py index 8c080d6..3de3582 100644 --- a/tests/test_rows_from_file.py +++ b/tests/test_rows_from_file.py @@ -20,6 +20,13 @@ def test_rows_from_file_detect_format(input, expected_format): assert rows_list == [{"id": "1", "name": "Cleo"}] +@pytest.mark.parametrize("input", (b"", b" \n\t")) +def test_rows_from_file_empty_input(input): + rows, format = rows_from_file(BytesIO(input)) + assert format == Format.CSV + assert list(rows) == [] + + @pytest.mark.parametrize( "ignore_extras,extras_key,expected", ( From c5063f67b10ff194392dcbce7b64f409f866dd72 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 14:14:27 -0700 Subject: [PATCH 21/33] Use quoted SQL identifiers in convert --dry-run, closes #829 --- sqlite_utils/cli.py | 10 +++++----- tests/test_cli_convert.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c90c137..a8baff8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3283,12 +3283,12 @@ def convert( db.conn.create_function("preview_transform", 1, preview) sql = """ select - [{column}] as value, - preview_transform([{column}]) as preview - from [{table}]{where} limit 10 + {column} as value, + preview_transform({column}) as preview + from {table}{where} limit 10 """.format( - column=columns[0], - table=table, + column=quote_identifier(columns[0]), + table=quote_identifier(table), where=f" where {where}" if where is not None else "", ) for row in db.conn.execute(sql, where_args).fetchall(): diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 1101f0f..9f59d59 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -181,6 +181,34 @@ def test_convert_dryrun(test_db_and_path): assert result.output.strip().split("\n")[-1] == "Would affect 1 row" +def test_convert_dryrun_table_and_column_names_containing_closing_bracket( + fresh_db_and_path, +): + db, db_path = fresh_db_and_path + table_name = "table]name" + column_name = "column]name" + db[table_name].insert({column_name: "hello"}) + + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + table_name, + column_name, + "value.upper()", + "--dry-run", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output.strip() == ( + "hello\n --- becomes:\nHELLO\n\nWould affect 1 row" + ) + assert list(db[table_name].rows) == [{column_name: "hello"}] + + def test_convert_multi_dryrun(test_db_and_path): db_path = test_db_and_path[1] result = CliRunner().invoke( From e6be6267a4eda2d35e57a50400208fe1bb66d6d3 Mon Sep 17 00:00:00 2001 From: nyxst4ck Date: Wed, 12 Aug 2026 18:15:17 -0300 Subject: [PATCH 22/33] Use quote_identifier() in indexes/xindexes PRAGMA statements (#825) Closes #824 --- sqlite_utils/db.py | 14 ++++---------- tests/test_introspect.py | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 66dc700..a59597b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2375,14 +2375,11 @@ class Table(Queryable): @property def indexes(self) -> list[Index]: "List of indexes defined on this table." - sql = f'PRAGMA index_list("{self.name}")' + sql = f"PRAGMA index_list({quote_identifier(self.name)})" indexes = [] for row in self.db.execute_returning_dicts(sql): index_name = row["name"] - index_name_quoted = ( - f'"{index_name}"' if not index_name.startswith('"') else index_name - ) - column_sql = f"PRAGMA index_info({index_name_quoted})" + column_sql = f"PRAGMA index_info({quote_identifier(index_name)})" columns = [] for seqno, cid, name in self.db.execute(column_sql).fetchall(): columns.append(name) @@ -2397,14 +2394,11 @@ class Table(Queryable): @property def xindexes(self) -> list[XIndex]: "List of indexes defined on this table using the more detailed ``XIndex`` format." - sql = f'PRAGMA index_list("{self.name}")' + sql = f"PRAGMA index_list({quote_identifier(self.name)})" indexes = [] for row in self.db.execute_returning_dicts(sql): index_name = row["name"] - index_name_quoted = ( - f'"{index_name}"' if not index_name.startswith('"') else index_name - ) - column_sql = f"PRAGMA index_xinfo({index_name_quoted})" + column_sql = f"PRAGMA index_xinfo({quote_identifier(index_name)})" index_columns = [] for info in self.db.execute(column_sql).fetchall(): index_columns.append(XIndexColumn(*info)) diff --git a/tests/test_introspect.py b/tests/test_introspect.py index b0953f1..03b02cc 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -161,6 +161,31 @@ def test_xindexes(fresh_db): ] +def test_indexes_with_double_quotes_in_identifiers(fresh_db): + fresh_db['Go"sh'].insert({"id": 1, 'c"1': 2}, pk="id") + fresh_db['Go"sh'].create_index(['c"1']) + assert [(index.name, index.columns) for index in fresh_db['Go"sh'].indexes] == [ + ('idx_Go"sh_c"1', ['c"1']) + ] + assert fresh_db['Go"sh'].xindexes == [ + XIndex( + name='idx_Go"sh_c"1', + columns=[ + XIndexColumn(seqno=0, cid=1, name='c"1', desc=0, coll="BINARY", key=1), + XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll="BINARY", key=0), + ], + ) + ] + + +def test_transform_table_with_double_quotes_in_identifiers(fresh_db): + fresh_db['Go"sh'].insert({"id": 1, 'c"1': 2, "c2": 3}, pk="id") + fresh_db['Go"sh'].create_index(['c"1']) + fresh_db['Go"sh'].transform(types={"c2": str}) + assert fresh_db['Go"sh'].columns_dict["c2"] is str + assert [index.columns for index in fresh_db['Go"sh'].indexes] == [['c"1']] + + @pytest.mark.parametrize( "column,expected_table_guess", ( From 88b48fa1674c396bfda330d1c609bc7108f952f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 14:19:30 -0700 Subject: [PATCH 23/33] Fixed introspection of default values TRUE / FALSE / NULL Closes #836 --- sqlite_utils/db.py | 7 +++++++ tests/test_create.py | 20 ++++++++++++++++++++ tests/test_introspect.py | 15 +++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a59597b..2478189 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -5270,6 +5270,13 @@ def _decode_default_value(value: str) -> object: # It's a binary string, stored as hex to_decode = value[2:-1] return binascii.unhexlify(to_decode) + upper = value.upper() + if upper == "TRUE": + return True + if upper == "FALSE": + return False + if upper == "NULL": + return None # If it is a string containing a floating point number: try: return float(value) diff --git a/tests/test_create.py b/tests/test_create.py index 83ce403..e900aee 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1509,6 +1509,26 @@ def test_create_transform(fresh_db, cols, kwargs, expected_schema, should_transf assert fresh_db.table("demo").count == 1 +def test_create_transform_keyword_literal_defaults_unchanged(fresh_db): + fresh_db.execute( + "create table demo (" + "id integer primary key, " + "enabled integer default TRUE, " + "disabled integer default FALSE, " + "nullable text default NULL" + ")" + ) + traces = [] + with fresh_db.tracer(lambda sql, parameters: traces.append((sql, parameters))): + fresh_db.table("demo").create( + {"id": int, "enabled": int, "disabled": int, "nullable": str}, + pk="id", + defaults={"enabled": True, "disabled": False, "nullable": None}, + transform=True, + ) + assert not any(sql.startswith("CREATE TABLE") for sql, _ in traces) + + def test_rename_table(fresh_db): fresh_db.table("t").insert({"foo": "bar"}) assert ["t"] == fresh_db.table_names() diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 03b02cc..343424d 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -393,6 +393,21 @@ def test_table_default_values_escaped_quotes(fresh_db): assert fresh_db.table("t").default_values == {"name": "O'Brien"} +def test_table_default_values_keyword_literals(fresh_db): + fresh_db.execute( + "create table t (" + "enabled integer default TRUE, " + "disabled integer default false, " + "nullable text default NULL" + ")" + ) + assert fresh_db.table("t").default_values == { + "enabled": True, + "disabled": False, + "nullable": None, + } + + def test_pks_use_primary_key_declaration_order(fresh_db): # PRIMARY KEY (a, b) declared against columns stored in order (b, a) - # pks must follow the declaration order, which is what SQLite uses to From e4784ec1200b7408a037c50009dc07d88a5ac577 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 14:38:20 -0700 Subject: [PATCH 24/33] Changelog updates Refs #808, #811, #816, #821, #824, #825, #828, #829, #833, #836, #837 --- docs/changelog.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2950dd4..8540e1b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,14 @@ Unreleased - ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) - ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) - ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) +- ``table.default_values`` now unescapes doubled single quotes in string defaults, so a default such as ``'O''Brien'`` is returned as ``"O'Brien"``. Thanks, `ikatyal2110 `__. (`#811 `__) +- ``table.default_values`` now decodes unquoted ``TRUE``, ``FALSE`` and ``NULL`` default literals as ``True``, ``False`` and ``None`` respectively. (:issue:`836`) +- ``table.enable_fts(..., tokenize=...)`` and ``sqlite-utils enable-fts --tokenize`` now safely quote the tokenizer argument, preventing a crafted value from injecting additional SQL. Thanks, `Bunlong Heng `__. (`#828 `__) +- ``rows_where()``, ``pks_and_rows_where()``, ``search()`` and ``search_sql()`` now support ``offset=`` without requiring ``limit=``. The ``sqlite-utils rows --offset`` option now works without ``--limit`` too. Thanks, `ethanhawkes-gif `__. (:issue:`816`, `#821 `__) +- Empty or whitespace-only input passed to ``rows_from_file()`` is now handled as an empty CSV file instead of raising ``csv.Error``. Thanks, `Rami Abdelrazzaq `__. (:issue:`808`, `#837 `__) +- ``sqlite-utils convert --dry-run`` now works for table and column names containing closing square brackets. (:issue:`829`) +- ``table.indexes`` and ``table.xindexes`` now work for table, index and column names containing double quotes. This also fixes ``table.transform()`` for tables with those identifiers. Thanks, `nyxst4ck `__. (:issue:`824`, `#825 `__) +- Improved type annotations throughout the package and added Pyright regression checks to CI. (:issue:`833`) .. _v3_39_1: From 57192ef4e36c334bc2946a10547bf64d63621127 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 14:42:24 -0700 Subject: [PATCH 25/33] table.transform(rename=...) now preserves indexes, closes #822 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 98 +++++++++++++++++++++++++++++++---------- tests/test_transform.py | 80 ++++++++++++++++++++++++++++----- 3 files changed, 146 insertions(+), 33 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8540e1b..31b4ea3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ Unreleased - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) - ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) - ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) +- ``table.transform(rename=...)`` now preserves explicit indexes on renamed columns by dropping and recreating those indexes against the new column names. Previously this raised a ``TransformError``. (:issue:`822`) - ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) - ``table.default_values`` now unescapes doubled single quotes in string defaults, so a default such as ``'O''Brien'`` is returned as ``"O'Brien"``. Thanks, `ikatyal2110 `__. (`#811 `__) - ``table.default_values`` now decodes unquoted ``TRUE``, ``FALSE`` and ``NULL`` default literals as ``True``, ``False`` and ``None`` respectively. (:issue:`836`) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2478189..edefbab 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2947,6 +2947,80 @@ class Table(Queryable): new_cols=", ".join(quote_identifier(col) for col in new_cols), ) sqls.append(copy_sql) + # Capture indexes before the old table is changed. Simple indexes that + # reference renamed columns are recreated from structured PRAGMA + # metadata instead of editing their stored CREATE INDEX SQL. + index_drop_sqls = [] + index_create_sqls = [] + xindexes_by_name = {index.name: index for index in self.xindexes} + for index in self.indexes: + if index.origin == "pk": + continue + index_sql = self.db.execute( + """SELECT sql FROM sqlite_master WHERE type = 'index' AND name = :index_name;""", + {"index_name": index.name}, + ).fetchall()[0][0] + if index_sql is None: + raise TransformError( + f"Index '{index.name}' on table '{self.name}' does not have a " + "CREATE INDEX statement. You must manually drop this index prior to running this " + "transformation and manually recreate the new index after running this transformation." + ) + dropped_index_column = next( + (column for column in index.columns if column in drop), None + ) + renamed_index_column = next( + (column for column in index.columns if column in rename), None + ) + if dropped_index_column is not None: + raise TransformError( + f"Index '{index.name}' column '{dropped_index_column}' is not in updated table '{self.name}'. " + f"You must manually drop this index prior to running this transformation " + f"and manually recreate the new index after running this transformation. " + f"The original index sql statement is: `{index_sql}`. No changes have been applied to this table." + ) + xindex = xindexes_by_name[index.name] + indexed_columns = sorted( + (column for column in xindex.columns if column.key), + key=lambda column: column.seqno, + ) + if (rename or drop) and ( + index.partial or any(column.name is None for column in indexed_columns) + ): + raise TransformError( + f"Index '{index.name}' is a partial or expression index, so it " + f"cannot be safely recreated while columns are renamed or dropped. " + f"You must manually drop this index prior to running this transformation " + f"and manually recreate the new index after running this transformation. " + f"The original index sql statement is: `{index_sql}`. No changes have been applied to this table." + ) + if renamed_index_column is not None: + columns_sql = [] + for column in indexed_columns: + assert column.name is not None + column_sql = quote_identifier( + rename.get(column.name) or column.name + ) + if column.coll and column.coll.upper() != "BINARY": + column_sql += f" COLLATE {quote_identifier(column.coll)}" + if column.desc: + column_sql += " DESC" + columns_sql.append(column_sql) + index_sql = "CREATE {unique}INDEX {index_name} ON {table_name} ({columns})".format( + unique="UNIQUE " if index.unique else "", + index_name=quote_identifier(index.name), + table_name=quote_identifier(self.name), + columns=", ".join(columns_sql), + ) + index_drop_sqls.append( + f"DROP INDEX IF EXISTS {quote_identifier(index.name)};" + ) + elif keep_table: + index_drop_sqls.append( + f"DROP INDEX IF EXISTS {quote_identifier(index.name)};" + ) + index_create_sqls.append(index_sql) + sqls.extend(index_drop_sqls) # Drop (or keep) the old table, then rename the new one into place. # Since SQLite 3.25 ALTER TABLE ... RENAME TO rewrites references to # the renamed table in every view definition, which fails if a view @@ -2976,29 +3050,7 @@ class Table(Queryable): ) ) # Re-add existing indexes - for index in self.indexes: - if index.origin != "pk": - index_sql = self.db.execute( - """SELECT sql FROM sqlite_master WHERE type = 'index' AND name = :index_name;""", - {"index_name": index.name}, - ).fetchall()[0][0] - if index_sql is None: - raise TransformError( - f"Index '{index.name}' on table '{self.name}' does not have a " - "CREATE INDEX statement. You must manually drop this index prior to running this " - "transformation and manually recreate the new index after running this transformation." - ) - if keep_table: - sqls.append(f"DROP INDEX IF EXISTS {quote_identifier(index.name)};") - for col in index.columns: - if col in rename or col in drop: - raise TransformError( - f"Index '{index.name}' column '{col}' is not in updated table '{self.name}'. " - f"You must manually drop this index prior to running this transformation " - f"and manually recreate the new index after running this transformation. " - f"The original index sql statement is: `{index_sql}`. No changes have been applied to this table." - ) - sqls.append(index_sql) + sqls.extend(index_create_sqls) return sqls def extract( diff --git a/tests/test_transform.py b/tests/test_transform.py index 28fa4d7..5793f10 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -893,22 +893,15 @@ def test_transform_retains_indexes_with_foreign_keys(fresh_db): ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}" -@pytest.mark.parametrize( - "transform_params", - [ - {"rename": {"age": "dog_age"}}, - {"drop": ["age"]}, - ], -) -def test_transform_with_indexes_errors(fresh_db, transform_params): - # Should error with a compound (name, age) index if age is renamed or dropped +def test_transform_with_indexes_errors(fresh_db): + # Should error with a compound (name, age) index if age is dropped dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id") dogs.create_index(["name", "age"]) with pytest.raises(TransformError) as excinfo: - dogs.transform(**transform_params) + dogs.transform(drop=["age"]) assert ( "Index 'idx_dogs_name_age' column 'age' is not in updated table 'dogs'. " @@ -917,6 +910,73 @@ def test_transform_with_indexes_errors(fresh_db, transform_params): ) +@pytest.mark.parametrize( + ("table_name", "index_name"), + (("name", "idx_name"), ("t", "name")), +) +def test_transform_rename_column_with_index(fresh_db, table_name, index_name): + # https://github.com/simonw/sqlite-utils/issues/822 + # Use the same name for the table, column and index to ensure only the + # indexed column changes. + table = fresh_db.table(table_name) + table.insert({"id": 1, "name": "Cleo"}, pk="id") + table.create_index(["name"], index_name=index_name) + + sqls = table.transform_sql(rename={"name": "full_name"}, tmp_suffix="suffix") + drop_index_sql = f'DROP INDEX IF EXISTS "{index_name}";' + assert drop_index_sql in sqls + assert sqls.index(drop_index_sql) < sqls.index(f'DROP TABLE "{table_name}";') + + table.transform(rename={"name": "full_name"}) + + assert [column.name for column in table.columns] == ["id", "full_name"] + assert [(index.name, index.columns) for index in table.indexes] == [ + (index_name, ["full_name"]) + ] + + +def test_transform_recreates_renamed_index_from_metadata(fresh_db): + table = fresh_db.table("t") + table.insert({"alpha": "one", "beta": "two"}) + # Deliberately use unquoted SQL and index details that need to survive the + # reconstruction. Renaming both columns also guards against cascading + # string substitutions. + fresh_db.execute( + "CREATE UNIQUE INDEX swap_idx ON t(alpha COLLATE NOCASE DESC, beta)" + ) + + table.transform(rename={"alpha": "beta", "beta": "alpha"}) + + assert table.columns_dict == {"beta": str, "alpha": str} + assert [(index.name, index.unique, index.columns) for index in table.indexes] == [ + ("swap_idx", 1, ["beta", "alpha"]) + ] + key_columns = [column for column in table.xindexes[0].columns if column.key] + assert [(column.name, column.desc, column.coll) for column in key_columns] == [ + ("beta", 1, "NOCASE"), + ("alpha", 0, "BINARY"), + ] + + +@pytest.mark.parametrize( + "index_sql", + ( + "CREATE INDEX idx_t_name ON t(lower(name))", + "CREATE INDEX idx_t_name ON t(name) WHERE name IS NOT NULL", + ), +) +def test_transform_rename_complex_index_errors(fresh_db, index_sql): + table = fresh_db.table("t") + table.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute(index_sql) + + with pytest.raises(TransformError, match="partial or expression index"): + table.transform(rename={"name": "full_name"}) + + assert table.columns_dict == {"id": int, "name": str} + assert [index.name for index in table.indexes] == ["idx_t_name"] + + def test_transform_with_unique_constraint_implicit_index(fresh_db): dogs = fresh_db.table("dogs") # Create a table with a UNIQUE constraint on 'name', which creates an implicit index From fcfccea8132e4aa6167a14f9afec5a690de7485c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 16:43:33 -0700 Subject: [PATCH 26/33] Support ANY column types for strict tables Closes #790, #820 --- docs/changelog.rst | 1 + docs/cli-reference.rst | 6 +-- docs/cli.rst | 17 +++++++- docs/python-api.rst | 23 +++++++++- sqlite_utils/__init__.py | 11 ++++- sqlite_utils/cli.py | 22 ++++++---- sqlite_utils/db.py | 14 +++++++ sqlite_utils/utils.py | 6 +++ tests/test_cli.py | 79 ++++++++++++++++++++++++++++++++++- tests/test_column_affinity.py | 3 ++ tests/test_create.py | 40 ++++++++++++++++++ tests/test_extract.py | 36 ++++++++++++++++ tests/test_transform.py | 50 ++++++++++++++++++++++ 13 files changed, 292 insertions(+), 16 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 31b4ea3..0ef85ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) +- New ``sqlite_utils.ANY`` marker type for creating and introspecting SQLite ``ANY`` columns. The Python API and CLI can create, add and transform these columns, and ``table.transform()`` and ``table.extract()`` now preserve ``ANY`` columns and their values in ``STRICT`` tables. (:issue:`790`) - ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) - ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) - ``table.transform(rename=...)`` now preserves explicit indexes on renamed columns by dropping and recreating those indexes against the new column names. Previously this raised a ``TransformError``. (:issue:`822`) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index a4ec402..c53d642 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -494,7 +494,7 @@ See :ref:`cli_transform_table`. Options: --type ... Change column type to INTEGER, TEXT, FLOAT, - REAL or BLOB + REAL, BLOB or ANY --drop TEXT Drop this column --rename ... Rename this column to X -o, --column-order TEXT Reorder columns @@ -963,7 +963,7 @@ See :ref:`cli_create_table`. height real \ photo blob --pk id - Valid column types are text, integer, real, float and blob. + Valid column types are text, integer, real, float, blob and any. Options: --pk TEXT Column to use as primary key @@ -1257,7 +1257,7 @@ See :ref:`cli_add_column`. :: Usage: sqlite-utils add-column [OPTIONS] PATH TABLE COL_NAME - [integer|int|float|real|text|str|blob|bytes] + [integer|int|float|real|text|str|blob|bytes|any] Add a column to the specified table diff --git a/docs/cli.rst b/docs/cli.rst index cf241aa..417911a 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1390,7 +1390,14 @@ Use ``--type column-name type`` to override the type automatically chosen when t This is useful for values such as ZIP codes, which may look like integers but should be stored as ``TEXT`` to preserve leading zeros. -The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL`` or ``BLOB``. Column types are matched case-insensitively. +The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL``, ``BLOB`` or ``ANY``. Column types are matched case-insensitively. + +``ANY`` is especially useful with ``--strict``. An ``ANY`` column in a strict table preserves values without coercion, so text such as ``000123`` remains text instead of being converted to an integer: + +.. code-block:: bash + + sqlite-utils insert events.db events events.csv --csv --strict \ + --type payload any As with detected column types, ``--type`` only affects tables created by the command. If the table already exists, its existing column types are left unchanged. @@ -2141,6 +2148,12 @@ You can create a table in `SQLite STRICT mode ` @@ -1569,7 +1582,7 @@ You can specify the ``col_type`` argument either using a SQLite type as a string The ``col_type`` is optional - if you omit it the type of ``TEXT`` will be used. -SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"``, ``"REAL"`` or ``"BLOB"``. +SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"``, ``"REAL"``, ``"BLOB"`` or ``"ANY"``. You can use the ``sqlite_utils.ANY`` marker instead of the ``"ANY"`` string. If you pass a Python type, it will be mapped to SQLite types as shown here:: @@ -1582,6 +1595,7 @@ If you pass a Python type, it will be mapped to SQLite types as shown here:: datetime.date: "TEXT" datetime.time: "TEXT" datetime.timedelta: "TEXT" + sqlite_utils.ANY: "ANY" # If numpy is installed np.int8: "INTEGER" @@ -1831,6 +1845,8 @@ Pass ``strict=False`` to convert a strict table back to a regular non-strict tab table.transform(strict=False) +If the table has ``ANY`` columns, converting it to non-strict mode can coerce text values that look numeric. For example, SQLite converts ``"000123"`` to the integer ``123`` when copying it into an ordinary ``ANY`` column. This is SQLite's documented distinction between `STRICT and ordinary ANY columns `__. + The default is ``strict=None``, which preserves the table's existing strict mode. Passing ``strict=True`` raises ``sqlite_utils.db.TransformError`` if the available SQLite version does not support strict tables. @@ -2458,6 +2474,11 @@ The ``.columns_dict`` property returns a dictionary version of the columns with >>> db.table("PlantType").columns_dict {'id': , 'value': } +SQLite ``ANY`` columns are represented by the ``sqlite_utils.ANY`` marker type:: + + >>> db.table("events").columns_dict + {'id': , 'payload': } + .. _python_api_introspection_default_values: .default_values diff --git a/sqlite_utils/__init__.py b/sqlite_utils/__init__.py index 0d25716..3f350e1 100644 --- a/sqlite_utils/__init__.py +++ b/sqlite_utils/__init__.py @@ -1,6 +1,13 @@ from .db import Database from .hookspecs import hookimpl, hookspec from .migrations import Migrations -from .utils import suggest_column_types +from .utils import ANY, suggest_column_types -__all__ = ["Database", "Migrations", "hookimpl", "hookspec", "suggest_column_types"] +__all__ = [ + "ANY", + "Database", + "Migrations", + "hookimpl", + "hookspec", + "suggest_column_types", +] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index a8baff8..c230902 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -76,7 +76,7 @@ def _close_databases(ctx): pass -VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "REAL", "BLOB") +VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "REAL", "BLOB", "ANY") UNICODE_ERROR = """ {} @@ -489,7 +489,17 @@ def dump(path, load_extension): @click.argument( "col_type", type=click.Choice( - ["integer", "int", "float", "real", "text", "str", "blob", "bytes"], + [ + "integer", + "int", + "float", + "real", + "text", + "str", + "blob", + "bytes", + "any", + ], case_sensitive=False, ), required=False, @@ -1758,7 +1768,7 @@ def create_table( height real \\ photo blob --pk id - Valid column types are text, integer, real, float and blob. + Valid column types are text, integer, real, float, blob and any. """ db = sqlite_utils.Database(path) _register_db_for_cleanup(db) @@ -2668,12 +2678,10 @@ def schema( "--type", type=( str, - click.Choice( - ["INTEGER", "TEXT", "FLOAT", "REAL", "BLOB"], case_sensitive=False - ), + click.Choice(list(VALID_COLUMN_TYPES), case_sensitive=False), ), multiple=True, - help="Change column type to INTEGER, TEXT, FLOAT, REAL or BLOB", + help="Change column type to INTEGER, TEXT, FLOAT, REAL, BLOB or ANY", ) @click.option("--drop", type=str, multiple=True, help="Drop this column") @click.option( diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index edefbab..9c0b402 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -39,6 +39,7 @@ from .create_table_parser import ( sql_ends_in_line_comment, ) from .utils import ( + ANY, OperationalError, chunks, column_affinity, @@ -366,6 +367,7 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = { decimal.Decimal: "REAL", None.__class__: "TEXT", uuid.UUID: "TEXT", + ANY: "ANY", # SQLite explicit types "TEXT": "TEXT", "INTEGER": "INTEGER", @@ -380,6 +382,8 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = { "real": "REAL", "blob": "BLOB", "bytes": "BLOB", + "ANY": "ANY", + "any": "ANY", } # If numpy is available, add more types if np: @@ -3092,6 +3096,15 @@ class Table(Queryable): if col in columns } if lookup_table.exists(): + if ( + self.strict + and ANY in lookup_columns_definition.values() + and not lookup_table.strict + ): + raise InvalidColumns( + f"Lookup table {table} already exists but is not STRICT, " + "so it cannot preserve ANY column values" + ) if not set(lookup_columns_definition.items()).issubset( lookup_table.columns_dict.items() ): @@ -3105,6 +3118,7 @@ class Table(Queryable): **lookup_columns_definition, }, pk="id", + strict=self.strict, ) lookup_columns = [(rename.get(col) or col) for col in columns] lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 06404eb..ee6695b 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -59,6 +59,10 @@ Row = dict[str, RowValue] T = TypeVar("T") +class ANY: + """Marker type for an SQLite ``ANY`` column.""" + + class _CloseableIterator(Iterator[Row]): """Iterator wrapper that closes a file when iteration is complete.""" @@ -178,6 +182,8 @@ def column_affinity(column_type: str) -> type: return bytes if "REAL" in column_type or "FLOA" in column_type or "DOUB" in column_type: return float + if column_type == "ANY": + return ANY # Default is 'NUMERIC', which we currently also treat as float return float diff --git a/tests/test_cli.py b/tests/test_cli.py index f60c7d5..064026a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,7 +9,7 @@ from pathlib import Path import pytest from click.testing import CliRunner -from sqlite_utils import Database, cli +from sqlite_utils import ANY, Database, cli from sqlite_utils.db import ForeignKey, Index @@ -355,6 +355,7 @@ def test_create_index_desc(db_path): ("blob", "BLOB", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), ("blob", "bytes", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), ("blob", "BYTES", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), + ("anything", "any", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "anything" ANY)'), ("default", None, 'CREATE TABLE "dogs" (\n "name" TEXT\n, "default" TEXT)'), ), ) @@ -2007,6 +2008,25 @@ def test_transform_strict_option_with_invalid_data(db_path): assert not any(name.startswith("dogs_new_") for name in db.table_names()) +def test_transform_column_to_any(db_path): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db.table("items").create({"data": str}, strict=True) + db.table("items").insert({"data": "000123"}) + + result = CliRunner().invoke( + cli.cli, ["transform", db_path, "items", "--type", "data", "any"] + ) + + assert result.exit_code == 0, result.output + assert db.table("items").columns_dict == {"data": ANY} + assert db.execute("select typeof(data), data from items").fetchone() == ( + "text", + "000123", + ) + + @pytest.mark.parametrize( "extra_args,expected_schema", ( @@ -2872,6 +2892,30 @@ def test_create_table_strict(strict): assert db.table("items").columns_dict == {"id": int, "w": float} +def test_create_table_strict_any(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + result = runner.invoke( + cli.cli, + [ + "create-table", + "test.db", + "items", + "id", + "integer", + "data", + "any", + "--strict", + ], + ) + assert result.exit_code == 0, result.output + assert db.table("items").strict is True + assert db.table("items").columns_dict == {"id": int, "data": ANY} + + @pytest.mark.parametrize("method", ("insert", "upsert")) @pytest.mark.parametrize("strict", (False, True)) def test_insert_upsert_strict(tmpdir, method, strict): @@ -2887,6 +2931,39 @@ def test_insert_upsert_strict(tmpdir, method, strict): assert db.table("items").strict == strict or not db.supports_strict +@pytest.mark.parametrize("method", ("insert", "upsert")) +def test_insert_upsert_strict_any(tmpdir, method): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db.close() + result = CliRunner().invoke( + cli.cli, + [ + method, + db_path, + "items", + "-", + "--csv", + "--pk", + "id", + "--type", + "data", + "any", + "--strict", + ], + input="id,data\n1,000123", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert db.table("items").columns_dict == {"id": int, "data": ANY} + assert db.execute("select typeof(data), data from items").fetchone() == ( + "text", + "000123", + ) + + def test_extract_bad_column_clean_error(db_path): db = Database(db_path) db.table("trees").insert({"id": 1, "species": "Palm"}, pk="id") diff --git a/tests/test_column_affinity.py b/tests/test_column_affinity.py index 8c619e1..2d7846e 100644 --- a/tests/test_column_affinity.py +++ b/tests/test_column_affinity.py @@ -1,5 +1,6 @@ import pytest +from sqlite_utils import ANY from sqlite_utils.utils import column_affinity EXAMPLES = [ @@ -26,6 +27,8 @@ EXAMPLES = [ ("DOUBLE", float), ("DOUBLE PRECISION", float), ("FLOAT", float), + ("ANY", ANY), + ("any", ANY), # Numeric, treated as float: ("NUMERIC", float), ("DECIMAL(10,5)", float), diff --git a/tests/test_create.py b/tests/test_create.py index e900aee..b738df9 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -7,6 +7,7 @@ import uuid import pytest +from sqlite_utils import ANY from sqlite_utils.db import ( AlterError, Database, @@ -1366,6 +1367,18 @@ def test_quote(fresh_db, input, expected): {"col": list}, '"col" TEXT', ), + ( + {"col": ANY}, + '"col" ANY', + ), + ( + {"col": "ANY"}, + '"col" ANY', + ), + ( + {"col": "any"}, + '"col" ANY', + ), ), ) def test_create_table_sql(fresh_db, columns, expected_sql_middle): @@ -1589,6 +1602,33 @@ def test_create_strict(fresh_db, strict): assert table.strict == strict or not fresh_db.supports_strict +def test_create_strict_with_any(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + table = fresh_db.table("items").create( + {"id": int, "data": ANY}, pk="id", strict=True + ) + table.insert_all( + [ + {"id": 1, "data": 42}, + {"id": 2, "data": "000123"}, + {"id": 3, "data": 3.14}, + {"id": 4, "data": b"bytes"}, + {"id": 5, "data": None}, + ] + ) + assert table.columns_dict == {"id": int, "data": ANY} + assert fresh_db.execute( + "select typeof(data), data from items order by id" + ).fetchall() == [ + ("integer", 42), + ("text", "000123"), + ("real", 3.14), + ("blob", b"bytes"), + ("null", None), + ] + + def test_bad_table_and_view_exceptions(fresh_db): fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.create_view("v", "select * from t") diff --git a/tests/test_extract.py b/tests/test_extract.py index 72579c4..f855041 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -2,6 +2,7 @@ import itertools import pytest +from sqlite_utils import ANY from sqlite_utils.db import InvalidColumns @@ -305,3 +306,38 @@ def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db): fresh_db.table("t1").extract(["species"], table="lk") fresh_db.table("t2").extract(["species"], table="lk") assert fresh_db.table("lk").count == 1 + + +def test_extract_preserves_strict_any(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (id integer primary key, data any) strict") + fresh_db.execute("insert into items values (1, ?)", ("000123",)) + + fresh_db["items"].extract("data", table="data_values") + + lookup = fresh_db["data_values"] + assert lookup.strict is True + assert lookup.columns_dict == {"id": int, "data": ANY} + assert fresh_db.execute( + "select typeof(data), data from data_values" + ).fetchone() == ("text", "000123") + + +def test_extract_strict_any_rejects_non_strict_lookup(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (data any) strict") + fresh_db.execute("insert into items values (?)", ("000123",)) + fresh_db.execute("create table data_values (id integer primary key, data any)") + + with pytest.raises( + InvalidColumns, + match="is not STRICT, so it cannot preserve ANY column values", + ): + fresh_db["items"].extract("data", table="data_values") + + assert fresh_db.execute("select typeof(data), data from items").fetchone() == ( + "text", + "000123", + ) diff --git a/tests/test_transform.py b/tests/test_transform.py index 5793f10..6a8a143 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -2,6 +2,7 @@ import sqlite3 import pytest +from sqlite_utils import ANY from sqlite_utils.db import Check, ForeignKey, TransactionError, TransformError from sqlite_utils.utils import OperationalError @@ -823,6 +824,55 @@ def test_transform_to_strict_not_supported(fresh_db, method_name): assert table.strict is False +def test_transform_preserves_any_column_in_strict_table(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (id integer primary key, data any) strict") + fresh_db.conn.executemany( + "insert into items values (?, ?)", + [ + (1, 42), + (2, "000123"), + (3, 3.14), + (4, b"bytes"), + (5, None), + ], + ) + table = fresh_db["items"] + + table.transform() + + assert table.strict is True + assert table.columns_dict == {"id": int, "data": ANY} + assert fresh_db.execute( + "select typeof(data), data from items order by id" + ).fetchall() == [ + ("integer", 42), + ("text", "000123"), + ("real", 3.14), + ("blob", b"bytes"), + ("null", None), + ] + + +def test_transform_any_column_from_strict_to_non_strict(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (data any) strict") + fresh_db.execute("insert into items values (?)", ("000123",)) + table = fresh_db["items"] + + table.transform(strict=False) + + assert table.strict is False + assert table.columns_dict == {"data": ANY} + # Ordinary non-STRICT ANY columns apply NUMERIC affinity + assert fresh_db.execute("select typeof(data), data from items").fetchone() == ( + "integer", + 123, + ) + + @pytest.mark.parametrize( "indexes, transform_params", [ From 2b52b5ed6f4a6e553e3620d8424374fc7cbf95fd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 18:39:00 -0700 Subject: [PATCH 27/33] Preserve AUTOINCREMENT through transforms --- docs/changelog.rst | 1 + sqlite_utils/create_table_parser.py | 30 ++++++++++++++- sqlite_utils/db.py | 60 +++++++++++++++++++++++++++++ tests/test_create_table_parser.py | 31 +++++++++++++++ tests/test_transform.py | 18 +++++++++ 5 files changed, 139 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0ef85ca..624808f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Unreleased ---------- +- ``table.transform()`` now preserves ``AUTOINCREMENT`` primary keys and their sequence high-water marks. Previously a transform removed ``AUTOINCREMENT`` and could reuse deleted row IDs. (:issue:`602`) - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) - New ``sqlite_utils.ANY`` marker type for creating and introspecting SQLite ``ANY`` columns. The Python API and CLI can create, add and transform these columns, and ``table.transform()`` and ``table.extract()`` now preserve ``ANY`` columns and their values in ``STRICT`` tables. (:issue:`790`) - ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py index 2d891ae..d426286 100644 --- a/sqlite_utils/create_table_parser.py +++ b/sqlite_utils/create_table_parser.py @@ -1,4 +1,4 @@ -"""Helpers for parsing CHECK constraints from SQLite CREATE TABLE SQL. +"""Helpers for parsing constraints from SQLite CREATE TABLE SQL. SQLite does not expose CHECK constraints through a pragma, so preserving them across a table rebuild requires reading ``sqlite_schema.sql``. This module is @@ -564,6 +564,34 @@ def parse_checks(create_sql: str) -> list[Check]: return checks +def parse_autoincrement(create_sql: str) -> str | None: + """Return the AUTOINCREMENT column from a valid CREATE TABLE statement.""" + body_info = _table_body(create_sql) + if body_info is None: + return None + body, _ = body_info + for item, _, _ in _split_spans(body, _lex(body)): + item_tokens = _meaningful(_lex(item)) + if not item_tokens: + continue + head = item_tokens[0] + if ( + head.kind == "word" and head.text.upper() in _TABLE_CONSTRAINT_KEYWORDS + ) or head.is_keyword("CONSTRAINT"): + continue + column = _unquote(head.text) + index = 1 + while index < len(item_tokens): + token = item_tokens[index] + if token.text == "(": + index = _matching_paren(item_tokens, index) + 1 + continue + if token.is_keyword("AUTOINCREMENT"): + return column + index += 1 + return None + + def parse_column_comments(create_sql: str) -> dict[str, ColumnComments]: """Return comments immediately before and after each column definition.""" body_info = _table_body(create_sql) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9c0b402..48c5d5d 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -33,6 +33,7 @@ from .create_table_parser import ( ColumnComments, ParseError, check_references_identifier, + parse_autoincrement, parse_checks, parse_column_comments, rewrite_check_expression, @@ -1422,6 +1423,7 @@ class Database: strict: bool = False, _checks: Iterable[Check] | None = None, _column_comments: Mapping[str, ColumnComments] | None = None, + _autoincrement: str | None = None, ) -> str: """ Returns the SQL ``CREATE TABLE`` statement for creating the specified table. @@ -1525,10 +1527,22 @@ class Database: column_items.insert(0, (pk, int)) elif pk: pk = [resolve_casing(p, [c[0] for c in column_items]) for p in pk] + if _autoincrement is not None: + _autoincrement = resolve_casing( + _autoincrement, [c[0] for c in column_items] + ) + if _autoincrement != single_pk: + raise ValueError("AUTOINCREMENT requires a single-column primary key") for column_name, column_type in column_items: column_extras = [] if column_name == single_pk: column_extras.append("PRIMARY KEY") + if column_name == _autoincrement: + if COLUMN_TYPE_MAPPING[column_type] != "INTEGER": + raise ValueError( + "AUTOINCREMENT requires an INTEGER PRIMARY KEY column" + ) + column_extras.append("AUTOINCREMENT") if column_name in not_null: column_extras.append("NOT NULL") if column_name in defaults and defaults[column_name] is not None: @@ -2748,6 +2762,7 @@ class Table(Queryable): try: existing_checks = self.checks existing_column_comments = parse_column_comments(self.schema) + existing_autoincrement = parse_autoincrement(self.schema) except ParseError as ex: raise TransformError( f"Could not parse table schema for table {self.name!r}: {ex}" @@ -2870,6 +2885,11 @@ class Table(Queryable): new_column_pairs.append((new_name, type_)) copy_from_to[name] = new_name + if existing_autoincrement: + existing_autoincrement = resolve_casing( + existing_autoincrement, existing_columns + ) + if pk is DEFAULT: pks_renamed = tuple( rename.get(pk_name) or pk_name @@ -2880,6 +2900,28 @@ class Table(Queryable): else: pk = pks_renamed + create_table_autoincrement = None + if existing_autoincrement and existing_autoincrement not in drop: + renamed_autoincrement = ( + rename.get(existing_autoincrement) or existing_autoincrement + ) + single_pk = pk[0] if isinstance(pk, (list, tuple)) and len(pk) == 1 else pk + new_column_types = dict(new_column_pairs) + if ( + single_pk == renamed_autoincrement + and COLUMN_TYPE_MAPPING.get(new_column_types.get(renamed_autoincrement)) + == "INTEGER" + ): + create_table_autoincrement = renamed_autoincrement + + autoincrement_sequence = None + if create_table_autoincrement: + sequence_row = self.db.execute( + "SELECT seq FROM sqlite_sequence WHERE name = ?", [self.name] + ).fetchone() + if sequence_row is not None: + autoincrement_sequence = sequence_row[0] + # not_null may be a set or dict, need to convert to a set create_table_not_null = { rename.get(c.name) or c.name @@ -2931,6 +2973,7 @@ class Table(Queryable): strict=self.strict if strict is None else strict, _checks=create_table_checks, _column_comments=create_table_column_comments, + _autoincrement=create_table_autoincrement, ).strip() ) @@ -3053,6 +3096,23 @@ class Table(Queryable): "ON" if legacy_alter_table_was_on else "OFF" ) ) + if autoincrement_sequence is not None: + table_name_literal = self.db.quote(self.name) + sqls.extend( + ( + "UPDATE sqlite_sequence SET seq = MAX(seq, {sequence}) " + "WHERE name = {table_name};".format( + sequence=autoincrement_sequence, + table_name=table_name_literal, + ), + "INSERT INTO sqlite_sequence (name, seq) " + "SELECT {table_name}, {sequence} WHERE NOT EXISTS " + "(SELECT 1 FROM sqlite_sequence WHERE name = {table_name});".format( + sequence=autoincrement_sequence, + table_name=table_name_literal, + ), + ) + ) # Re-add existing indexes sqls.extend(index_create_sqls) return sqls diff --git a/tests/test_create_table_parser.py b/tests/test_create_table_parser.py index a7aa0c0..54bf221 100644 --- a/tests/test_create_table_parser.py +++ b/tests/test_create_table_parser.py @@ -8,6 +8,7 @@ from sqlite_utils.create_table_parser import ( Check, ColumnComments, ParseError, + parse_autoincrement, parse_checks, parse_column_comments, ) @@ -117,6 +118,36 @@ def test_virtual_table_has_no_checks(): ) +@pytest.mark.parametrize( + "sql,expected", + [ + ( + "CREATE TABLE t(id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT)", + "id", + ), + ( + 'CREATE TABLE t("quoted id" INTEGER PRIMARY KEY AUTOINCREMENT)', + "quoted id", + ), + ( + 'CREATE TABLE t("autoincrement" INTEGER PRIMARY KEY, value TEXT)', + None, + ), + ( + "CREATE TABLE t(id INTEGER PRIMARY KEY /* AUTOINCREMENT */, value TEXT)", + None, + ), + ( + "CREATE TABLE t(id INTEGER PRIMARY KEY, value TEXT CHECK(value != 'AUTOINCREMENT'))", + None, + ), + ], +) +def test_parse_autoincrement(sql, expected): + sqlite3.connect(":memory:").execute(sql) + assert parse_autoincrement(sql) == expected + + comment_or_space = st.sampled_from( [ " ", diff --git a/tests/test_transform.py b/tests/test_transform.py index 6a8a143..e6096cd 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1053,6 +1053,24 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db): ) +def test_transform_preserves_autoincrement_and_sequence(fresh_db): + fresh_db.execute( + "CREATE TABLE entries (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT)" + ) + entries = fresh_db.table("entries") + entries.insert_all(({"value": "one"}, {"value": "two"})) + entries.delete(2) + + entries.transform(rename={"value": "label"}) + + assert "PRIMARY KEY AUTOINCREMENT" in entries.schema + entries.insert({"label": "three"}) + assert list(entries.rows) == [ + {"id": 1, "label": "one"}, + {"id": 3, "label": "three"}, + ] + + def test_transform_preserves_view(fresh_db): # https://github.com/simonw/sqlite-utils/issues/831 dogs = fresh_db.table("dogs") From 75ba58846206b2c1beb39836134e763bf34177aa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 18:46:18 -0700 Subject: [PATCH 28/33] Preserve composite UNIQUE constraints in transforms --- docs/changelog.rst | 1 + sqlite_utils/create_table_parser.py | 193 ++++++++++++++++++++++++++++ sqlite_utils/db.py | 119 +++++++++++++++++ tests/test_create_table_parser.py | 48 +++++++ tests/test_transform.py | 74 +++++++++-- 5 files changed, 425 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 624808f..fe215fd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Unreleased ---------- +- ``table.transform()`` now preserves column-level and composite ``UNIQUE`` constraints, including constraint names, collations, sort order and ``ON CONFLICT`` behavior. Renaming columns updates those constraints, while dropping any constituent column removes the entire constraint. (:issue:`762`) - ``table.transform()`` now preserves ``AUTOINCREMENT`` primary keys and their sequence high-water marks. Previously a transform removed ``AUTOINCREMENT`` and could reuse deleted row IDs. (:issue:`602`) - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) - New ``sqlite_utils.ANY`` marker type for creating and introspecting SQLite ``ANY`` columns. The Python API and CLI can create, add and transform these columns, and ``table.transform()`` and ``table.extract()`` now preserve ``ANY`` columns and their values in ``STRICT`` tables. (:issue:`790`) diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py index d426286..7377f4f 100644 --- a/sqlite_utils/create_table_parser.py +++ b/sqlite_utils/create_table_parser.py @@ -32,6 +32,24 @@ class ColumnComments: after: str = "" +@dataclass(frozen=True) +class UniqueColumn: + name: str + collation: str = "" + order: str = "" + + +@dataclass +class Unique: + columns: tuple[UniqueColumn, ...] + name: str = "" + column: str = "" + conflict: str = "" + sql: str = field(default="", compare=False, repr=False) + start: int = field(default=-1, compare=False, repr=False) + end: int = field(default=-1, compare=False, repr=False) + + class ParseError(ValueError): pass @@ -592,6 +610,181 @@ def parse_autoincrement(create_sql: str) -> str | None: return None +_CONFLICT_ACTIONS = frozenset(("ROLLBACK", "ABORT", "FAIL", "IGNORE", "REPLACE")) + + +def _conflict_after(tokens: list[_Token], index: int) -> tuple[str, int]: + if index >= len(tokens) or not tokens[index].is_keyword("ON"): + return "", index + if index + 2 >= len(tokens) or not tokens[index + 1].is_keyword("CONFLICT"): + raise ParseError("ON after UNIQUE must be followed by CONFLICT and an action") + action = tokens[index + 2].text.upper() + if tokens[index + 2].kind != "word" or action not in _CONFLICT_ACTIONS: + raise ParseError("Invalid UNIQUE ON CONFLICT action") + return action, index + 3 + + +def _unique_columns( + item: str, tokens: list[_Token], open_index: int +) -> tuple[tuple[UniqueColumn, ...], int]: + close = _matching_paren(tokens, open_index) + inner = item[tokens[open_index].end : tokens[close].start] + columns: list[UniqueColumn] = [] + for raw_column in _split_ranges(inner, _lex(inner)): + column_tokens = _meaningful(_lex(raw_column)) + if not column_tokens or column_tokens[0].kind not in ( + "word", + "identifier", + "string", + ): + raise ParseError("UNIQUE constraint has an invalid column") + name = _unquote(column_tokens[0].text) + collation = "" + order = "" + index = 1 + if index < len(column_tokens) and column_tokens[index].is_keyword("COLLATE"): + if index + 1 >= len(column_tokens): + raise ParseError("COLLATE in UNIQUE constraint is missing its name") + collation = _unquote(column_tokens[index + 1].text) + index += 2 + if index < len(column_tokens) and ( + column_tokens[index].is_keyword("ASC") + or column_tokens[index].is_keyword("DESC") + ): + order = column_tokens[index].text.upper() + index += 1 + if index != len(column_tokens): + raise ParseError("UNIQUE constraint has an invalid indexed column") + columns.append(UniqueColumn(name, collation=collation, order=order)) + if not columns: + raise ParseError("UNIQUE constraint must include at least one column") + return tuple(columns), close + 1 + + +def _column_uniques( + item: str, tokens: list[_Token], column: str, base_offset: int +) -> list[Unique]: + uniques: list[Unique] = [] + collation = "" + collation_index = 1 + while collation_index < len(tokens): + token = tokens[collation_index] + if token.text == "(": + collation_index = _matching_paren(tokens, collation_index) + 1 + continue + if token.is_keyword("COLLATE"): + if collation_index + 1 >= len(tokens): + raise ParseError("COLLATE is missing its name") + collation = _unquote(tokens[collation_index + 1].text) + collation_index += 2 + continue + collation_index += 1 + pending_name = "" + pending_start: int | None = None + index = 1 + while index < len(tokens): + token = tokens[index] + if token.text == "(": + index = _matching_paren(tokens, index) + 1 + continue + if token.is_keyword("CONSTRAINT"): + if index + 1 >= len(tokens): + raise ParseError("CONSTRAINT is missing its name") + pending_name = _unquote(tokens[index + 1].text) + pending_start = index + index += 2 + continue + if token.is_keyword("UNIQUE"): + source_start = tokens[ + pending_start if pending_start is not None else index + ].start + conflict, next_index = _conflict_after(tokens, index + 1) + source_end = tokens[next_index - 1].end + uniques.append( + Unique( + (UniqueColumn(column, collation=collation),), + name=pending_name, + column=column, + conflict=conflict, + sql=item[source_start:source_end], + start=base_offset + source_start, + end=base_offset + source_end, + ) + ) + pending_name = "" + pending_start = None + index = next_index + continue + if ( + token.kind == "word" + and token.text.upper() in _OTHER_COLUMN_CONSTRAINT_KEYWORDS + ): + pending_name = "" + pending_start = None + index += 1 + return uniques + + +def parse_uniques(create_sql: str) -> list[Unique]: + """Return column-level and table-level UNIQUE constraints.""" + body_info = _table_body(create_sql) + if body_info is None: + return [] + body, body_start = body_info + uniques: list[Unique] = [] + for item, item_start, _ in _split_spans(body, _lex(body)): + item_tokens = _meaningful(_lex(item)) + if not item_tokens: + continue + item_index = 0 + constraint_name = "" + if item_tokens[item_index].is_keyword("CONSTRAINT"): + if len(item_tokens) < 2: + raise ParseError("CONSTRAINT is missing its name") + constraint_name = _unquote(item_tokens[1].text) + item_index = 2 + head = item_tokens[item_index] if item_index < len(item_tokens) else None + if head and head.is_keyword("UNIQUE"): + if ( + item_index + 1 >= len(item_tokens) + or item_tokens[item_index + 1].text != "(" + ): + raise ParseError("Table UNIQUE must be followed by a column list") + columns, next_index = _unique_columns(item, item_tokens, item_index + 1) + conflict, next_index = _conflict_after(item_tokens, next_index) + if next_index != len(item_tokens): + raise ParseError("Unexpected SQL after UNIQUE constraint") + source_start = item_tokens[0].start + source_end = item_tokens[next_index - 1].end + uniques.append( + Unique( + columns, + name=constraint_name, + conflict=conflict, + sql=item[source_start:source_end], + start=body_start + item_start + source_start, + end=body_start + item_start + source_end, + ) + ) + continue + if ( + head + and head.kind == "word" + and head.text.upper() in _TABLE_CONSTRAINT_KEYWORDS + ): + continue + column = _unquote(item_tokens[0].text) + uniques.extend( + _column_uniques( + item, + item_tokens, + column, + body_start + item_start, + ) + ) + return uniques + + def parse_column_comments(create_sql: str) -> dict[str, ColumnComments]: """Return comments immediately before and after each column definition.""" body_info = _table_body(create_sql) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 48c5d5d..48987a6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -32,10 +32,13 @@ from .create_table_parser import ( Check, ColumnComments, ParseError, + Unique, + UniqueColumn, check_references_identifier, parse_autoincrement, parse_checks, parse_column_comments, + parse_uniques, rewrite_check_expression, sql_ends_in_line_comment, ) @@ -104,6 +107,24 @@ def _check_constraint_sql(check: Check) -> str: return f"{prefix}CHECK ({check.check}{newline})" +def _unique_constraint_sql(unique: Unique) -> str: + prefix = f"CONSTRAINT {quote_identifier(unique.name)} " if unique.name else "" + if unique.column: + constraint = "UNIQUE" + else: + columns = [] + for column in unique.columns: + column_sql = quote_identifier(column.name) + if column.collation: + column_sql += f" COLLATE {quote_identifier(column.collation)}" + if column.order: + column_sql += f" {column.order}" + columns.append(column_sql) + constraint = "UNIQUE ({})".format(", ".join(columns)) + conflict = f" ON CONFLICT {unique.conflict}" if unique.conflict else "" + return f"{prefix}{constraint}{conflict}" + + def _column_definition_with_comments( definition: str, comments: ColumnComments | None ) -> str: @@ -1424,6 +1445,7 @@ class Database: _checks: Iterable[Check] | None = None, _column_comments: Mapping[str, ColumnComments] | None = None, _autoincrement: str | None = None, + _uniques: Iterable[Unique] | None = None, ) -> str: """ Returns the SQL ``CREATE TABLE`` statement for creating the specified table. @@ -1486,6 +1508,60 @@ class Database: checks_by_column.setdefault(column, []).append(check) else: table_checks.append(check) + uniques_by_column: dict[str, list[Unique]] = {} + table_uniques: list[Unique] = [] + for unique in _uniques or (): + resolved_unique = Unique( + tuple( + UniqueColumn( + resolve_casing(column.name, columns), + collation=column.collation, + order=column.order, + ) + for column in unique.columns + ), + name=unique.name, + column=( + resolve_casing(unique.column, columns) if unique.column else "" + ), + conflict=unique.conflict, + ) + missing = [ + column.name + for column in resolved_unique.columns + if column.name not in columns + ] + if missing: + raise AlterError( + "No such column for UNIQUE constraint: {}".format( + ", ".join(missing) + ) + ) + if resolved_unique.column: + if ( + len(resolved_unique.columns) != 1 + or resolved_unique.columns[0].name != resolved_unique.column + ): + raise AlterError("Invalid column-level UNIQUE constraint") + if any( + column.collation or column.order + for column in resolved_unique.columns + ): + # Render this as a table constraint so the collation or sort + # order that governs uniqueness can be represented explicitly. + table_uniques.append( + Unique( + resolved_unique.columns, + name=resolved_unique.name, + conflict=resolved_unique.conflict, + ) + ) + else: + uniques_by_column.setdefault(resolved_unique.column, []).append( + resolved_unique + ) + else: + table_uniques.append(resolved_unique) if not columns: raise ValueError("Tables must have at least one column") if not all(n in columns for n in not_null): @@ -1554,6 +1630,10 @@ class Database: column_extras.append( f"REFERENCES {quote_identifier(fk.other_table)}({quote_identifier(cast(str, fk.other_column))}){_fk_actions_sql(fk)}" ) + column_extras.extend( + _unique_constraint_sql(unique) + for unique in uniques_by_column.get(column_name, ()) + ) column_extras.extend( _check_constraint_sql(check) for check in checks_by_column.get(column_name, ()) @@ -1600,6 +1680,9 @@ class Database: actions=_fk_actions_sql(fk), ) ) + column_defs.extend( + f" {_unique_constraint_sql(unique)}" for unique in table_uniques + ) column_defs.extend( f" {_check_constraint_sql(check)}" for check in table_checks ) @@ -2763,6 +2846,7 @@ class Table(Queryable): existing_checks = self.checks existing_column_comments = parse_column_comments(self.schema) existing_autoincrement = parse_autoincrement(self.schema) + existing_uniques = parse_uniques(self.schema) except ParseError as ex: raise TransformError( f"Could not parse table schema for table {self.name!r}: {ex}" @@ -2789,6 +2873,37 @@ class Table(Queryable): ) ) + create_table_uniques: list[Unique] = [] + for unique in existing_uniques: + columns = tuple( + UniqueColumn( + resolve_casing(column.name, existing_columns), + collation=column.collation, + order=column.order, + ) + for column in unique.columns + ) + if any(column.name in drop for column in columns): + continue + owner = ( + resolve_casing(unique.column, existing_columns) if unique.column else "" + ) + create_table_uniques.append( + Unique( + tuple( + UniqueColumn( + rename.get(column.name) or column.name, + collation=column.collation, + order=column.order, + ) + for column in columns + ), + name=unique.name, + column=rename.get(owner) or owner, + conflict=unique.conflict, + ) + ) + create_table_column_comments: dict[str, ColumnComments] = {} for column, comments in existing_column_comments.items(): owner = resolve_casing(column, existing_columns) @@ -2974,6 +3089,7 @@ class Table(Queryable): _checks=create_table_checks, _column_comments=create_table_column_comments, _autoincrement=create_table_autoincrement, + _uniques=create_table_uniques, ).strip() ) @@ -3008,6 +3124,9 @@ class Table(Queryable): {"index_name": index.name}, ).fetchall()[0][0] if index_sql is None: + if index.origin == "u": + # UNIQUE constraints are reproduced in CREATE TABLE above. + continue raise TransformError( f"Index '{index.name}' on table '{self.name}' does not have a " "CREATE INDEX statement. You must manually drop this index prior to running this " diff --git a/tests/test_create_table_parser.py b/tests/test_create_table_parser.py index 54bf221..74a089c 100644 --- a/tests/test_create_table_parser.py +++ b/tests/test_create_table_parser.py @@ -8,9 +8,12 @@ from sqlite_utils.create_table_parser import ( Check, ColumnComments, ParseError, + Unique, + UniqueColumn, parse_autoincrement, parse_checks, parse_column_comments, + parse_uniques, ) @@ -148,6 +151,51 @@ def test_parse_autoincrement(sql, expected): assert parse_autoincrement(sql) == expected +def test_parse_column_and_table_uniques(): + sql = """ + CREATE TABLE memberships ( + email TEXT COLLATE RTRIM CONSTRAINT unique_email UNIQUE ON CONFLICT IGNORE, + account_id INTEGER, + CONSTRAINT unique_membership UNIQUE ( + account_id DESC, + email COLLATE NOCASE ASC + ) ON CONFLICT REPLACE + ) + """ + sqlite3.connect(":memory:").execute(sql) + assert parse_uniques(sql) == [ + Unique( + (UniqueColumn("email", collation="RTRIM"),), + name="unique_email", + column="email", + conflict="IGNORE", + ), + Unique( + ( + UniqueColumn("account_id", order="DESC"), + UniqueColumn("email", collation="NOCASE", order="ASC"), + ), + name="unique_membership", + conflict="REPLACE", + ), + ] + uniques = parse_uniques(sql) + assert uniques[0].sql == "CONSTRAINT unique_email UNIQUE ON CONFLICT IGNORE" + assert sql[uniques[1].start : uniques[1].end] == uniques[1].sql + + +def test_unique_like_text_in_comments_and_checks_is_ignored(): + sql = """ + CREATE TABLE t ( + value TEXT /* UNIQUE ON CONFLICT REPLACE */ + CHECK(value != 'UNIQUE(other)'), + other TEXT + ) + """ + sqlite3.connect(":memory:").execute(sql) + assert parse_uniques(sql) == [] + + comment_or_space = st.sampled_from( [ " ", diff --git a/tests/test_transform.py b/tests/test_transform.py index e6096cd..3be6c6f 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1033,24 +1033,78 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db): fresh_db.execute(""" CREATE TABLE dogs ( id INTEGER PRIMARY KEY, - name TEXT UNIQUE, + name TEXT UNIQUE ON CONFLICT IGNORE, age INTEGER ); """) dogs.insert({"id": 1, "name": "Cleo", "age": 5}) - # Attempt to transform the table without modifying 'name' - with pytest.raises(TransformError) as excinfo: - dogs.transform(types={"age": str}) + dogs.transform(types={"age": str}, rename={"name": "dog_name"}) + + assert 'dog_name" TEXT UNIQUE ON CONFLICT IGNORE' in dogs.schema + dogs.insert({"id": 2, "dog_name": "Cleo", "age": "6"}) + assert list(dogs.rows) == [{"id": 1, "dog_name": "Cleo", "age": "5"}] + + +def test_transform_preserves_composite_unique_constraint(fresh_db): + fresh_db.execute(""" + CREATE TABLE memberships ( + account_id INTEGER, + email TEXT, + note TEXT, + CONSTRAINT unique_membership + UNIQUE (account_id DESC, email COLLATE NOCASE) + ON CONFLICT ABORT + ) + """) + memberships = fresh_db.table("memberships") + memberships.insert({"account_id": 1, "email": "one@example.com", "note": "x"}) + + memberships.transform(rename={"account_id": "organization_id"}, types={"note": str}) assert ( - "Index 'sqlite_autoindex_dogs_1' on table 'dogs' does not have a CREATE INDEX statement." - in str(excinfo.value) - ) - assert ( - "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation." - in str(excinfo.value) + 'CONSTRAINT "unique_membership" UNIQUE ' + '("organization_id" DESC, "email" COLLATE "NOCASE") ON CONFLICT ABORT' + in memberships.schema ) + with pytest.raises(sqlite3.IntegrityError): + memberships.insert( + {"organization_id": 1, "email": "ONE@example.com", "note": "y"} + ) + + +def test_transform_preserves_column_unique_collation(fresh_db): + fresh_db.execute(""" + CREATE TABLE people ( + id INTEGER PRIMARY KEY, + name TEXT COLLATE NOCASE UNIQUE + ) + """) + people = fresh_db.table("people") + people.insert({"id": 1, "name": "Cleo"}) + + people.transform(rename={"name": "full_name"}) + + assert 'UNIQUE ("full_name" COLLATE "NOCASE")' in people.schema + with pytest.raises(sqlite3.IntegrityError): + people.insert({"id": 2, "full_name": "cleo"}) + + +def test_transform_drops_entire_composite_unique_constraint(fresh_db): + fresh_db.execute(""" + CREATE TABLE memberships ( + account_id INTEGER, + email TEXT, + UNIQUE (account_id, email) + ) + """) + memberships = fresh_db.table("memberships") + memberships.insert({"account_id": 1, "email": "one@example.com"}) + + memberships.transform(drop={"email"}) + + assert "UNIQUE" not in memberships.schema + memberships.insert({"account_id": 1}) def test_transform_preserves_autoincrement_and_sequence(fresh_db): From e4935e064407bc995f77795c025c33cef52d742e Mon Sep 17 00:00:00 2001 From: ikatyal2110 <134458944+ikatyal2110@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:56:51 -0500 Subject: [PATCH 29/33] transform: coerce empty strings to NULL when converting TEXT columns to numeric types (#805) * transform: coerce empty strings to NULL when converting TEXT columns to numeric types When a TEXT column is transformed to INTEGER, FLOAT, or REAL and a row contains an empty string, the empty string is now converted to NULL during the INSERT...SELECT copy, matching the expected behavior described in #488. Fixes #488 --- docs/changelog.rst | 1 + docs/cli.rst | 2 +- docs/python-api.rst | 2 ++ sqlite_utils/db.py | 21 ++++++++++++++++++++- tests/test_transform.py | 36 +++++++++++++++++++++++++++++++++--- 5 files changed, 57 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fe215fd..b3203ad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ Unreleased - ``sqlite-utils convert --dry-run`` now works for table and column names containing closing square brackets. (:issue:`829`) - ``table.indexes`` and ``table.xindexes`` now work for table, index and column names containing double quotes. This also fixes ``table.transform()`` for tables with those identifiers. Thanks, `nyxst4ck `__. (:issue:`824`, `#825 `__) - Improved type annotations throughout the package and added Pyright regression checks to CI. (:issue:`833`) +- Changing a ``TEXT`` column to ``INTEGER``, ``FLOAT`` or ``REAL`` using ``table.transform()`` or ``sqlite-utils transform`` now converts exact empty strings to ``NULL``. Previously they remained empty strings in the numeric column. Thanks, `ikatyal2110 `__. (:issue:`488`, `#805 `__) .. _v3_39_1: diff --git a/docs/cli.rst b/docs/cli.rst index 417911a..78c33b8 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2236,7 +2236,7 @@ The ``transform`` command allows you to apply complex transformations to a table Every option for this table (with the exception of ``--pk-none``) can be specified multiple times. The options are as follows: ``--type column-name new-type`` - Change the type of the specified column. Valid types are ``integer``, ``text``, ``float``, ``real``, ``blob`` and ``any``. + Change the type of the specified column. Valid types are ``integer``, ``text``, ``float``, ``real``, ``blob`` and ``any``. Changing a ``TEXT`` column to ``INTEGER``, ``FLOAT`` or ``REAL`` converts exact empty-string values to ``NULL``. ``--drop column-name`` Drop the specified column. diff --git a/docs/python-api.rst b/docs/python-api.rst index 88cc3e3..d515642 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1826,6 +1826,8 @@ To alter the type of a column, use the ``types=`` argument: # Convert the 'age' column to an integer, and 'weight' to a float table.transform(types={"age": int, "weight": float}) +When a ``TEXT`` column is changed to ``INTEGER``, ``FLOAT`` or ``REAL``, exact empty-string values are stored as ``NULL``. Other values, including whitespace-only strings, are copied normally. + See :ref:`python_api_add_column` for a list of available types. .. _python_api_transform_strict: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 48987a6..37825bb 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3093,6 +3093,19 @@ class Table(Queryable): ).strip() ) + # Columns being changed from TEXT to a numeric type: coerce empty strings to NULL + _numeric_sql_types = {"INTEGER", "REAL", "FLOAT", "NUMERIC"} + text_to_numeric_cols = { + col_name + for col_name, new_type in types.items() + if existing_columns.get(col_name) == str + and COLUMN_TYPE_MAPPING.get( + new_type, + new_type.upper() if isinstance(new_type, str) else "", + ) + in _numeric_sql_types + } + # Copy across data, respecting any renamed columns new_cols = [] old_cols = [] @@ -3103,10 +3116,16 @@ class Table(Queryable): if "rowid" not in new_cols: new_cols.insert(0, "rowid") old_cols.insert(0, "rowid") + + def _copy_expr(col): + if col in text_to_numeric_cols: + return "NULLIF({}, '')".format(quote_identifier(col)) + return quote_identifier(col) + copy_sql = "INSERT INTO {} ({new_cols})\n SELECT {old_cols} FROM {};".format( quote_identifier(new_table_name), quote_identifier(self.name), - old_cols=", ".join(quote_identifier(col) for col in old_cols), + old_cols=", ".join(_copy_expr(col) for col in old_cols), new_cols=", ".join(quote_identifier(col) for col in new_cols), ) sqls.append(copy_sql) diff --git a/tests/test_transform.py b/tests/test_transform.py index 3be6c6f..8738713 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -27,7 +27,7 @@ from sqlite_utils.utils import OperationalError {"types": {"age": int}}, [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER\n);', - 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";', 'DROP TABLE "dogs";', "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', @@ -63,7 +63,7 @@ from sqlite_utils.utils import OperationalError {"types": {"age": int}, "rename": {"age": "dog_age"}}, [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" INTEGER\n);', - 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";', 'DROP TABLE "dogs";', "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', @@ -168,7 +168,7 @@ def test_transform_sql_table_with_primary_key( {"types": {"age": int}}, [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" INTEGER\n);', - 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";', 'DROP TABLE "dogs";', "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', @@ -1125,6 +1125,36 @@ def test_transform_preserves_autoincrement_and_sequence(fresh_db): ] +@pytest.mark.parametrize( + "new_type,expected_value,expected_type", + [ + (int, 42, int), + (float, 42.0, float), + ("integer", 42, int), + ("float", 42.0, float), + ("REAL", 42.0, float), + ], +) +def test_transform_empty_string_to_null_for_numeric_types( + fresh_db, new_type, expected_value, expected_type +): + fresh_db["test"].insert_all( + [ + {"id": 1, "value": "42"}, + {"id": 2, "value": ""}, + {"id": 3, "value": None}, + {"id": 4, "value": " "}, + ] + ) + fresh_db["test"].transform(types={"value": new_type}) + rows = {r["id"]: r["value"] for r in fresh_db["test"].rows} + assert rows[1] == expected_value + assert type(rows[1]) is expected_type + assert rows[2] is None + assert rows[3] is None + assert rows[4] == " " + + def test_transform_preserves_view(fresh_db): # https://github.com/simonw/sqlite-utils/issues/831 dogs = fresh_db.table("dogs") From 1d98613f28b8edab5fd0deb5ba65d54fb286e7ba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Aug 2026 13:09:42 -0700 Subject: [PATCH 30/33] Release 4.2 Refs #488, #602, #762, #790, #805, #808, #811, #816, #821, #822, #824, #825, #828, #829, #831, #833, #834, #836, #837 --- docs/changelog.rst | 22 +++++++++++++--------- pyproject.toml | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b3203ad..31bd961 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,19 +4,13 @@ Changelog =========== -.. _unreleased: +.. _v4_2: -Unreleased ----------- +4.2 (2026-08-13) +---------------- -- ``table.transform()`` now preserves column-level and composite ``UNIQUE`` constraints, including constraint names, collations, sort order and ``ON CONFLICT`` behavior. Renaming columns updates those constraints, while dropping any constituent column removes the entire constraint. (:issue:`762`) -- ``table.transform()`` now preserves ``AUTOINCREMENT`` primary keys and their sequence high-water marks. Previously a transform removed ``AUTOINCREMENT`` and could reuse deleted row IDs. (:issue:`602`) - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) - New ``sqlite_utils.ANY`` marker type for creating and introspecting SQLite ``ANY`` columns. The Python API and CLI can create, add and transform these columns, and ``table.transform()`` and ``table.extract()`` now preserve ``ANY`` columns and their values in ``STRICT`` tables. (:issue:`790`) -- ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) -- ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) -- ``table.transform(rename=...)`` now preserves explicit indexes on renamed columns by dropping and recreating those indexes against the new column names. Previously this raised a ``TransformError``. (:issue:`822`) -- ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) - ``table.default_values`` now unescapes doubled single quotes in string defaults, so a default such as ``'O''Brien'`` is returned as ``"O'Brien"``. Thanks, `ikatyal2110 `__. (`#811 `__) - ``table.default_values`` now decodes unquoted ``TRUE``, ``FALSE`` and ``NULL`` default literals as ``True``, ``False`` and ``None`` respectively. (:issue:`836`) - ``table.enable_fts(..., tokenize=...)`` and ``sqlite-utils enable-fts --tokenize`` now safely quote the tokenizer argument, preventing a crafted value from injecting additional SQL. Thanks, `Bunlong Heng `__. (`#828 `__) @@ -27,6 +21,16 @@ Unreleased - Improved type annotations throughout the package and added Pyright regression checks to CI. (:issue:`833`) - Changing a ``TEXT`` column to ``INTEGER``, ``FLOAT`` or ``REAL`` using ``table.transform()`` or ``sqlite-utils transform`` now converts exact empty strings to ``NULL``. Previously they remained empty strings in the numeric column. Thanks, `ikatyal2110 `__. (:issue:`488`, `#805 `__) +``table.transform()`` can handle many more edge-cases: + +- ``table.transform()`` now preserves column-level and composite ``UNIQUE`` constraints, including constraint names, collations, sort order and ``ON CONFLICT`` behavior. Renaming columns updates those constraints, while dropping any constituent column removes the entire constraint. (:issue:`762`) +- ``table.transform()`` now preserves ``AUTOINCREMENT`` primary keys and their sequence high-water marks. Previously a transform removed ``AUTOINCREMENT`` and could reuse deleted row IDs. (:issue:`602`) +- ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) +- ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) +- ``table.transform(rename=...)`` now preserves explicit indexes on renamed columns by dropping and recreating those indexes against the new column names. Previously this raised a ``TransformError``. (:issue:`822`) +- ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) + + .. _v3_39_1: 3.39.1 (2026-07-25) diff --git a/pyproject.toml b/pyproject.toml index 9b4d6f5..6dac11c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.1.1" +version = "4.2" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From f6d73112c8368cd6eb2ac596966e8148747c7b4e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Aug 2026 16:52:03 -0700 Subject: [PATCH 31/33] Fix for sqlite-utils 4.2 crashing bug (#843) - Remove from typing_extensions import Self - Smoke test: uv run --no-default-groups sqlite-utils --help Closes #842 --- .github/workflows/test.yml | 5 +++++ sqlite_utils/db.py | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6c720a1..5924fd8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,11 @@ jobs: run: | pip install uv uv run ty check sqlite_utils + - name: Check no accidental dev= dependencies needed + if: matrix.os == 'ubuntu-latest' + run: | + pip install uv + uv run --no-default-groups sqlite-utils --help - name: Check formatting run: black . --check - name: Check if cog needs to be run diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 37825bb..c011d9b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -24,8 +24,6 @@ from typing import ( ) from sqlite_fts4 import rank_bm25 -from typing_extensions import Self - from sqlite_utils.plugins import ensure_plugins_loaded, pm from .create_table_parser import ( @@ -637,7 +635,7 @@ class Database: pm.hook.prepare_connection(conn=self.conn) self.strict = strict - def __enter__(self) -> Self: + def __enter__(self): return self def __exit__( From 28dc6278cc03a9245325d056e6986818544abc68 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Aug 2026 16:52:30 -0700 Subject: [PATCH 32/33] Release 4.2.1 Refs #842, #843 --- docs/changelog.rst | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 31bd961..5d024e1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _v4_2_1: + +4.2.1 (2026-08-13) +------------------ + +- Fix for ``No module named 'typing_extensions'`` crashing bug accidentally shipped in version 4.2. (:issue:`842`) + .. _v4_2: 4.2 (2026-08-13) diff --git a/pyproject.toml b/pyproject.toml index 6dac11c..92650e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.2" +version = "4.2.1" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 56dd09702fdb9e899f577ffd51693c1f2176cb08 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Aug 2026 17:01:47 -0700 Subject: [PATCH 33/33] Run no-default-groups smoke test from Justfile Refs #842 I had to add --isolated because otherwise this test would pass if a .venv folder already existed with the dev dependencies installed in it. --- Justfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index e93075f..7347534 100644 --- a/Justfile +++ b/Justfile @@ -2,9 +2,12 @@ @default: test lint # Run pytest with supplied options -@test *options: +@test *options: test-no-dev-dependencies uv run pytest {{options}} +@test-no-dev-dependencies: + uv run --isolated --no-default-groups sqlite-utils --help > /dev/null + @run *options: uv run -- {{options}}