From 87cf1c5a007ff724ed803111898d9658d53e320a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 11:49:59 -0700 Subject: [PATCH 001/110] Replace SQL prefix matching with a first-token scanner query() and execute() classified statements using sql.lstrip().upper().startswith(...), which a leading SQL comment defeated: db.query('/* c */ COMMIT') inside db.atomic() committed the caller's transaction and masked the ValueError with a 'no such savepoint' error, comment-prefixed PRAGMAs ran inside the savepoint guard where journal mode changes are refused, and a comment-prefixed BEGIN passed to db.execute() was instantly auto-committed. The new _first_keyword() helper skips leading whitespace and -- or /* */ comments - the only things SQLite's tokenizer allows before the first token - and returns that token for exact comparison, so keyword detection now matches what SQLite itself will see. Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 51 +++++++++++++++++++++++++++++------ tests/test_atomic.py | 11 ++++++++ tests/test_query.py | 63 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 116 insertions(+), 9 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8e6a194..f796a63 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -324,14 +324,51 @@ CREATE TABLE IF NOT EXISTS "{}"( """.strip() -_TRANSACTION_CONTROL_PREFIXES = ( +_TRANSACTION_CONTROL_KEYWORDS = { "BEGIN", "COMMIT", "END", "ROLLBACK", "SAVEPOINT", "RELEASE", -) +} + +# Statements that never return rows and cannot run inside (or would break +# out of) the savepoint guard used by query() +_QUERY_REJECTED_KEYWORDS = _TRANSACTION_CONTROL_KEYWORDS | { + "VACUUM", + "ATTACH", + "DETACH", +} + + +def _first_keyword(sql: str) -> str: + """ + Return the first keyword of a SQL statement, uppercased, skipping any + leading whitespace and ``--`` or ``/* ... */`` comments - the only + things SQLite's tokenizer allows before the first token. Returns an + empty string if there is no leading keyword. + """ + i, n = 0, len(sql) + while i < n: + if sql[i].isspace(): + i += 1 + elif sql.startswith("--", i): + newline = sql.find("\n", i) + if newline == -1: + return "" + i = newline + 1 + elif sql.startswith("/*", i): + end = sql.find("*/", i + 2) + if end == -1: + return "" + i = end + 2 + else: + break + j = i + while j < n and (sql[j].isalpha() or sql[j] == "_"): + j += 1 + return sql[i:j].upper() class Database: @@ -667,16 +704,14 @@ class Database: "query() can only be used with SQL that returns rows - " "use execute() for other statements" ) - prefix = sql.lstrip().upper() - if prefix.startswith( - _TRANSACTION_CONTROL_PREFIXES + ("VACUUM", "ATTACH", "DETACH") - ): + keyword = _first_keyword(sql) + if keyword in _QUERY_REJECTED_KEYWORDS: # None of these return rows - reject them without executing anything raise ValueError(message) if self._tracer: self._tracer(sql, params) args: tuple = (params,) if params is not None else () - if prefix.startswith("PRAGMA"): + if keyword == "PRAGMA": # Some PRAGMA statements refuse to run inside a transaction, so # execute these without the savepoint guard used below. PRAGMAs # never open an implicit transaction, so there is nothing to @@ -741,7 +776,7 @@ class Database: not was_in_transaction and self.conn.in_transaction and cursor.description is None - and not sql.lstrip().upper().startswith(_TRANSACTION_CONTROL_PREFIXES) + and _first_keyword(sql) not in _TRANSACTION_CONTROL_KEYWORDS ): # The statement opened an implicit transaction - commit it, so # that execute() behaves consistently with the rest of the diff --git a/tests/test_atomic.py b/tests/test_atomic.py index a8e3563..f75c4ee 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -247,6 +247,17 @@ def test_execute_write_respects_explicit_transaction(fresh_db): assert [r["id"] for r in fresh_db["t"].rows] == [1] +def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): + # A BEGIN hidden behind a leading comment must not be auto-committed + # out from under the caller + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.execute("-- start a transaction\nbegin") + assert fresh_db.conn.in_transaction + fresh_db.execute("insert into t (id) values (2)") + fresh_db.rollback() + assert [r["id"] for r in fresh_db["t"].rows] == [1] + + def test_query_returning_commits_after_iteration(tmpdir): if sqlite3.sqlite_version_info < (3, 35, 0): import pytest as _pytest diff --git a/tests/test_query.py b/tests/test_query.py index cf8d9f5..1d5c1f6 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -48,7 +48,18 @@ def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db): @pytest.mark.parametrize( - "sql", ["begin", "commit", "rollback", "vacuum", "detach database foo"] + "sql", + [ + "begin", + "commit", + "rollback", + "vacuum", + "detach database foo", + "/* comment */ commit", + "-- comment\nbegin", + "/* multi\nline */ -- and another\n vacuum", + "\t /* a */ /* b */ savepoint s1", + ], ) def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql): with pytest.raises(ValueError) as ex: @@ -57,6 +68,21 @@ def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql): assert not fresh_db.conn.in_transaction +def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db): + # A COMMIT hidden behind a leading comment must not slip past the + # keyword check - previously it committed the caller's open + # transaction before the ValueError was raised + fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.begin() + fresh_db.execute("insert into dogs (name) values ('Pancakes')") + with pytest.raises(ValueError): + fresh_db.query("/* comment */ COMMIT") + # The explicit transaction is still open and can still be rolled back + assert fresh_db.conn.in_transaction + fresh_db.rollback() + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + + def test_query_error_leaves_no_transaction_open(fresh_db): with pytest.raises(sqlite3.OperationalError): fresh_db.query("select * from missing_table") @@ -75,6 +101,41 @@ def test_query_pragma(tmpdir): db.close() +def test_query_comment_prefixed_pragma(tmpdir): + from sqlite_utils import Database + + db = Database(str(tmpdir / "test.db")) + # A leading comment must not stop a PRAGMA being recognized as one - + # previously it was executed inside the savepoint guard, where + # journal mode changes are refused + assert list(db.query("-- set WAL mode\npragma journal_mode = wal")) == [ + {"journal_mode": "wal"} + ] + db.close() + + +@pytest.mark.parametrize( + "sql,expected", + [ + ("select 1", "SELECT"), + (" \t\n select 1", "SELECT"), + ("-- comment\nbegin", "BEGIN"), + ("/* one */ /* two */ pragma user_version", "PRAGMA"), + ("/* multi\nline */vacuum", "VACUUM"), + ("insert into t values (1)", "INSERT"), + ("-- only a comment", ""), + ("/* unterminated", ""), + ("", ""), + (" ", ""), + ("123", ""), + ], +) +def test_first_keyword(sql, expected): + from sqlite_utils.db import _first_keyword + + assert _first_keyword(sql) == expected + + @pytest.mark.skipif( sqlite3.sqlite_version_info < (3, 35, 0), reason="RETURNING requires SQLite 3.35.0 or higher", From 61619498fa730b4bdaa6f4891465840ad177f2f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 11:50:10 -0700 Subject: [PATCH 002/110] Give db.query() SQL restrictions top billing in upgrading docs Rejecting statements that return no rows is the change most likely to require code edits when upgrading to 4.0, so it now leads the Python API changes section - before db.table() - with the full list of rejected statement types and a before/after example. Co-Authored-By: Claude Fable 5 --- docs/upgrading.rst | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 1974ad6..f743a3e 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -46,6 +46,18 @@ Two related things have been removed: Python API changes ------------------ +**db.query() now rejects SQL that does not return rows.** This is likely the most common change you will need to make to existing code. ``db.query()`` used to accept any SQL statement - passing one that returns no rows, such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause or a ``CREATE TABLE``, did nothing at all, silently. Those statements now raise a ``ValueError``, and are rolled back so they have no effect on the database. Transaction control statements (``BEGIN``, ``COMMIT``, ``END``, ``ROLLBACK``, ``SAVEPOINT``, ``RELEASE``) plus ``VACUUM``, ``ATTACH`` and ``DETACH`` are also rejected with a ``ValueError``, without being executed at all. Use ``db.execute()`` for statements that do not return rows: + +.. code-block:: python + + # 3.x accepted this but silently did nothing: + db.query("update dogs set name = 'Cleopaws'") + + # In 4.0 use execute() for SQL that does not return rows: + db.execute("update dogs set name = 'Cleopaws'") + +**db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily, but errors in your SQL raise at the ``db.query()`` call site rather than on first iteration, and a write with a ``RETURNING`` clause takes effect even if you never iterate over its results. + **db.table() no longer returns views.** ``db.table(name)`` now raises a ``sqlite_utils.db.NoTable`` exception if ``name`` is a SQL view. Use the new ``db.view(name)`` method for views: .. code-block:: python @@ -55,11 +67,6 @@ Python API changes ``db["name"]`` still returns either a ``Table`` or a ``View`` depending on what exists in the database. -**db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily. Two consequences: - -- Errors in your SQL now raise at the ``db.query()`` call site rather than on first iteration. -- Passing a statement that returns no rows - such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause - previously did nothing at all, silently. It now raises a ``ValueError``, and the statement is rolled back so it has no effect on the database. Use ``db.execute()`` for statements that do not return rows. - **Upserts use INSERT ... ON CONFLICT.** Upsert operations now use SQLite's ``INSERT ... ON CONFLICT SET`` syntax rather than the previous ``INSERT OR IGNORE`` followed by ``UPDATE``. If your code depends on the old behavior, pass ``use_old_upsert=True`` to the ``Database()`` constructor - see :ref:`python_api_old_upsert`. **Upsert records must include their primary keys.** ``table.upsert()`` and ``table.upsert_all()`` now raise ``sqlite_utils.db.PrimaryKeyRequired`` if a record is missing a value for any primary key column (or has ``None`` for one). Previously such records were quietly inserted as new rows. Relatedly, ``pk=`` is now optional when the table already exists with a primary key - it is detected automatically. From 623331b3f4ee78e1ed2618ccfc2e18e0bd77cd36 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 12:16:11 -0700 Subject: [PATCH 003/110] Fix for test failure against sqlean --- sqlite_utils/db.py | 12 ++++++++---- tests/test_query.py | 9 +++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f796a63..a8f8e17 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -713,10 +713,14 @@ class Database: args: tuple = (params,) if params is not None else () if keyword == "PRAGMA": # Some PRAGMA statements refuse to run inside a transaction, so - # execute these without the savepoint guard used below. PRAGMAs - # never open an implicit transaction, so there is nothing to - # undo if this one turns out not to return rows - cursor = self.conn.execute(sql, *args) + # execute these without the savepoint guard used below. Some + # adapters open an implicit transaction before comment-prefixed + # PRAGMAs, so temporarily use driver autocommit when it is safe. + if self.conn.in_transaction: + cursor = self.conn.execute(sql, *args) + else: + with self.ensure_autocommit_off(): + cursor = self.conn.execute(sql, *args) if cursor.description is None: raise ValueError(message) keys = [d[0] for d in cursor.description] diff --git a/tests/test_query.py b/tests/test_query.py index 1d5c1f6..b9822e1 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -114,6 +114,15 @@ def test_query_comment_prefixed_pragma(tmpdir): db.close() +def test_query_comment_prefixed_pragma_inside_transaction(fresh_db): + fresh_db.begin() + assert list(fresh_db.query("-- check version\npragma user_version")) == [ + {"user_version": 0} + ] + assert fresh_db.conn.in_transaction + fresh_db.rollback() + + @pytest.mark.parametrize( "sql,expected", [ From f10459cffbdaaada4a61b6b1b1b4bdb59041e88d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 13:59:25 -0700 Subject: [PATCH 004/110] table.foreign_keys now handles compound foreign keys, refs #594 This is a breaking change, since the return type is now a dataclass and not a namedtuple. --- docs/changelog.rst | 9 +++ docs/python-api.rst | 32 +++++++-- docs/upgrading.rst | 16 +++++ sqlite_utils/db.py | 129 ++++++++++++++++++++++++++++--------- tests/test_foreign_keys.py | 82 +++++++++++++++++++++++ 5 files changed, 232 insertions(+), 36 deletions(-) create mode 100644 tests/test_foreign_keys.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 2e1f69c..01d59a0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog =========== +.. _v_unreleased: + +Unreleased +---------- + +Breaking changes: + +- ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` lists, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) + .. _v4_0rc2: 4.0rc2 (2026-07-04) diff --git a/docs/python-api.rst b/docs/python-api.rst index 3f7e12c..d1f466f 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2204,17 +2204,35 @@ Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly .foreign_keys ------------- -The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey(table, column, other_table, other_column)`` named tuples. It is not available on views. +The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey`` objects. It is not available on views. + +Each ``ForeignKey`` has the following attributes: + +``table`` + The table the foreign key is defined on. +``column`` + The column on this table, or ``None`` for a compound foreign key. +``other_table`` + The table being referenced. +``other_column`` + The referenced column, or ``None`` for a compound foreign key. +``columns`` + A list of the columns on this table, always populated (a single-item list for single-column foreign keys). +``other_columns`` + A list of the referenced columns. +``is_compound`` + ``True`` if this is a compound (multi-column) foreign key. + +``ForeignKey`` was a ``namedtuple`` prior to sqlite-utils 4.0. It is now a dataclass and can no longer be unpacked or indexed as a tuple - access its fields by name instead. See :ref:`upgrading_3_to_4` for details. :: >>> db.table("Street_Tree_List").foreign_keys - [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qSiteInfo', other_table='qSiteInfo', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qSpecies', other_table='qSpecies', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qCaretaker', other_table='qCaretaker', other_column='id'), - ForeignKey(table='Street_Tree_List', column='PlantType', other_table='PlantType', other_column='id')] + [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=['qLegalStatus'], other_columns=['id'], is_compound=False), + ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=['qCareAssistant'], other_columns=['id'], is_compound=False), + ...] + +Compound foreign keys - defined with ``FOREIGN KEY (col_a, col_b) REFERENCES other(col_a, col_b)`` - are returned as a single ``ForeignKey`` with ``is_compound=True``, ``column`` and ``other_column`` set to ``None``, and the participating columns available in the ``columns`` and ``other_columns`` lists. .. _python_api_introspection_schema: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index f743a3e..eb997f6 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -79,6 +79,22 @@ Python API changes **View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method. +**ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained three new fields - ``columns``, ``other_columns`` and ``is_compound`` - so that compound (multi-column) foreign keys can be represented as a single object. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead: + +.. code-block:: python + + # 3.x - tuple unpacking, no longer works: + for table, column, other_table, other_column in db["courses"].foreign_keys: + ... + + # 4.0 - access fields by name: + for fk in db["courses"].foreign_keys: + fk.table, fk.column, fk.other_table, fk.other_column + +Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. + +Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` lists instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are single-item lists. + **Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``. **Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a8f8e17..b1cdf83 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -11,6 +11,7 @@ from .utils import ( ) import binascii from collections import namedtuple +from dataclasses import dataclass, field from collections.abc import Mapping import contextlib import datetime @@ -161,9 +162,43 @@ Summary information about a column, see :ref:`python_api_analyze_column`. The ``N`` least common values as a list of ``(value, count)`` tuples, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) """ -ForeignKey = namedtuple( - "ForeignKey", ("table", "column", "other_table", "other_column") -) + + +@dataclass(order=True) +class ForeignKey: + """ + A foreign key defined on a table. + + For single-column foreign keys ``column`` and ``other_column`` hold the + column names, and ``columns``/``other_columns`` are single-item lists. + + For compound (multi-column) foreign keys ``column`` and ``other_column`` + are ``None`` - use ``columns`` and ``other_columns`` instead, and check + ``is_compound``. + + Prior to sqlite-utils 4.0 this was a ``namedtuple`` and could be unpacked + or indexed as ``(table, column, other_table, other_column)``. It is now a + dataclass - access its fields by name instead. + """ + + table: str + column: Optional[str] + other_table: str + other_column: Optional[str] + columns: List[str] = field(default_factory=list) + other_columns: List[str] = field(default_factory=list) + is_compound: bool = False + + def __post_init__(self): + # Populate columns/other_columns for single-column foreign keys + if not self.columns: + self.columns = [self.column] if self.column is not None else [] + if not self.other_columns: + self.other_columns = ( + [self.other_column] if self.other_column is not None else [] + ) + + Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns")) XIndex = namedtuple("XIndex", ("name", "columns")) XIndexColumn = namedtuple( @@ -1124,7 +1159,7 @@ class Database: if not all(isinstance(fk, (tuple, list)) for fk in foreign_keys): raise ValueError("foreign_keys= should be a list of tuples") fks = [] - for tuple_or_list in foreign_keys: + for tuple_or_list in cast(Iterable[Sequence[str]], foreign_keys): if len(tuple_or_list) == 4: if tuple_or_list[0] != name: raise ValueError( @@ -1234,7 +1269,7 @@ class Database: pk = hash_id # Soundness check foreign_keys point to existing tables for fk in foreign_keys: - if fk.other_table == name and columns.get(fk.other_column): + if fk.other_table == name and columns.get(cast(str, fk.other_column)): continue if fk.other_column != "rowid" and not any( c for c in self[fk.other_table].columns if c.name == fk.other_column @@ -1269,7 +1304,7 @@ class Database: foreign_keys_by_column[column_name].other_table ), quote_identifier( - foreign_keys_by_column[column_name].other_column + cast(str, foreign_keys_by_column[column_name].other_column) ), ) ) @@ -1499,7 +1534,9 @@ class Database: """ # foreign_keys is a list of explicit 4-tuples if not all( - len(fk) == 4 and isinstance(fk, (list, tuple)) for fk in foreign_keys + isinstance(fk, ForeignKey) + or (isinstance(fk, (list, tuple)) and len(fk) == 4) + for fk in foreign_keys ): raise ValueError( "foreign_keys must be a list of 4-tuples, " @@ -1509,7 +1546,16 @@ class Database: foreign_keys_to_create = [] # Verify that all tables and columns exist - for table, column, other_table, other_column in foreign_keys: + for fk in foreign_keys: + if isinstance(fk, ForeignKey): + table, column, other_table, other_column = ( + fk.table, + fk.column, + fk.other_table, + fk.other_column, + ) + else: + table, column, other_table, other_column = fk if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) table_obj = self.table(table) @@ -1555,8 +1601,11 @@ class Database: i.columns[0] for i in table.indexes if len(i.columns) == 1 } for fk in table.foreign_keys: - if fk.column not in existing_indexes: - table.create_index([fk.column], find_unique_name=True) + # Compound foreign keys expose their columns via fk.columns; + # single-column keys yield a one-item list + for column in fk.columns: + if column not in existing_indexes: + table.create_index([column], find_unique_name=True) def vacuum(self) -> None: "Run a SQLite ``VACUUM`` against the database." @@ -1907,21 +1956,40 @@ class Table(Queryable): @property def foreign_keys(self) -> List["ForeignKey"]: - "List of foreign keys defined on this table." - fks = [] + """ + List of foreign keys defined on this table. + + Compound (multi-column) foreign keys are returned as a single + ``ForeignKey`` with ``is_compound=True`` and populated + ``columns``/``other_columns`` lists. + """ + # PRAGMA foreign_key_list returns one row per column, grouped by "id" + # with "seq" giving the column order within a compound foreign key. + by_id: Dict[int, list] = {} for row in self.db.execute( "PRAGMA foreign_key_list({})".format(quote_identifier(self.name)) ).fetchall(): if row is not None: id, seq, table_name, from_, to_, on_update, on_delete, match = row - fks.append( - ForeignKey( - table=self.name, - column=from_, - other_table=table_name, - other_column=to_, - ) + by_id.setdefault(id, []).append((seq, table_name, from_, to_)) + fks = [] + for id in sorted(by_id): + rows = sorted(by_id[id]) # order columns by seq + other_table = rows[0][1] + columns = [row[2] for row in rows] + other_columns = [row[3] for row in rows] + is_compound = len(rows) > 1 + fks.append( + ForeignKey( + table=self.name, + column=None if is_compound else columns[0], + other_table=other_table, + other_column=None if is_compound else other_columns[0], + columns=columns, + other_columns=other_columns, + is_compound=is_compound, ) + ) return fks @property @@ -2254,17 +2322,20 @@ class Table(Queryable): else: # Construct foreign_keys from current, plus add_foreign_keys, minus drop_foreign_keys create_table_foreign_keys = [] - for table, column, other_table, other_column in self.foreign_keys: - # Copy over old foreign keys, unless we are dropping them - if (drop_foreign_keys is None) or (column not in drop_foreign_keys): - create_table_foreign_keys.append( - ForeignKey( - table, - rename.get(column) or column, - other_table, - other_column, + for fk in self.foreign_keys: + # Expand compound foreign keys into per-column references; + # for single-column keys this iterates exactly once + for column, other_column in zip(fk.columns, fk.other_columns): + # Copy over old foreign keys, unless we are dropping them + if (drop_foreign_keys is None) or (column not in drop_foreign_keys): + create_table_foreign_keys.append( + ForeignKey( + fk.table, + rename.get(column) or column, + fk.other_table, + other_column, + ) ) - ) # Add new foreign keys if add_foreign_keys is not None: for fk in self.db.resolve_foreign_keys(self.name, add_foreign_keys): diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py new file mode 100644 index 0000000..c785b59 --- /dev/null +++ b/tests/test_foreign_keys.py @@ -0,0 +1,82 @@ +"""Tests for reading compound (multi-column) foreign keys - issue #594.""" + +import pytest +from sqlite_utils import Database + +COMPOUND_SCHEMA = """ +CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + dept_name TEXT, + PRIMARY KEY (campus_name, dept_code) +); +CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + course_name TEXT, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + FOREIGN KEY (campus_name, dept_code) + REFERENCES departments(campus_name, dept_code) +); +""" + + +@pytest.fixture +def compound_db(): + db = Database(memory=True) + db.executescript(COMPOUND_SCHEMA) + return db + + +def test_compound_foreign_key(compound_db): + fks = compound_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.table == "courses" + assert fk.other_table == "departments" + assert fk.columns == ["campus_name", "dept_code"] + assert fk.other_columns == ["campus_name", "dept_code"] + # Scalar column/other_column can't sensibly hold a compound key + assert fk.column is None + assert fk.other_column is None + + +def test_single_foreign_key_gets_columns_fields(fresh_db): + fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) + fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fk = fresh_db["books"].foreign_keys[0] + assert fk.is_compound is False + assert fk.column == "author_id" + assert fk.other_column == "id" + assert fk.columns == ["author_id"] + assert fk.other_columns == ["id"] + + +def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): + # Clean break in 4.0: ForeignKey is a dataclass, not a namedtuple, so the + # old tuple unpacking and indexing patterns now fail hard. + fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) + fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fk = fresh_db["books"].foreign_keys[0] + with pytest.raises(TypeError): + table, column, other_table, other_column = fk + with pytest.raises(TypeError): + fk[0] + + +def test_foreign_keys_are_sortable(fresh_db): + fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db["categories"].insert({"id": 1, "name": "Wildlife"}, pk="id") + fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1, "category_id": 1}) + fresh_db.add_foreign_keys( + [ + ("books", "author_id", "authors", "id"), + ("books", "category_id", "categories", "id"), + ] + ) + fks = sorted(fresh_db["books"].foreign_keys) + assert fks[0].column == "author_id" + assert fks[1].column == "category_id" From d5bf51df35a527aae19417c97eb3949ffb6df5bc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:26:56 -0700 Subject: [PATCH 005/110] Fix TypeError when sorting mixed compound/single foreign keys sorted() on a foreign_keys list containing both compound (column=None) and single-column ForeignKeys raised TypeError because dataclass ordering compared None against str. column/other_column are now excluded from comparison - ordering and equality use the always populated columns/other_columns lists instead. Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 6 ++++-- tests/test_foreign_keys.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b1cdf83..67623fe 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -182,9 +182,11 @@ class ForeignKey: """ table: str - column: Optional[str] + # column/other_column are None for compound keys, which would break + # ordering against str values - comparison uses columns/other_columns + column: Optional[str] = field(compare=False) other_table: str - other_column: Optional[str] + other_column: Optional[str] = field(compare=False) columns: List[str] = field(default_factory=list) other_columns: List[str] = field(default_factory=list) is_compound: bool = False diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index c785b59..4b371a9 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -80,3 +80,31 @@ def test_foreign_keys_are_sortable(fresh_db): fks = sorted(fresh_db["books"].foreign_keys) assert fks[0].column == "author_id" assert fks[1].column == "category_id" + + +def test_mixed_compound_and_single_foreign_keys_are_sortable(): + # compound FKs have column=None, which must not break sorting + # against single-column FKs (None < str raises TypeError) + db = Database(memory=True) + db.executescript(""" + CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + PRIMARY KEY (campus_name, dept_code) + ); + CREATE TABLE accreditations (id INTEGER PRIMARY KEY); + CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + accreditation_id INTEGER REFERENCES accreditations(id), + FOREIGN KEY (campus_name, dept_code) + REFERENCES departments(campus_name, dept_code) + ); + """) + fks = db["courses"].foreign_keys + assert len(fks) == 2 + assert {fk.is_compound for fk in fks} == {True, False} + fks_sorted = sorted(fks) + assert fks_sorted[0].other_table == "accreditations" + assert fks_sorted[1].other_table == "departments" From 577f3011e5bbc5a043b7e4f9d10b06da76e21bf1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:30:41 -0700 Subject: [PATCH 006/110] Create tables with compound foreign keys, refs #594 create_table() and friends now accept compound foreign keys, specified as lists of column names in the existing tuple forms: foreign_keys=[ (["campus_name", "dept_code"], "departments"), (["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"]), ] The two-item form guesses the compound primary key of the other table. Compound keys are rendered as table-level FOREIGN KEY constraints, while single-column keys keep their inline REFERENCES clauses. Co-Authored-By: Claude Fable 5 --- docs/python-api.rst | 36 ++++++++++++ sqlite_utils/db.py | 110 +++++++++++++++++++++++++++++-------- tests/test_foreign_keys.py | 96 +++++++++++++++++++++++++++++++- 3 files changed, 218 insertions(+), 24 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index d1f466f..5e55256 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -817,6 +817,42 @@ You can leave off the third item in the tuple to have the referenced column auto ("author_id", "authors") ]) +.. _python_api_compound_foreign_keys: + +Compound foreign keys +~~~~~~~~~~~~~~~~~~~~~ + +To create a compound (multi-column) foreign key, use lists of column names in place of the single column names: + +.. code-block:: python + + db.table("courses").create({ + "course_code": str, + "campus_name": str, + "dept_code": str, + }, pk="course_code", foreign_keys=[ + (["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"]) + ]) + +This creates a table-level constraint: + +.. code-block:: sql + + CREATE TABLE "courses" ( + "course_code" TEXT PRIMARY KEY, + "campus_name" TEXT, + "dept_code" TEXT, + FOREIGN KEY ("campus_name", "dept_code") REFERENCES "departments"("campus_name", "dept_code") + ) + +As with single columns, you can leave off the list of other columns to reference the compound primary key of the other table: + +.. code-block:: python + + foreign_keys=[ + (["campus_name", "dept_code"], "departments") + ] + .. _python_api_table_configuration: Table configuration options diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 67623fe..12914d1 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -219,6 +219,10 @@ ForeignKeyIndicator = Union[ Tuple[str, str], Tuple[str, str, str], Tuple[str, str, str, str], + # Compound foreign keys use lists of columns: + Tuple[List[str], str], + Tuple[List[str], str, List[str]], + Tuple[str, List[str], str, List[str]], ] ForeignKeysType = Union[Iterable[ForeignKeyIndicator], List[ForeignKeyIndicator]] @@ -1142,9 +1146,12 @@ class Database: :param name: Name of table that foreign keys are being defined for :param foreign_keys: List of foreign keys, each of which can be a - string, a ForeignKey() named tuple, a tuple of (column, other_table), + string, a ForeignKey() object, a tuple of (column, other_table), or a tuple of (column, other_table, other_column), or a tuple of - (table, column, other_table, other_column) + (table, column, other_table, other_column). For compound foreign + keys the column elements can be lists of column names, e.g. + (["campus_name", "dept_code"], "departments") or + (["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"]) """ table = self.table(name) if all(isinstance(fk, ForeignKey) for fk in foreign_keys): @@ -1161,7 +1168,7 @@ class Database: if not all(isinstance(fk, (tuple, list)) for fk in foreign_keys): raise ValueError("foreign_keys= should be a list of tuples") fks = [] - for tuple_or_list in cast(Iterable[Sequence[str]], foreign_keys): + for tuple_or_list in cast(Iterable[Sequence[Any]], foreign_keys): if len(tuple_or_list) == 4: if tuple_or_list[0] != name: raise ValueError( @@ -1169,28 +1176,61 @@ class Database: tuple_or_list, name ) ) - if len(tuple_or_list) not in (2, 3, 4): + tuple_or_list = tuple_or_list[1:] + if len(tuple_or_list) not in (2, 3): raise ValueError( "foreign_keys= should be a list of tuple pairs or triples" ) - if len(tuple_or_list) in (3, 4): - if len(tuple_or_list) == 4: - tuple_or_list = cast(Tuple[str, str, str], tuple_or_list[1:]) + column_or_columns = tuple_or_list[0] + other_table = tuple_or_list[1] + if isinstance(column_or_columns, (list, tuple)): + # Compound foreign key + columns = list(column_or_columns) + if len(tuple_or_list) == 3: + other_columns = tuple_or_list[2] + if not isinstance(other_columns, (list, tuple)): + raise ValueError( + "Compound foreign key {} should reference a list " + "of other columns".format(tuple(tuple_or_list)) + ) + other_columns = list(other_columns) else: - tuple_or_list = cast(Tuple[str, str, str], tuple_or_list) - fks.append( - ForeignKey( - name, tuple_or_list[0], tuple_or_list[1], tuple_or_list[2] + # Guess the compound primary key of the other table + other_columns = self.table(other_table).pks + if len(columns) != len(other_columns): + raise ValueError( + "Compound foreign key {} should have the same number " + "of columns on both sides".format(tuple(tuple_or_list)) ) + if len(columns) == 1: + # Single-column key passed as a one-item list + fks.append( + ForeignKey(name, columns[0], other_table, other_columns[0]) + ) + else: + fks.append( + ForeignKey( + name, + None, + other_table, + None, + columns=columns, + other_columns=other_columns, + is_compound=True, + ) + ) + elif len(tuple_or_list) == 3: + fks.append( + ForeignKey(name, column_or_columns, other_table, tuple_or_list[2]) ) else: # Guess the primary key fks.append( ForeignKey( name, - tuple_or_list[0], - tuple_or_list[1], - table.guess_foreign_column(tuple_or_list[1]), + column_or_columns, + other_table, + table.guess_foreign_column(other_table), ) ) return fks @@ -1229,7 +1269,11 @@ class Database: if hash_id_columns and (hash_id is None): hash_id = "id" foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) - foreign_keys_by_column = {fk.column: fk for fk in foreign_keys} + # Compound foreign keys are rendered as table-level constraints; + # single-column ones as inline REFERENCES on their column + foreign_keys_by_column = { + fk.column: fk for fk in foreign_keys if not fk.is_compound + } # any extracts will be treated as integer columns with a foreign key extracts = resolve_extracts(extracts) for extract_column, extract_table in extracts.items(): @@ -1271,14 +1315,15 @@ class Database: pk = hash_id # Soundness check foreign_keys point to existing tables for fk in foreign_keys: - if fk.other_table == name and columns.get(cast(str, fk.other_column)): - continue - if fk.other_column != "rowid" and not any( - c for c in self[fk.other_table].columns if c.name == fk.other_column - ): - raise AlterError( - "No such column: {}.{}".format(fk.other_table, fk.other_column) - ) + for other_column in fk.other_columns: + if fk.other_table == name and columns.get(other_column): + continue + if other_column != "rowid" and not any( + c for c in self[fk.other_table].columns if c.name == other_column + ): + raise AlterError( + "No such column: {}.{}".format(fk.other_table, other_column) + ) column_defs = [] # ensure pk is a tuple @@ -1329,6 +1374,25 @@ class Database: extra_pk = ",\n PRIMARY KEY ({pks})".format( pks=", ".join([quote_identifier(p) for p in pk]) ) + # Compound foreign keys become table-level FOREIGN KEY constraints + column_names = [c[0] for c in column_items] + for fk in foreign_keys: + if not fk.is_compound: + continue + missing = [c for c in fk.columns if c not in column_names] + if missing: + raise AlterError( + "No such column: {}".format(", ".join(sorted(missing))) + ) + column_defs.append( + " FOREIGN KEY ({columns}) REFERENCES {other_table}({other_columns})".format( + columns=", ".join(quote_identifier(c) for c in fk.columns), + other_table=quote_identifier(fk.other_table), + other_columns=", ".join( + quote_identifier(c) for c in fk.other_columns + ), + ) + ) columns_sql = ",\n".join(column_defs) sql = """CREATE TABLE {if_not_exists}{table} ( {columns_sql}{extra_pk} diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 4b371a9..457f307 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -1,7 +1,8 @@ -"""Tests for reading compound (multi-column) foreign keys - issue #594.""" +"""Tests for compound (multi-column) foreign keys - issue #594.""" import pytest from sqlite_utils import Database +from sqlite_utils.db import AlterError, ForeignKey COMPOUND_SCHEMA = """ CREATE TABLE departments ( @@ -108,3 +109,96 @@ def test_mixed_compound_and_single_foreign_keys_are_sortable(): fks_sorted = sorted(fks) assert fks_sorted[0].other_table == "accreditations" assert fks_sorted[1].other_table == "departments" + + +@pytest.fixture +def departments_db(): + db = Database(memory=True) + db.create_table( + "departments", + {"campus_name": str, "dept_code": str, "dept_name": str}, + pk=("campus_name", "dept_code"), + ) + return db + + +EXPECTED_COURSES_SCHEMA = ( + 'CREATE TABLE "courses" (\n' + ' "course_code" TEXT PRIMARY KEY,\n' + ' "campus_name" TEXT,\n' + ' "dept_code" TEXT,\n' + ' FOREIGN KEY ("campus_name", "dept_code") ' + 'REFERENCES "departments"("campus_name", "dept_code")\n' + ")" +) + + +@pytest.mark.parametrize( + "foreign_keys", + ( + [ + ForeignKey( + table="courses", + column=None, + other_table="departments", + other_column=None, + columns=["campus_name", "dept_code"], + other_columns=["campus_name", "dept_code"], + is_compound=True, + ) + ], + [(["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"])], + # Two-item form guesses the other table's primary key: + [(["campus_name", "dept_code"], "departments")], + ), +) +def test_create_table_with_compound_foreign_key(departments_db, foreign_keys): + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + foreign_keys=foreign_keys, + ) + assert departments_db["courses"].schema == EXPECTED_COURSES_SCHEMA + fks = departments_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ["campus_name", "dept_code"] + assert fk.other_table == "departments" + assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_create_table_compound_foreign_key_enforced(departments_db): + departments_db.execute("PRAGMA foreign_keys = ON") + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + foreign_keys=[(["campus_name", "dept_code"], "departments")], + ) + departments_db["departments"].insert( + {"campus_name": "Berkeley", "dept_code": "CS", "dept_name": "Computer Science"} + ) + departments_db["courses"].insert( + {"course_code": "CS101", "campus_name": "Berkeley", "dept_code": "CS"} + ) + import sqlite3 + + with pytest.raises(sqlite3.IntegrityError): + departments_db.execute( + "insert into courses (course_code, campus_name, dept_code) " + "values ('X1', 'Nowhere', 'NOPE')" + ) + + +def test_create_table_compound_foreign_key_missing_other_column(departments_db): + with pytest.raises(AlterError): + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + foreign_keys=[ + (["campus_name", "dept_code"], "departments", ["campus_name", "nope"]) + ], + ) From 7d43fd50e186c359cd0c61a6655f15f0d156da87 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:33:15 -0700 Subject: [PATCH 007/110] transform() now round-trips compound foreign keys, refs #594 Compound foreign keys are passed through table.transform() intact instead of being degraded to per-column single foreign keys: - rename= applies to each member column of a compound key - drop= of any member column drops the whole constraint, matching the existing single-column behavior - drop_foreign_keys= accepts a bare column name (drops any foreign key that column participates in) or a tuple of columns (drops the compound key with exactly those columns) Co-Authored-By: Claude Fable 5 --- docs/python-api.rst | 10 ++++++ sqlite_utils/db.py | 63 ++++++++++++++++++++++++-------------- tests/test_foreign_keys.py | 48 +++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 23 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 5e55256..465df79 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1793,6 +1793,16 @@ This example drops two foreign keys - the one from ``places.country`` to ``count drop_foreign_keys=("country", "continent") ) +A bare column name drops any foreign key that column participates in, including compound foreign keys. To target a compound foreign key precisely, pass a tuple of its columns: + +.. code-block:: python + + db.table("courses").transform( + drop_foreign_keys=[("campus_name", "dept_code")] + ) + +Renaming a column with ``rename=`` updates any foreign keys that use it, and dropping a column with ``drop=`` also drops any foreign keys it participates in - for a compound foreign key this removes the whole constraint. + .. _python_api_transform_sql: Custom transformations with .transform_sql() diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 12914d1..c37f750 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2275,7 +2275,9 @@ class Table(Queryable): :param pk: New primary key for the table :param not_null: Columns to set as ``NOT NULL`` :param defaults: Default values for columns - :param drop_foreign_keys: Names of columns that should have their foreign key constraints removed + :param drop_foreign_keys: Foreign key constraints to remove - a column name + drops any foreign key that column participates in, a tuple of column names + drops the compound foreign key with exactly those columns :param add_foreign_keys: List of foreign keys to add to the table :param foreign_keys: List of foreign keys to set for the table, replacing any existing foreign keys :param column_order: List of strings specifying a full or partial column order @@ -2360,7 +2362,9 @@ class Table(Queryable): :param pk: New primary key for the table :param not_null: Columns to set as ``NOT NULL`` :param defaults: Default values for columns - :param drop_foreign_keys: Names of columns that should have their foreign key constraints removed + :param drop_foreign_keys: Foreign key constraints to remove - a column name + drops any foreign key that column participates in, a tuple of column names + drops the compound foreign key with exactly those columns :param add_foreign_keys: List of foreign keys to add to the table :param foreign_keys: List of foreign keys to set for the table, replacing any existing foreign keys :param column_order: List of strings specifying a full or partial column order @@ -2387,32 +2391,45 @@ class Table(Queryable): create_table_foreign_keys.extend(foreign_keys) else: # Construct foreign_keys from current, plus add_foreign_keys, minus drop_foreign_keys + + def fk_should_be_dropped(fk: ForeignKey) -> bool: + if drop_foreign_keys is not None: + for spec in drop_foreign_keys: + if isinstance(spec, str): + # A column name matches any foreign key it participates in + if spec in fk.columns: + return True + elif list(spec) == fk.columns: + # A tuple/list must match a compound key's columns exactly + return True + # Dropping any of a foreign key's columns drops the whole key + return any(column in drop for column in fk.columns) + + def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: + columns = [rename.get(column) or column for column in fk.columns] + if fk.is_compound: + return ForeignKey( + self.name, + None, + fk.other_table, + None, + columns=columns, + other_columns=fk.other_columns, + is_compound=True, + ) + return ForeignKey( + self.name, columns[0], fk.other_table, fk.other_columns[0] + ) + create_table_foreign_keys = [] + # Copy over old foreign keys, unless we are dropping them for fk in self.foreign_keys: - # Expand compound foreign keys into per-column references; - # for single-column keys this iterates exactly once - for column, other_column in zip(fk.columns, fk.other_columns): - # Copy over old foreign keys, unless we are dropping them - if (drop_foreign_keys is None) or (column not in drop_foreign_keys): - create_table_foreign_keys.append( - ForeignKey( - fk.table, - rename.get(column) or column, - fk.other_table, - other_column, - ) - ) + if not fk_should_be_dropped(fk): + create_table_foreign_keys.append(fk_with_renamed_columns(fk)) # Add new foreign keys if add_foreign_keys is not None: for fk in self.db.resolve_foreign_keys(self.name, add_foreign_keys): - create_table_foreign_keys.append( - ForeignKey( - self.name, - rename.get(fk.column) or fk.column, - fk.other_table, - fk.other_column, - ) - ) + create_table_foreign_keys.append(fk_with_renamed_columns(fk)) new_table_name = "{}_new_{}".format( self.name, tmp_suffix or os.urandom(6).hex() diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 457f307..d6a7071 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -202,3 +202,51 @@ def test_create_table_compound_foreign_key_missing_other_column(departments_db): (["campus_name", "dept_code"], "departments", ["campus_name", "nope"]) ], ) + + +def test_transform_preserves_compound_foreign_key(compound_db): + compound_db["courses"].transform(rename={"course_name": "title"}) + fks = compound_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ["campus_name", "dept_code"] + assert fk.other_table == "departments" + assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_transform_rename_member_column_updates_compound_foreign_key(compound_db): + compound_db["courses"].transform(rename={"campus_name": "campus"}) + fks = compound_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ["campus", "dept_code"] + # Referenced columns in the other table are unchanged + assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_transform_drop_member_column_drops_compound_foreign_key(compound_db): + # Matches single-column behavior: dropping the column silently + # drops the foreign key that used it + compound_db["courses"].transform(drop={"dept_code"}) + assert compound_db["courses"].foreign_keys == [] + assert "FOREIGN KEY" not in compound_db["courses"].schema + + +@pytest.mark.parametrize( + "drop_foreign_keys", + ( + # A bare column name matches any foreign key it participates in: + ["campus_name"], + # A tuple must match the full compound key: + [("campus_name", "dept_code")], + ), +) +def test_transform_drop_compound_foreign_key(compound_db, drop_foreign_keys): + compound_db["courses"].transform(drop_foreign_keys=drop_foreign_keys) + assert compound_db["courses"].foreign_keys == [] + # The columns themselves survive + assert {"campus_name", "dept_code"} <= set( + compound_db["courses"].columns_dict.keys() + ) From b75edf4b30dd500747f6f9f5f1dcb52c03a5b442 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:37:32 -0700 Subject: [PATCH 008/110] add_foreign_key() accepts compound keys, composite FK indexes, refs #594 table.add_foreign_key() and db.add_foreign_keys() now accept lists of column names to create compound foreign keys: table.add_foreign_key( ["campus_name", "dept_code"], "departments" ) Omitting the other columns uses the compound primary key of the other table. Duplicate detection compares the full column lists. db.index_foreign_keys() now creates a single composite index across the columns of a compound foreign key, rather than an index per column. Co-Authored-By: Claude Fable 5 --- docs/python-api.rst | 12 ++++ sqlite_utils/db.py | 138 ++++++++++++++++++++++++------------- tests/test_foreign_keys.py | 74 ++++++++++++++++++++ 3 files changed, 177 insertions(+), 47 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 465df79..882f952 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1562,6 +1562,16 @@ To ignore the case where the key already exists, use ``ignore=True``: db.table("books").add_foreign_key("author_id", "authors", "id", ignore=True) +To add a compound foreign key, pass lists of columns: + +.. code-block:: python + + db.table("courses").add_foreign_key( + ["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"] + ) + +As with single columns, omitting the other columns will use the compound primary key of the other table. ``other_table`` must always be specified for a compound foreign key. + .. _python_api_add_foreign_keys: Adding multiple foreign key constraints at once @@ -1591,6 +1601,8 @@ If you want to ensure that every foreign key column in your database has a corre db.index_foreign_keys() +Compound foreign keys get a single composite index across their columns. + .. _python_api_drop: Dropping a table or view diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c37f750..3c78e3b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1590,13 +1590,14 @@ class Database: return candidates def add_foreign_keys( - self, foreign_keys: Iterable[Tuple[str, str, str, str]] + self, foreign_keys: Iterable[Union[ForeignKey, Tuple[str, Any, str, Any]]] ) -> None: """ See :ref:`python_api_add_foreign_keys`. :param foreign_keys: A list of ``(table, column, other_table, other_column)`` - tuples + tuples - for compound foreign keys, ``column`` and ``other_column`` can + be lists of column names """ # foreign_keys is a list of explicit 4-tuples if not all( @@ -1609,44 +1610,64 @@ class Database: "(table, column, other_table, other_column)" ) - foreign_keys_to_create = [] + foreign_keys_to_create: List[Tuple[str, Any, str, Any]] = [] # Verify that all tables and columns exist for fk in foreign_keys: if isinstance(fk, ForeignKey): - table, column, other_table, other_column = ( + table, columns, other_table, other_columns = ( fk.table, - fk.column, + fk.columns, fk.other_table, - fk.other_column, + fk.other_columns, ) else: - table, column, other_table, other_column = fk + table, column_or_columns, other_table, other_column_or_columns = fk + # Compound foreign keys use lists of columns + columns = ( + [column_or_columns] + if isinstance(column_or_columns, str) + else list(column_or_columns) + ) + other_columns = ( + [other_column_or_columns] + if isinstance(other_column_or_columns, str) + else list(other_column_or_columns) + ) if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) table_obj = self.table(table) - if column not in table_obj.columns_dict: - raise AlterError("No such column: {} in {}".format(column, table)) + for column in columns: + if column not in table_obj.columns_dict: + raise AlterError("No such column: {} in {}".format(column, table)) if not self[other_table].exists(): raise AlterError("No such other_table: {}".format(other_table)) - if ( - other_column != "rowid" - and other_column not in self[other_table].columns_dict - ): - raise AlterError( - "No such other_column: {} in {}".format(other_column, other_table) - ) + for other_column in other_columns: + if ( + other_column != "rowid" + and other_column not in self[other_table].columns_dict + ): + raise AlterError( + "No such other_column: {} in {}".format( + other_column, other_table + ) + ) # We will silently skip foreign keys that exist already if not any( fk for fk in table_obj.foreign_keys - if fk.column == column + if fk.columns == columns and fk.other_table == other_table - and fk.other_column == other_column + and fk.other_columns == other_columns ): - foreign_keys_to_create.append( - (table, column, other_table, other_column) - ) + if len(columns) == 1: + foreign_keys_to_create.append( + (table, columns[0], other_table, other_columns[0]) + ) + else: + foreign_keys_to_create.append( + (table, columns, other_table, other_columns) + ) # Group them by table by_table: Dict[str, List] = {} @@ -1663,15 +1684,12 @@ class Database: "Create indexes for every foreign key column on every table in the database." for table_name in self.table_names(): table = self.table(table_name) - existing_indexes = { - i.columns[0] for i in table.indexes if len(i.columns) == 1 - } + existing_indexes = {tuple(i.columns) for i in table.indexes} for fk in table.foreign_keys: - # Compound foreign keys expose their columns via fk.columns; - # single-column keys yield a one-item list - for column in fk.columns: - if column not in existing_indexes: - table.create_index([column], find_unique_name=True) + # A compound foreign key gets a single composite index + if tuple(fk.columns) not in existing_indexes: + table.create_index(fk.columns, find_unique_name=True) + existing_indexes.add(tuple(fk.columns)) def vacuum(self) -> None: "Run a SQLite ``VACUUM`` against the database." @@ -2862,52 +2880,78 @@ class Table(Queryable): def add_foreign_key( self, - column: str, + column: Union[str, List[str]], other_table: Optional[str] = None, - other_column: Optional[str] = None, + other_column: Optional[Union[str, List[str]]] = None, ignore: bool = False, ): """ Alter the schema to mark the specified column as a foreign key to another table. - :param column: The column to mark as a foreign key. + :param column: The column to mark as a foreign key - use a list of columns + for a compound foreign key. :param other_table: The table it refers to - if omitted, will be guessed based on the column name. :param other_column: The column on the other table it - if omitted, will be guessed. + Use a list of columns for a compound foreign key. :param ignore: Set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. """ - # Ensure column exists - if column not in self.columns_dict: - raise AlterError("No such column: {}".format(column)) + columns = [column] if isinstance(column, str) else list(column) + # Ensure columns exist + for col in columns: + if col not in self.columns_dict: + raise AlterError("No such column: {}".format(col)) # If other_table is not specified, attempt to guess it from the column if other_table is None: - other_table = self.guess_foreign_table(column) + if len(columns) > 1: + raise ValueError( + "other_table must be specified for a compound foreign key" + ) + other_table = self.guess_foreign_table(columns[0]) # If other_column is not specified, detect the primary key on other_table if other_column is None: - other_column = self.guess_foreign_column(other_table) + if len(columns) > 1: + other_columns = self.db.table(other_table).pks + else: + other_columns = [self.guess_foreign_column(other_table)] + elif isinstance(other_column, str): + other_columns = [other_column] + else: + other_columns = list(other_column) + if len(columns) != len(other_columns): + raise ValueError( + "Compound foreign key must have the same number of columns " + "on both sides" + ) - # Soundness check that the other column exists - if ( - not [c for c in self.db[other_table].columns if c.name == other_column] - and other_column != "rowid" - ): - raise AlterError("No such column: {}.{}".format(other_table, other_column)) + # Soundness check that the other columns exist + for other_col in other_columns: + if ( + not [c for c in self.db[other_table].columns if c.name == other_col] + and other_col != "rowid" + ): + raise AlterError("No such column: {}.{}".format(other_table, other_col)) # Check we do not already have an existing foreign key if any( fk for fk in self.foreign_keys - if fk.column == column + if fk.columns == columns and fk.other_table == other_table - and fk.other_column == other_column + and fk.other_columns == other_columns ): if ignore: return self else: raise AlterError( "Foreign key already exists for {} => {}.{}".format( - column, other_table, other_column + ", ".join(columns), other_table, ", ".join(other_columns) ) ) - self.db.add_foreign_keys([(self.name, column, other_table, other_column)]) + if len(columns) == 1: + self.db.add_foreign_keys( + [(self.name, columns[0], other_table, other_columns[0])] + ) + else: + self.db.add_foreign_keys([(self.name, columns, other_table, other_columns)]) return self def enable_counts(self) -> None: diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index d6a7071..1fabce7 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -250,3 +250,77 @@ def test_transform_drop_compound_foreign_key(compound_db, drop_foreign_keys): assert {"campus_name", "dept_code"} <= set( compound_db["courses"].columns_dict.keys() ) + + +@pytest.fixture +def courses_db(departments_db): + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + ) + return departments_db + + +def test_add_compound_foreign_key(courses_db): + t = courses_db["courses"].add_foreign_key( + ["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"] + ) + # Returns self + assert t.name == "courses" + fks = courses_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ["campus_name", "dept_code"] + assert fk.other_table == "departments" + assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_add_compound_foreign_key_guesses_other_columns(courses_db): + courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") + fk = courses_db["courses"].foreign_keys[0] + assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_add_compound_foreign_key_error_if_already_exists(courses_db): + courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") + with pytest.raises(AlterError) as ex: + courses_db["courses"].add_foreign_key( + ["campus_name", "dept_code"], "departments" + ) + assert "already exists" in ex.value.args[0] + # ignore=True should not raise + courses_db["courses"].add_foreign_key( + ["campus_name", "dept_code"], "departments", ignore=True + ) + + +def test_add_compound_foreign_key_error_if_column_missing(courses_db): + with pytest.raises(AlterError): + courses_db["courses"].add_foreign_key(["campus_name", "nope"], "departments") + + +def test_db_add_foreign_keys_compound(courses_db): + courses_db.add_foreign_keys( + [ + ( + "courses", + ["campus_name", "dept_code"], + "departments", + ["campus_name", "dept_code"], + ) + ] + ) + fk = courses_db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.columns == ["campus_name", "dept_code"] + + +def test_index_foreign_keys_compound_creates_composite_index(compound_db): + compound_db.index_foreign_keys() + index_columns = [i.columns for i in compound_db["courses"].indexes] + assert ["campus_name", "dept_code"] in index_columns + # No separate single-column indexes for the members + assert ["campus_name"] not in index_columns + assert ["dept_code"] not in index_columns From be27a96484b69fcc062b54c384e8713a9d500d9a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:40:57 -0700 Subject: [PATCH 009/110] Capture and preserve foreign key ON DELETE/ON UPDATE actions ForeignKey gains on_delete and on_update fields (default "NO ACTION"), populated from PRAGMA foreign_key_list. create_table_sql() renders the corresponding clauses for both inline and compound table-level foreign keys, which means table.transform() now preserves actions such as ON DELETE CASCADE - previously they were silently stripped whenever a table was transformed. Co-Authored-By: Claude Fable 5 --- docs/python-api.rst | 23 +++++++++++ sqlite_utils/db.py | 42 ++++++++++++++----- tests/test_foreign_keys.py | 84 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 10 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 882f952..f44a10b 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -853,6 +853,25 @@ As with single columns, you can leave off the list of other columns to reference (["campus_name", "dept_code"], "departments") ] +To specify ``ON DELETE`` or ``ON UPDATE`` actions, pass ``ForeignKey`` objects instead: + +.. code-block:: python + + from sqlite_utils.db import ForeignKey + + db.table("books").create({ + "id": int, + "author_id": int, + }, pk="id", foreign_keys=[ + ForeignKey( + table="books", column="author_id", + other_table="authors", other_column="id", + on_delete="CASCADE", + ) + ]) + +Foreign key actions are preserved by :ref:`table.transform() ` - prior to sqlite-utils 4.0 they were silently dropped when a table was transformed. + .. _python_api_table_configuration: Table configuration options @@ -2280,6 +2299,10 @@ Each ``ForeignKey`` has the following attributes: A list of the referenced columns. ``is_compound`` ``True`` if this is a compound (multi-column) foreign key. +``on_delete`` + The ``ON DELETE`` action, e.g. ``"CASCADE"`` - ``"NO ACTION"`` if not set. +``on_update`` + The ``ON UPDATE`` action - ``"NO ACTION"`` if not set. ``ForeignKey`` was a ``namedtuple`` prior to sqlite-utils 4.0. It is now a dataclass and can no longer be unpacked or indexed as a tuple - access its fields by name instead. See :ref:`upgrading_3_to_4` for details. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3c78e3b..1910b84 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -190,6 +190,8 @@ class ForeignKey: columns: List[str] = field(default_factory=list) other_columns: List[str] = field(default_factory=list) is_compound: bool = False + on_delete: str = "NO ACTION" + on_update: str = "NO ACTION" def __post_init__(self): # Populate columns/other_columns for single-column foreign keys @@ -201,6 +203,16 @@ class ForeignKey: ) +def _fk_actions_sql(fk: ForeignKey) -> str: + "ON UPDATE/ON DELETE clauses for a foreign key, or an empty string." + actions = "" + if fk.on_update and fk.on_update != "NO ACTION": + actions += " ON UPDATE {}".format(fk.on_update) + if fk.on_delete and fk.on_delete != "NO ACTION": + actions += " ON DELETE {}".format(fk.on_delete) + return actions + + Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns")) XIndex = namedtuple("XIndex", ("name", "columns")) XIndexColumn = namedtuple( @@ -1345,14 +1357,12 @@ class Database: "DEFAULT {}".format(self.quote_default_value(defaults[column_name])) ) if column_name in foreign_keys_by_column: + fk = foreign_keys_by_column[column_name] column_extras.append( - "REFERENCES {}({})".format( - quote_identifier( - foreign_keys_by_column[column_name].other_table - ), - quote_identifier( - cast(str, foreign_keys_by_column[column_name].other_column) - ), + "REFERENCES {}({}){}".format( + quote_identifier(fk.other_table), + quote_identifier(cast(str, fk.other_column)), + _fk_actions_sql(fk), ) ) column_type_str = COLUMN_TYPE_MAPPING[column_type] @@ -1385,12 +1395,13 @@ class Database: "No such column: {}".format(", ".join(sorted(missing))) ) column_defs.append( - " FOREIGN KEY ({columns}) REFERENCES {other_table}({other_columns})".format( + " FOREIGN KEY ({columns}) REFERENCES {other_table}({other_columns}){actions}".format( columns=", ".join(quote_identifier(c) for c in fk.columns), other_table=quote_identifier(fk.other_table), other_columns=", ".join( quote_identifier(c) for c in fk.other_columns ), + actions=_fk_actions_sql(fk), ) ) columns_sql = ",\n".join(column_defs) @@ -2055,7 +2066,9 @@ class Table(Queryable): ).fetchall(): if row is not None: id, seq, table_name, from_, to_, on_update, on_delete, match = row - by_id.setdefault(id, []).append((seq, table_name, from_, to_)) + by_id.setdefault(id, []).append( + (seq, table_name, from_, to_, on_update, on_delete) + ) fks = [] for id in sorted(by_id): rows = sorted(by_id[id]) # order columns by seq @@ -2072,6 +2085,8 @@ class Table(Queryable): columns=columns, other_columns=other_columns, is_compound=is_compound, + on_update=rows[0][4], + on_delete=rows[0][5], ) ) return fks @@ -2434,9 +2449,16 @@ class Table(Queryable): columns=columns, other_columns=fk.other_columns, is_compound=True, + on_delete=fk.on_delete, + on_update=fk.on_update, ) return ForeignKey( - self.name, columns[0], fk.other_table, fk.other_columns[0] + self.name, + columns[0], + fk.other_table, + fk.other_columns[0], + on_delete=fk.on_delete, + on_update=fk.on_update, ) create_table_foreign_keys = [] diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 1fabce7..afe333f 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -324,3 +324,87 @@ def test_index_foreign_keys_compound_creates_composite_index(compound_db): # No separate single-column indexes for the members assert ["campus_name"] not in index_columns assert ["dept_code"] not in index_columns + + +def test_foreign_key_captures_on_delete_and_on_update(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + author_id INTEGER REFERENCES authors(id) + ON DELETE CASCADE ON UPDATE RESTRICT + ); + """) + fk = db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert fk.on_update == "RESTRICT" + + +def test_foreign_key_on_delete_defaults_to_no_action(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fk = fresh_db["books"].foreign_keys[0] + assert fk.on_delete == "NO ACTION" + assert fk.on_update == "NO ACTION" + + +def test_create_table_foreign_key_with_on_delete(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db.create_table( + "books", + {"id": int, "author_id": int}, + pk="id", + foreign_keys=[ + ForeignKey( + table="books", + column="author_id", + other_table="authors", + other_column="id", + on_delete="CASCADE", + ) + ], + ) + assert "ON DELETE CASCADE" in fresh_db["books"].schema + assert fresh_db["books"].foreign_keys[0].on_delete == "CASCADE" + + +def test_transform_preserves_on_delete_cascade(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + db["books"].transform(rename={"title": "book_title"}) + fk = db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert fk.on_update == "NO ACTION" + assert "ON DELETE CASCADE" in db["books"].schema + + +def test_transform_preserves_compound_foreign_key_on_delete(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + PRIMARY KEY (campus_name, dept_code) + ); + CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + FOREIGN KEY (campus_name, dept_code) + REFERENCES departments(campus_name, dept_code) ON DELETE CASCADE + ); + """) + db["courses"].transform(rename={"course_code": "code"}) + fk = db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.on_delete == "CASCADE" + assert "ON DELETE CASCADE" in db["courses"].schema From 8443d7f3ba8a6762a1894afcadb342820130323e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:42:30 -0700 Subject: [PATCH 010/110] Resolve implicit primary key references in table.foreign_keys For foreign keys declared as "REFERENCES other_table" with no explicit columns, PRAGMA foreign_key_list returns None for the referenced column. The foreign_keys property now resolves these to the other table's primary key columns, so other_column=None unambiguously indicates a compound foreign key. Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 6 ++++++ tests/test_foreign_keys.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1910b84..dbb7445 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2075,6 +2075,12 @@ class Table(Queryable): other_table = rows[0][1] columns = [row[2] for row in rows] other_columns = [row[3] for row in rows] + if all(c is None for c in other_columns): + # "REFERENCES other_table" with no columns - the pragma + # returns None, meaning the other table's primary key + other_table_pks = self.db.table(other_table).pks + if len(other_table_pks) == len(columns): + other_columns = other_table_pks is_compound = len(rows) > 1 fks.append( ForeignKey( diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index afe333f..dc4ea68 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -408,3 +408,40 @@ def test_transform_preserves_compound_foreign_key_on_delete(): assert fk.is_compound is True assert fk.on_delete == "CASCADE" assert "ON DELETE CASCADE" in db["courses"].schema + + +def test_implicit_primary_key_reference_is_resolved(): + # REFERENCES authors (no column) has "to" of None in the pragma - + # it should be resolved to the primary key of the other table + db = Database(memory=True) + db.executescript(""" + CREATE TABLE authors (author_id INTEGER PRIMARY KEY); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + author_id INTEGER REFERENCES authors + ); + """) + fk = db["books"].foreign_keys[0] + assert fk.is_compound is False + assert fk.other_column == "author_id" + assert fk.other_columns == ["author_id"] + + +def test_implicit_compound_primary_key_reference_is_resolved(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + PRIMARY KEY (campus_name, dept_code) + ); + CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + FOREIGN KEY (campus_name, dept_code) REFERENCES departments + ); + """) + fk = db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.other_columns == ["campus_name", "dept_code"] From 42c1dd0d5feeebf295bc3c64a9a4b71544edcabf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:45:08 -0700 Subject: [PATCH 011/110] Documentation for compound foreign key support, refs #594 - Changelog entries for all the compound foreign key work - Upgrading guide notes the transform() behavior changes - ForeignKey added to the API reference - Updated .foreign_keys introspection example output Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 13 +++++++++++++ docs/python-api.rst | 4 ++-- docs/reference.rst | 7 +++++++ docs/upgrading.rst | 4 +++- sqlite_utils/db.py | 3 +++ 5 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 01d59a0..b182d88 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,19 @@ Breaking changes: - ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` lists, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) +Compound foreign key support, everywhere: + +- Tables can now be created with compound foreign keys, by passing lists of column names in ``foreign_keys=``: ``foreign_keys=[(["campus_name", "dept_code"], "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. See :ref:`python_api_compound_foreign_keys`. +- ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely. +- ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept lists of column names to add a compound foreign key to an existing table. +- ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key. + +Other foreign key improvements: + +- ``ForeignKey`` now exposes ``on_delete`` and ``on_update`` fields reflecting the foreign key's ``ON DELETE``/``ON UPDATE`` actions, and ``table.transform()`` preserves those actions. Previously a transform silently stripped clauses such as ``ON DELETE CASCADE`` from the table schema. +- Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``. +- Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound. + .. _v4_0rc2: 4.0rc2 (2026-07-04) diff --git a/docs/python-api.rst b/docs/python-api.rst index f44a10b..9c31841 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2309,8 +2309,8 @@ Each ``ForeignKey`` has the following attributes: :: >>> db.table("Street_Tree_List").foreign_keys - [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=['qLegalStatus'], other_columns=['id'], is_compound=False), - ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=['qCareAssistant'], other_columns=['id'], is_compound=False), + [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=['qLegalStatus'], other_columns=['id'], is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), + ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=['qCareAssistant'], other_columns=['id'], is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), ...] Compound foreign keys - defined with ``FOREIGN KEY (col_a, col_b) REFERENCES other(col_a, col_b)`` - are returned as a single ``ForeignKey`` with ``is_compound=True``, ``column`` and ``other_column`` set to ``None``, and the participating columns available in the ``columns`` and ``other_columns`` lists. diff --git a/docs/reference.rst b/docs/reference.rst index 5b5fd25..a9fdf29 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -70,6 +70,13 @@ sqlite_utils.db.ColumnDetails .. autoclass:: sqlite_utils.db.ColumnDetails +.. _reference_db_other_foreign_key: + +sqlite_utils.db.ForeignKey +-------------------------- + +.. autoclass:: sqlite_utils.db.ForeignKey + sqlite_utils.utils ================== diff --git a/docs/upgrading.rst b/docs/upgrading.rst index eb997f6..67a6b5e 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -79,7 +79,7 @@ Python API changes **View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method. -**ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained three new fields - ``columns``, ``other_columns`` and ``is_compound`` - so that compound (multi-column) foreign keys can be represented as a single object. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead: +**ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained new fields - ``columns``, ``other_columns``, ``is_compound``, ``on_delete`` and ``on_update`` - so that compound (multi-column) foreign keys and foreign key actions can be represented. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead: .. code-block:: python @@ -95,6 +95,8 @@ Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` lists instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are single-item lists. +Two related behavior changes to ``table.transform()``: compound foreign keys now survive a transform (previously they were split into separate single-column keys), and ``ON DELETE``/``ON UPDATE`` actions such as ``ON DELETE CASCADE`` are now preserved (previously they were silently stripped from the schema). + **Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``. **Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index dbb7445..009bafc 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -176,6 +176,9 @@ class ForeignKey: are ``None`` - use ``columns`` and ``other_columns`` instead, and check ``is_compound``. + ``on_delete`` and ``on_update`` hold the foreign key actions, e.g. + ``"CASCADE"`` - ``"NO ACTION"`` if not set. + Prior to sqlite-utils 4.0 this was a ``namedtuple`` and could be unpacked or indexed as ``(table, column, other_table, other_column)``. It is now a dataclass - access its fields by name instead. From c16edb2dc4a96490b69cdce2a5e8287ed3133924 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:03:23 -0700 Subject: [PATCH 012/110] Better type signatures for compound foreign key columns New ForeignKeyColumns type alias - a column name or a list/tuple of column names - used by add_foreign_key(), add_foreign_keys() (via the ForeignKeyTuple alias, replacing its Any slots) and the ForeignKeyIndicator union, which it also simplifies. Tuples now type-check anywhere lists are accepted. ForeignKey.__post_init__ normalizes tuple columns/other_columns to lists so direct construction with tuples compares equal to introspected foreign keys. Refs https://github.com/simonw/sqlite-utils/pull/770/changes#r3525703477 Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 33 ++++++++++++++++++++------------- tests/test_foreign_keys.py | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 009bafc..4e67a3c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -197,10 +197,15 @@ class ForeignKey: on_update: str = "NO ACTION" def __post_init__(self): - # Populate columns/other_columns for single-column foreign keys - if not self.columns: + # Populate columns/other_columns for single-column foreign keys, + # normalizing any tuples to lists + if self.columns: + self.columns = list(self.columns) + else: self.columns = [self.column] if self.column is not None else [] - if not self.other_columns: + if self.other_columns: + self.other_columns = list(self.other_columns) + else: self.other_columns = ( [self.other_column] if self.other_column is not None else [] ) @@ -228,16 +233,18 @@ class TransformError(Exception): pass +# A single column name, or a list/tuple of columns for a compound foreign key +ForeignKeyColumns = Union[str, List[str], Tuple[str, ...]] + +# (table, column(s), other_table, other_column(s)) +ForeignKeyTuple = Tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] + ForeignKeyIndicator = Union[ str, ForeignKey, - Tuple[str, str], - Tuple[str, str, str], - Tuple[str, str, str, str], - # Compound foreign keys use lists of columns: - Tuple[List[str], str], - Tuple[List[str], str, List[str]], - Tuple[str, List[str], str, List[str]], + Tuple[ForeignKeyColumns, str], + Tuple[ForeignKeyColumns, str, ForeignKeyColumns], + ForeignKeyTuple, ] ForeignKeysType = Union[Iterable[ForeignKeyIndicator], List[ForeignKeyIndicator]] @@ -1604,7 +1611,7 @@ class Database: return candidates def add_foreign_keys( - self, foreign_keys: Iterable[Union[ForeignKey, Tuple[str, Any, str, Any]]] + self, foreign_keys: Iterable[Union[ForeignKey, ForeignKeyTuple]] ) -> None: """ See :ref:`python_api_add_foreign_keys`. @@ -2911,9 +2918,9 @@ class Table(Queryable): def add_foreign_key( self, - column: Union[str, List[str]], + column: ForeignKeyColumns, other_table: Optional[str] = None, - other_column: Optional[Union[str, List[str]]] = None, + other_column: Optional[ForeignKeyColumns] = None, ignore: bool = False, ): """ diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index dc4ea68..54039b3 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -445,3 +445,19 @@ def test_implicit_compound_primary_key_reference_is_resolved(): fk = db["courses"].foreign_keys[0] assert fk.is_compound is True assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_foreign_key_normalizes_tuple_columns_to_lists(): + # Compound columns passed as tuples are normalized to lists, so they + # compare equal to introspected ForeignKeys + fk = ForeignKey( + table="courses", + column=None, + other_table="departments", + other_column=None, + columns=("campus_name", "dept_code"), + other_columns=("campus_name", "dept_code"), + is_compound=True, + ) + assert fk.columns == ["campus_name", "dept_code"] + assert fk.other_columns == ["campus_name", "dept_code"] From d100264e9c11f73c2f86328846a33e500eeb2118 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:10:21 -0700 Subject: [PATCH 013/110] Normalize compound foreign key columns to tuples, promote tuples in docs ForeignKey.columns and .other_columns are now tuples rather than lists, both when introspected and when constructed directly (lists are normalized in __post_init__). Tuples are now the documented form for specifying compound foreign keys everywhere - in create_table() foreign_keys=, add_foreign_key() and drop_foreign_keys=: foreign_keys=[(("campus_name", "dept_code"), "departments")] Lists continue to work as input but are no longer documented. Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 8 ++-- docs/python-api.rst | 22 +++++------ docs/upgrading.rst | 2 +- sqlite_utils/db.py | 77 +++++++++++++++++++------------------- tests/test_foreign_keys.py | 73 +++++++++++++++++++----------------- 5 files changed, 92 insertions(+), 90 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b182d88..cacd5a0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,13 +11,13 @@ Unreleased Breaking changes: -- ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` lists, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) +- ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) -Compound foreign key support, everywhere: +Compound foreign key support: -- Tables can now be created with compound foreign keys, by passing lists of column names in ``foreign_keys=``: ``foreign_keys=[(["campus_name", "dept_code"], "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. See :ref:`python_api_compound_foreign_keys`. +- Tables can now be created with compound foreign keys, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. See :ref:`python_api_compound_foreign_keys`. - ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely. -- ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept lists of column names to add a compound foreign key to an existing table. +- ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept tuples of column names to add a compound foreign key to an existing table. - ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key. Other foreign key improvements: diff --git a/docs/python-api.rst b/docs/python-api.rst index 9c31841..9ff89eb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -822,7 +822,7 @@ You can leave off the third item in the tuple to have the referenced column auto Compound foreign keys ~~~~~~~~~~~~~~~~~~~~~ -To create a compound (multi-column) foreign key, use lists of column names in place of the single column names: +To create a compound (multi-column) foreign key, use tuples of column names in place of the single column names: .. code-block:: python @@ -831,7 +831,7 @@ To create a compound (multi-column) foreign key, use lists of column names in pl "campus_name": str, "dept_code": str, }, pk="course_code", foreign_keys=[ - (["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"]) + (("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")) ]) This creates a table-level constraint: @@ -845,12 +845,12 @@ This creates a table-level constraint: FOREIGN KEY ("campus_name", "dept_code") REFERENCES "departments"("campus_name", "dept_code") ) -As with single columns, you can leave off the list of other columns to reference the compound primary key of the other table: +As with single columns, you can leave off the tuple of other columns to reference the compound primary key of the other table: .. code-block:: python foreign_keys=[ - (["campus_name", "dept_code"], "departments") + (("campus_name", "dept_code"), "departments") ] To specify ``ON DELETE`` or ``ON UPDATE`` actions, pass ``ForeignKey`` objects instead: @@ -1581,12 +1581,12 @@ To ignore the case where the key already exists, use ``ignore=True``: db.table("books").add_foreign_key("author_id", "authors", "id", ignore=True) -To add a compound foreign key, pass lists of columns: +To add a compound foreign key, pass tuples of columns: .. code-block:: python db.table("courses").add_foreign_key( - ["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"] + ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code") ) As with single columns, omitting the other columns will use the compound primary key of the other table. ``other_table`` must always be specified for a compound foreign key. @@ -2294,9 +2294,9 @@ Each ``ForeignKey`` has the following attributes: ``other_column`` The referenced column, or ``None`` for a compound foreign key. ``columns`` - A list of the columns on this table, always populated (a single-item list for single-column foreign keys). + A tuple of the columns on this table, always populated (a one-item tuple for single-column foreign keys). ``other_columns`` - A list of the referenced columns. + A tuple of the referenced columns. ``is_compound`` ``True`` if this is a compound (multi-column) foreign key. ``on_delete`` @@ -2309,11 +2309,11 @@ Each ``ForeignKey`` has the following attributes: :: >>> db.table("Street_Tree_List").foreign_keys - [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=['qLegalStatus'], other_columns=['id'], is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), - ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=['qCareAssistant'], other_columns=['id'], is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), + [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=('qLegalStatus',), other_columns=('id',), is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), + ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=('qCareAssistant',), other_columns=('id',), is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), ...] -Compound foreign keys - defined with ``FOREIGN KEY (col_a, col_b) REFERENCES other(col_a, col_b)`` - are returned as a single ``ForeignKey`` with ``is_compound=True``, ``column`` and ``other_column`` set to ``None``, and the participating columns available in the ``columns`` and ``other_columns`` lists. +Compound foreign keys - defined with ``FOREIGN KEY (col_a, col_b) REFERENCES other(col_a, col_b)`` - are returned as a single ``ForeignKey`` with ``is_compound=True``, ``column`` and ``other_column`` set to ``None``, and the participating columns available in the ``columns`` and ``other_columns`` tuples. .. _python_api_introspection_schema: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 67a6b5e..e7fd273 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -93,7 +93,7 @@ Python API changes Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. -Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` lists instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are single-item lists. +Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` tuples instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are one-item tuples. Two related behavior changes to ``table.transform()``: compound foreign keys now survive a transform (previously they were split into separate single-column keys), and ``ON DELETE``/``ON UPDATE`` actions such as ``ON DELETE CASCADE`` are now preserved (previously they were silently stripped from the schema). diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 4e67a3c..a739486 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -170,7 +170,7 @@ class ForeignKey: A foreign key defined on a table. For single-column foreign keys ``column`` and ``other_column`` hold the - column names, and ``columns``/``other_columns`` are single-item lists. + column names, and ``columns``/``other_columns`` are one-item tuples. For compound (multi-column) foreign keys ``column`` and ``other_column`` are ``None`` - use ``columns`` and ``other_columns`` instead, and check @@ -190,24 +190,24 @@ class ForeignKey: column: Optional[str] = field(compare=False) other_table: str other_column: Optional[str] = field(compare=False) - columns: List[str] = field(default_factory=list) - other_columns: List[str] = field(default_factory=list) + columns: Tuple[str, ...] = () + other_columns: Tuple[str, ...] = () is_compound: bool = False on_delete: str = "NO ACTION" on_update: str = "NO ACTION" def __post_init__(self): # Populate columns/other_columns for single-column foreign keys, - # normalizing any tuples to lists + # normalizing any lists to tuples if self.columns: - self.columns = list(self.columns) + self.columns = tuple(self.columns) else: - self.columns = [self.column] if self.column is not None else [] + self.columns = (self.column,) if self.column is not None else () if self.other_columns: - self.other_columns = list(self.other_columns) + self.other_columns = tuple(self.other_columns) else: self.other_columns = ( - [self.other_column] if self.other_column is not None else [] + (self.other_column,) if self.other_column is not None else () ) @@ -233,8 +233,8 @@ class TransformError(Exception): pass -# A single column name, or a list/tuple of columns for a compound foreign key -ForeignKeyColumns = Union[str, List[str], Tuple[str, ...]] +# A single column name, or a tuple of columns for a compound foreign key +ForeignKeyColumns = Union[str, Tuple[str, ...], List[str]] # (table, column(s), other_table, other_column(s)) ForeignKeyTuple = Tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] @@ -1171,9 +1171,9 @@ class Database: string, a ForeignKey() object, a tuple of (column, other_table), or a tuple of (column, other_table, other_column), or a tuple of (table, column, other_table, other_column). For compound foreign - keys the column elements can be lists of column names, e.g. - (["campus_name", "dept_code"], "departments") or - (["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"]) + keys the column elements can be tuples of column names, e.g. + (("campus_name", "dept_code"), "departments") or + (("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")) """ table = self.table(name) if all(isinstance(fk, ForeignKey) for fk in foreign_keys): @@ -1207,18 +1207,17 @@ class Database: other_table = tuple_or_list[1] if isinstance(column_or_columns, (list, tuple)): # Compound foreign key - columns = list(column_or_columns) + columns = tuple(column_or_columns) if len(tuple_or_list) == 3: - other_columns = tuple_or_list[2] - if not isinstance(other_columns, (list, tuple)): + if not isinstance(tuple_or_list[2], (list, tuple)): raise ValueError( - "Compound foreign key {} should reference a list " + "Compound foreign key {} should reference a tuple " "of other columns".format(tuple(tuple_or_list)) ) - other_columns = list(other_columns) + other_columns = tuple(tuple_or_list[2]) else: # Guess the compound primary key of the other table - other_columns = self.table(other_table).pks + other_columns = tuple(self.table(other_table).pks) if len(columns) != len(other_columns): raise ValueError( "Compound foreign key {} should have the same number " @@ -1618,7 +1617,7 @@ class Database: :param foreign_keys: A list of ``(table, column, other_table, other_column)`` tuples - for compound foreign keys, ``column`` and ``other_column`` can - be lists of column names + be tuples of column names """ # foreign_keys is a list of explicit 4-tuples if not all( @@ -1644,16 +1643,16 @@ class Database: ) else: table, column_or_columns, other_table, other_column_or_columns = fk - # Compound foreign keys use lists of columns + # Compound foreign keys use tuples of columns columns = ( - [column_or_columns] + (column_or_columns,) if isinstance(column_or_columns, str) - else list(column_or_columns) + else tuple(column_or_columns) ) other_columns = ( - [other_column_or_columns] + (other_column_or_columns,) if isinstance(other_column_or_columns, str) - else list(other_column_or_columns) + else tuple(other_column_or_columns) ) if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) @@ -1708,9 +1707,9 @@ class Database: existing_indexes = {tuple(i.columns) for i in table.indexes} for fk in table.foreign_keys: # A compound foreign key gets a single composite index - if tuple(fk.columns) not in existing_indexes: + if fk.columns not in existing_indexes: table.create_index(fk.columns, find_unique_name=True) - existing_indexes.add(tuple(fk.columns)) + existing_indexes.add(fk.columns) def vacuum(self) -> None: "Run a SQLite ``VACUUM`` against the database." @@ -2083,12 +2082,12 @@ class Table(Queryable): for id in sorted(by_id): rows = sorted(by_id[id]) # order columns by seq other_table = rows[0][1] - columns = [row[2] for row in rows] - other_columns = [row[3] for row in rows] + columns = tuple(row[2] for row in rows) + other_columns = tuple(row[3] for row in rows) if all(c is None for c in other_columns): # "REFERENCES other_table" with no columns - the pragma # returns None, meaning the other table's primary key - other_table_pks = self.db.table(other_table).pks + other_table_pks = tuple(self.db.table(other_table).pks) if len(other_table_pks) == len(columns): other_columns = other_table_pks is_compound = len(rows) > 1 @@ -2448,14 +2447,14 @@ class Table(Queryable): # A column name matches any foreign key it participates in if spec in fk.columns: return True - elif list(spec) == fk.columns: + elif tuple(spec) == fk.columns: # A tuple/list must match a compound key's columns exactly return True # Dropping any of a foreign key's columns drops the whole key return any(column in drop for column in fk.columns) def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: - columns = [rename.get(column) or column for column in fk.columns] + columns = tuple(rename.get(column) or column for column in fk.columns) if fk.is_compound: return ForeignKey( self.name, @@ -2926,14 +2925,14 @@ class Table(Queryable): """ Alter the schema to mark the specified column as a foreign key to another table. - :param column: The column to mark as a foreign key - use a list of columns + :param column: The column to mark as a foreign key - use a tuple of columns for a compound foreign key. :param other_table: The table it refers to - if omitted, will be guessed based on the column name. :param other_column: The column on the other table it - if omitted, will be guessed. - Use a list of columns for a compound foreign key. + Use a tuple of columns for a compound foreign key. :param ignore: Set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. """ - columns = [column] if isinstance(column, str) else list(column) + columns = (column,) if isinstance(column, str) else tuple(column) # Ensure columns exist for col in columns: if col not in self.columns_dict: @@ -2948,13 +2947,13 @@ class Table(Queryable): # If other_column is not specified, detect the primary key on other_table if other_column is None: if len(columns) > 1: - other_columns = self.db.table(other_table).pks + other_columns = tuple(self.db.table(other_table).pks) else: - other_columns = [self.guess_foreign_column(other_table)] + other_columns = (self.guess_foreign_column(other_table),) elif isinstance(other_column, str): - other_columns = [other_column] + other_columns = (other_column,) else: - other_columns = list(other_column) + other_columns = tuple(other_column) if len(columns) != len(other_columns): raise ValueError( "Compound foreign key must have the same number of columns " diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 54039b3..d1bc498 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -36,8 +36,8 @@ def test_compound_foreign_key(compound_db): assert fk.is_compound is True assert fk.table == "courses" assert fk.other_table == "departments" - assert fk.columns == ["campus_name", "dept_code"] - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") + assert fk.other_columns == ("campus_name", "dept_code") # Scalar column/other_column can't sensibly hold a compound key assert fk.column is None assert fk.other_column is None @@ -51,8 +51,8 @@ def test_single_foreign_key_gets_columns_fields(fresh_db): assert fk.is_compound is False assert fk.column == "author_id" assert fk.other_column == "id" - assert fk.columns == ["author_id"] - assert fk.other_columns == ["id"] + assert fk.columns == ("author_id",) + assert fk.other_columns == ("id",) def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): @@ -142,14 +142,16 @@ EXPECTED_COURSES_SCHEMA = ( column=None, other_table="departments", other_column=None, - columns=["campus_name", "dept_code"], - other_columns=["campus_name", "dept_code"], + columns=("campus_name", "dept_code"), + other_columns=("campus_name", "dept_code"), is_compound=True, ) ], - [(["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"])], + [(("campus_name", "dept_code"), "departments", ("campus_name", "dept_code"))], # Two-item form guesses the other table's primary key: - [(["campus_name", "dept_code"], "departments")], + [(("campus_name", "dept_code"), "departments")], + # Lists work too, though tuples are the documented form: + [(["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"])], ), ) def test_create_table_with_compound_foreign_key(departments_db, foreign_keys): @@ -164,9 +166,9 @@ def test_create_table_with_compound_foreign_key(departments_db, foreign_keys): assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True - assert fk.columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") assert fk.other_table == "departments" - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") def test_create_table_compound_foreign_key_enforced(departments_db): @@ -175,7 +177,7 @@ def test_create_table_compound_foreign_key_enforced(departments_db): "courses", {"course_code": str, "campus_name": str, "dept_code": str}, pk="course_code", - foreign_keys=[(["campus_name", "dept_code"], "departments")], + foreign_keys=[(("campus_name", "dept_code"), "departments")], ) departments_db["departments"].insert( {"campus_name": "Berkeley", "dept_code": "CS", "dept_name": "Computer Science"} @@ -199,7 +201,7 @@ def test_create_table_compound_foreign_key_missing_other_column(departments_db): {"course_code": str, "campus_name": str, "dept_code": str}, pk="course_code", foreign_keys=[ - (["campus_name", "dept_code"], "departments", ["campus_name", "nope"]) + (("campus_name", "dept_code"), "departments", ("campus_name", "nope")) ], ) @@ -210,9 +212,9 @@ def test_transform_preserves_compound_foreign_key(compound_db): assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True - assert fk.columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") assert fk.other_table == "departments" - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") def test_transform_rename_member_column_updates_compound_foreign_key(compound_db): @@ -221,9 +223,9 @@ def test_transform_rename_member_column_updates_compound_foreign_key(compound_db assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True - assert fk.columns == ["campus", "dept_code"] + assert fk.columns == ("campus", "dept_code") # Referenced columns in the other table are unchanged - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") def test_transform_drop_member_column_drops_compound_foreign_key(compound_db): @@ -264,7 +266,7 @@ def courses_db(departments_db): def test_add_compound_foreign_key(courses_db): t = courses_db["courses"].add_foreign_key( - ["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"] + ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code") ) # Returns self assert t.name == "courses" @@ -272,33 +274,34 @@ def test_add_compound_foreign_key(courses_db): assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True - assert fk.columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") assert fk.other_table == "departments" - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") def test_add_compound_foreign_key_guesses_other_columns(courses_db): + # Lists work here too, though tuples are the documented form courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") fk = courses_db["courses"].foreign_keys[0] - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") def test_add_compound_foreign_key_error_if_already_exists(courses_db): - courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") + courses_db["courses"].add_foreign_key(("campus_name", "dept_code"), "departments") with pytest.raises(AlterError) as ex: courses_db["courses"].add_foreign_key( - ["campus_name", "dept_code"], "departments" + ("campus_name", "dept_code"), "departments" ) assert "already exists" in ex.value.args[0] # ignore=True should not raise courses_db["courses"].add_foreign_key( - ["campus_name", "dept_code"], "departments", ignore=True + ("campus_name", "dept_code"), "departments", ignore=True ) def test_add_compound_foreign_key_error_if_column_missing(courses_db): with pytest.raises(AlterError): - courses_db["courses"].add_foreign_key(["campus_name", "nope"], "departments") + courses_db["courses"].add_foreign_key(("campus_name", "nope"), "departments") def test_db_add_foreign_keys_compound(courses_db): @@ -306,15 +309,15 @@ def test_db_add_foreign_keys_compound(courses_db): [ ( "courses", - ["campus_name", "dept_code"], + ("campus_name", "dept_code"), "departments", - ["campus_name", "dept_code"], + ("campus_name", "dept_code"), ) ] ) fk = courses_db["courses"].foreign_keys[0] assert fk.is_compound is True - assert fk.columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") def test_index_foreign_keys_compound_creates_composite_index(compound_db): @@ -424,7 +427,7 @@ def test_implicit_primary_key_reference_is_resolved(): fk = db["books"].foreign_keys[0] assert fk.is_compound is False assert fk.other_column == "author_id" - assert fk.other_columns == ["author_id"] + assert fk.other_columns == ("author_id",) def test_implicit_compound_primary_key_reference_is_resolved(): @@ -444,20 +447,20 @@ def test_implicit_compound_primary_key_reference_is_resolved(): """) fk = db["courses"].foreign_keys[0] assert fk.is_compound is True - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") -def test_foreign_key_normalizes_tuple_columns_to_lists(): - # Compound columns passed as tuples are normalized to lists, so they +def test_foreign_key_normalizes_list_columns_to_tuples(): + # Compound columns passed as lists are normalized to tuples, so they # compare equal to introspected ForeignKeys fk = ForeignKey( table="courses", column=None, other_table="departments", other_column=None, - columns=("campus_name", "dept_code"), - other_columns=("campus_name", "dept_code"), + columns=["campus_name", "dept_code"], + other_columns=["campus_name", "dept_code"], is_compound=True, ) - assert fk.columns == ["campus_name", "dept_code"] - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") + assert fk.other_columns == ("campus_name", "dept_code") From 8dfbfa80b8282c14ed9732eff04ba07dbd5388e5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:36:46 -0700 Subject: [PATCH 014/110] Fix ty errors in transform_sql foreign key closures The fk_should_be_dropped and fk_with_renamed_columns closures captured the drop and rename parameters, but type checkers do not narrow captured variables inside nested functions so ty saw their Optional declared types. Bind the already-normalized values to fresh names for the closures to use. Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a739486..1c06b62 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2439,6 +2439,11 @@ class Table(Queryable): create_table_foreign_keys.extend(foreign_keys) else: # Construct foreign_keys from current, plus add_foreign_keys, minus drop_foreign_keys + # Bind fresh names here - type checkers do not narrow captured + # variables inside nested functions, so the closures would + # otherwise see the Optional declared types of drop and rename + dropped_columns = drop + renamed_columns = rename def fk_should_be_dropped(fk: ForeignKey) -> bool: if drop_foreign_keys is not None: @@ -2451,10 +2456,12 @@ class Table(Queryable): # A tuple/list must match a compound key's columns exactly return True # Dropping any of a foreign key's columns drops the whole key - return any(column in drop for column in fk.columns) + return any(column in dropped_columns for column in fk.columns) def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: - columns = tuple(rename.get(column) or column for column in fk.columns) + columns = tuple( + renamed_columns.get(column) or column for column in fk.columns + ) if fk.is_compound: return ForeignKey( self.name, From 2f599fc7c6aeede4e97bd91487e5de5f5e190dca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:38:05 -0700 Subject: [PATCH 015/110] Run ty in just lint, matching CI Co-Authored-By: Claude Fable 5 --- Justfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index f4f0e7f..5caa120 100644 --- a/Justfile +++ b/Justfile @@ -8,11 +8,12 @@ @run *options: uv run -- {{options}} -# Run linters: black, flake8, mypy, cog +# Run linters: black, flake8, mypy, ty, cog @lint: just run black . --check uv run flake8 uv run mypy sqlite_utils tests + uv run ty check sqlite_utils uv run cog --check README.md docs/*.rst uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt From 5b61530965df0d56a09e71f29cc970b4ea3e43d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:42:41 -0700 Subject: [PATCH 016/110] db.add_foreign_keys() no longer drops ON DELETE/ON UPDATE actions ForeignKey objects passed to db.add_foreign_keys() were flattened to plain (table, column, other_table, other_column) tuples before being handed to transform(), silently discarding their on_delete and on_update actions. The method now carries ForeignKey objects through to transform() intact - tuple inputs are converted to ForeignKey objects up front, which also simplifies the validation loop. Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 42 ++++++++++++++++++++++---------------- tests/test_foreign_keys.py | 35 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1c06b62..7379859 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1630,17 +1630,12 @@ class Database: "(table, column, other_table, other_column)" ) - foreign_keys_to_create: List[Tuple[str, Any, str, Any]] = [] + foreign_keys_to_create: List[ForeignKey] = [] # Verify that all tables and columns exist for fk in foreign_keys: if isinstance(fk, ForeignKey): - table, columns, other_table, other_columns = ( - fk.table, - fk.columns, - fk.other_table, - fk.other_columns, - ) + fk_object = fk else: table, column_or_columns, other_table, other_column_or_columns = fk # Compound foreign keys use tuples of columns @@ -1654,6 +1649,24 @@ class Database: if isinstance(other_column_or_columns, str) else tuple(other_column_or_columns) ) + if len(columns) == 1: + fk_object = ForeignKey( + table, columns[0], other_table, other_columns[0] + ) + else: + fk_object = ForeignKey( + table, + None, + other_table, + None, + columns=columns, + other_columns=other_columns, + is_compound=True, + ) + table = fk_object.table + columns = fk_object.columns + other_table = fk_object.other_table + other_columns = fk_object.other_columns if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) table_obj = self.table(table) @@ -1680,19 +1693,12 @@ class Database: and fk.other_table == other_table and fk.other_columns == other_columns ): - if len(columns) == 1: - foreign_keys_to_create.append( - (table, columns[0], other_table, other_columns[0]) - ) - else: - foreign_keys_to_create.append( - (table, columns, other_table, other_columns) - ) + foreign_keys_to_create.append(fk_object) # Group them by table - by_table: Dict[str, List] = {} - for fk in foreign_keys_to_create: - by_table.setdefault(fk[0], []).append(fk) + by_table: Dict[str, List[ForeignKey]] = {} + for fk_object in foreign_keys_to_create: + by_table.setdefault(fk_object.table, []).append(fk_object) for table, fks in by_table.items(): self.table(table).transform(add_foreign_keys=fks) diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index d1bc498..b22638a 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -464,3 +464,38 @@ def test_foreign_key_normalizes_list_columns_to_tuples(): ) assert fk.columns == ("campus_name", "dept_code") assert fk.other_columns == ("campus_name", "dept_code") + + +def test_add_foreign_keys_preserves_actions(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/594 review finding: + # ForeignKey objects passed to db.add_foreign_keys() were flattened + # to plain tuples, losing on_delete/on_update + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db.add_foreign_keys( + [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] + ) + fk = fresh_db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert "ON DELETE CASCADE" in fresh_db["books"].schema + + +def test_add_foreign_keys_preserves_actions_compound(courses_db): + courses_db.add_foreign_keys( + [ + ForeignKey( + table="courses", + column=None, + other_table="departments", + other_column=None, + columns=("campus_name", "dept_code"), + other_columns=("campus_name", "dept_code"), + is_compound=True, + on_delete="CASCADE", + ) + ] + ) + fk = courses_db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.on_delete == "CASCADE" + assert "ON DELETE CASCADE" in courses_db["courses"].schema From 658185d297a307d0b9d43fd83019a5acab032f18 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:46:16 -0700 Subject: [PATCH 017/110] Fix compound foreign key test failure against sqlean test_create_table_compound_foreign_key_enforced caught the stdlib sqlite3.IntegrityError, but sqlean's IntegrityError is not a subclass of it. Import sqlite3 from sqlite_utils.utils like the other test modules, so the right exception is used whichever backend is installed. Co-Authored-By: Claude Fable 5 --- tests/test_foreign_keys.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index b22638a..7fcd788 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -3,6 +3,7 @@ import pytest from sqlite_utils import Database from sqlite_utils.db import AlterError, ForeignKey +from sqlite_utils.utils import sqlite3 COMPOUND_SCHEMA = """ CREATE TABLE departments ( @@ -185,8 +186,6 @@ def test_create_table_compound_foreign_key_enforced(departments_db): departments_db["courses"].insert( {"course_code": "CS101", "campus_name": "Berkeley", "dept_code": "CS"} ) - import sqlite3 - with pytest.raises(sqlite3.IntegrityError): departments_db.execute( "insert into courses (course_code, campus_name, dept_code) " From 0ec01804055d7a09acc6c4068562c8bbb05dfdd7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:49:20 -0700 Subject: [PATCH 018/110] add_foreign_key() on_delete= and on_update= parameters, closes #530 table.add_foreign_key() now accepts on_delete= and on_update= to create foreign keys with ON DELETE/ON UPDATE actions: table.add_foreign_key("author_id", "authors", "id", on_delete="CASCADE") Works for compound foreign keys too. Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/python-api.rst | 10 ++++++++++ sqlite_utils/db.py | 27 ++++++++++++++++++++++++--- tests/test_foreign_keys.py | 26 ++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cacd5a0..f6a8ea2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,7 @@ Compound foreign key support: Other foreign key improvements: - ``ForeignKey`` now exposes ``on_delete`` and ``on_update`` fields reflecting the foreign key's ``ON DELETE``/``ON UPDATE`` actions, and ``table.transform()`` preserves those actions. Previously a transform silently stripped clauses such as ``ON DELETE CASCADE`` from the table schema. +- ``table.add_foreign_key()`` accepts new ``on_delete=`` and ``on_update=`` parameters for creating foreign keys with actions, e.g. ``table.add_foreign_key("author_id", "authors", "id", on_delete="CASCADE")``. (:issue:`530`) - Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``. - Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound. diff --git a/docs/python-api.rst b/docs/python-api.rst index 9ff89eb..a33b256 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1591,6 +1591,16 @@ To add a compound foreign key, pass tuples of columns: As with single columns, omitting the other columns will use the compound primary key of the other table. ``other_table`` must always be specified for a compound foreign key. +Use ``on_delete=`` and ``on_update=`` to specify ``ON DELETE`` and ``ON UPDATE`` actions for the foreign key: + +.. code-block:: python + + db.table("books").add_foreign_key( + "author_id", "authors", "id", on_delete="CASCADE" + ) + +This creates a foreign key with an ``ON DELETE CASCADE`` clause, so deleting an author will also delete their books (provided foreign key enforcement is enabled with ``PRAGMA foreign_keys = ON``). Valid actions are ``"SET NULL"``, ``"SET DEFAULT"``, ``"CASCADE"``, ``"RESTRICT"`` and the default ``"NO ACTION"``. + .. _python_api_add_foreign_keys: Adding multiple foreign key constraints at once diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 7379859..3f23c7d 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2934,6 +2934,8 @@ class Table(Queryable): other_table: Optional[str] = None, other_column: Optional[ForeignKeyColumns] = None, ignore: bool = False, + on_delete: str = "NO ACTION", + on_update: str = "NO ACTION", ): """ Alter the schema to mark the specified column as a foreign key to another table. @@ -2944,6 +2946,9 @@ class Table(Queryable): :param other_column: The column on the other table it - if omitted, will be guessed. Use a tuple of columns for a compound foreign key. :param ignore: Set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. + :param on_delete: ``ON DELETE`` action for the foreign key, e.g. ``"CASCADE"`` + or ``"SET NULL"``. + :param on_update: ``ON UPDATE`` action for the foreign key. """ columns = (column,) if isinstance(column, str) else tuple(column) # Ensure columns exist @@ -2997,11 +3002,27 @@ class Table(Queryable): ) ) if len(columns) == 1: - self.db.add_foreign_keys( - [(self.name, columns[0], other_table, other_columns[0])] + fk_object = ForeignKey( + self.name, + columns[0], + other_table, + other_columns[0], + on_delete=on_delete, + on_update=on_update, ) else: - self.db.add_foreign_keys([(self.name, columns, other_table, other_columns)]) + fk_object = ForeignKey( + self.name, + None, + other_table, + None, + columns=columns, + other_columns=other_columns, + is_compound=True, + on_delete=on_delete, + on_update=on_update, + ) + self.db.add_foreign_keys([fk_object]) return self def enable_counts(self) -> None: diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 7fcd788..8950189 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -498,3 +498,29 @@ def test_add_foreign_keys_preserves_actions_compound(courses_db): assert fk.is_compound is True assert fk.on_delete == "CASCADE" assert "ON DELETE CASCADE" in courses_db["courses"].schema + + +def test_add_foreign_key_on_delete_on_update(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db["books"].add_foreign_key( + "author_id", "authors", "id", on_delete="CASCADE", on_update="RESTRICT" + ) + fk = fresh_db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert fk.on_update == "RESTRICT" + assert "ON UPDATE RESTRICT ON DELETE CASCADE" in fresh_db["books"].schema + # The cascade should actually fire + fresh_db.execute("PRAGMA foreign_keys = ON") + fresh_db.execute("delete from authors where id = 1") + assert fresh_db["books"].count == 0 + + +def test_add_compound_foreign_key_on_delete(courses_db): + courses_db["courses"].add_foreign_key( + ("campus_name", "dept_code"), "departments", on_delete="SET NULL" + ) + fk = courses_db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.on_delete == "SET NULL" + assert "ON DELETE SET NULL" in courses_db["courses"].schema From afbfd95273c51d6a14174e9e07fb63ed2a34f72a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 16:00:52 -0700 Subject: [PATCH 019/110] Remove sqlean.py support (#772) Closes #771, refs #769 --- .github/workflows/test.yml | 3 --- docs/changelog.rst | 1 + docs/installation.rst | 10 +++++----- docs/python-api.rst | 2 +- mypy.ini | 3 --- sqlite_utils/utils.py | 11 +++-------- tests/test_gis.py | 8 -------- 7 files changed, 10 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7e1e953..d85cbf8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,9 +31,6 @@ jobs: - name: Install SpatiaLite if: matrix.os == 'ubuntu-latest' run: sudo apt-get install libsqlite3-mod-spatialite - - name: On macOS with Python 3.10 test with sqlean.py - if: matrix.os == 'macos-latest' && matrix.python-version == '3.10' - run: pip install sqlean.py sqlite-dump - name: Build extension for --load-extension test if: matrix.os == 'ubuntu-latest' run: |- diff --git a/docs/changelog.rst b/docs/changelog.rst index f6a8ea2..7efeb95 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ Unreleased Breaking changes: - ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) +- Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library. Compound foreign key support: diff --git a/docs/installation.rst b/docs/installation.rst index beb6d4a..1333f5d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -52,15 +52,15 @@ On some platforms the ability to load additional extensions (via ``conn.load_ext You may also see the error ``sqlite3.OperationalError: table sqlite_master may not be modified`` when trying to alter an existing table. -You can work around these limitations by installing either the `pysqlite3 `__ package or the `sqlean.py `__ package, both of which provide drop-in replacements for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions. +You can work around these limitations by installing the `pysqlite3 `__ package, which provides a drop-in replacement for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions. -To install ``sqlean.py`` (which has compiled binary wheels available for all major platforms) run the following: +To install ``pysqlite3`` run the following: .. code-block:: bash - sqlite-utils install sqlean.py + sqlite-utils install pysqlite3 -``pysqlite3`` and ``sqlean.py`` do not provide implementations of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package: +``pysqlite3`` does not provide an implementation of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package: .. code-block:: bash @@ -87,4 +87,4 @@ For ``zsh``: Add this code to ``~/.zshrc`` or ``~/.bashrc`` to automatically run it when you start a new shell. -See `the Click documentation `__ for more details. \ No newline at end of file +See `the Click documentation `__ for more details. diff --git a/docs/python-api.rst b/docs/python-api.rst index a33b256..6723efb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2184,7 +2184,7 @@ The ``db.iterdump()`` method returns a sequence of SQL strings representing a co This uses the `sqlite3.Connection.iterdump() `__ method. -If you are using ``pysqlite3`` or ``sqlean.py`` the underlying method may be missing. If you install the `sqlite-dump `__ package then the ``db.iterdump()`` method will use that implementation instead: +If you are using ``pysqlite3`` the underlying method may be missing. If you install the `sqlite-dump `__ package then the ``db.iterdump()`` method will use that implementation instead: .. code-block:: bash diff --git a/mypy.ini b/mypy.ini index de0dc83..2f6a875 100644 --- a/mypy.ini +++ b/mypy.ini @@ -16,9 +16,6 @@ ignore_errors = True [mypy-pysqlite3.*] ignore_missing_imports = True -[mypy-sqlean.*] -ignore_missing_imports = True - [mypy-sqlite_dump.*] ignore_missing_imports = True diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 0ca98fc..865ee79 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -43,15 +43,10 @@ else: dbapi2 = importlib.import_module("pysqlite3.dbapi2") OperationalError = dbapi2.OperationalError except ImportError: - try: - sqlite3 = importlib.import_module("sqlean") - dbapi2 = importlib.import_module("sqlean.dbapi2") - OperationalError = dbapi2.OperationalError - except ImportError: - import sqlite3 # noqa: F401 - from sqlite3 import dbapi2 # noqa: F401 + import sqlite3 # noqa: F401 + from sqlite3 import dbapi2 # noqa: F401 - OperationalError = dbapi2.OperationalError + OperationalError = dbapi2.OperationalError SPATIALITE_PATHS = ( diff --git a/tests/test_gis.py b/tests/test_gis.py index 1b5ed70..e8f4c08 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -6,11 +6,6 @@ from sqlite_utils.cli import cli from sqlite_utils.db import Database from sqlite_utils.utils import find_spatialite, sqlite3 -try: - import sqlean # type: ignore[import-not-found] -except ImportError: - sqlean = None - pytestmark = [ pytest.mark.skipif( @@ -20,9 +15,6 @@ pytestmark = [ not hasattr(sqlite3.Connection, "enable_load_extension"), reason="sqlite3.Connection missing enable_load_extension", ), - pytest.mark.skipif( - sqlean is not None, reason="sqlean.py is not compatible with SpatiaLite" - ), ] From a00ed60efc3a242de38f666bc4ce1e431f86dcea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 21:17:41 -0700 Subject: [PATCH 020/110] Apply Black to tests/test_gis.py Fixes formatting left behind by afbfd95, which was merged while the Black check on its pull request was still failing. Co-Authored-By: Claude Fable 5 --- tests/test_gis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_gis.py b/tests/test_gis.py index e8f4c08..f39554e 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -6,7 +6,6 @@ from sqlite_utils.cli import cli from sqlite_utils.db import Database from sqlite_utils.utils import find_spatialite, sqlite3 - pytestmark = [ pytest.mark.skipif( not find_spatialite(), reason="Could not find SpatiaLite extension" From 07b603e562af19f80c3d00eb59b9cf29331127ce Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 21:20:39 -0700 Subject: [PATCH 021/110] Preserve duplicate column names in query results Queries returning duplicate column names - e.g. joins between tables sharing column names - silently lost values because rows were built with dict(zip(keys, row)), where the last duplicate wins. Later occurrences are now renamed with a numeric suffix: id, id becomes id, id_2 - skipping any suffix that would collide with a real column in the same query. The new utils.dedupe_keys() helper transforms the key list once per query, so the per-row dict construction is unchanged and there is no measurable performance impact. Applied in Database.query() (including the PRAGMA and RETURNING paths), Table.rows_where(), Table.search() and the CLI's JSON output. CSV, TSV and table output keep the original duplicate headers. Closes #624 --- docs/cli.rst | 2 ++ docs/python-api.rst | 11 +++++++++++ sqlite_utils/cli.py | 5 +++++ sqlite_utils/db.py | 9 +++++---- sqlite_utils/utils.py | 31 +++++++++++++++++++++++++++++++ tests/test_cli.py | 20 ++++++++++++++++++++ tests/test_fts.py | 14 ++++++++++++++ tests/test_query.py | 16 ++++++++++++++++ tests/test_rows.py | 7 +++++++ tests/test_utils.py | 17 +++++++++++++++++ 10 files changed, 128 insertions(+), 4 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index c9389d8..84a65d9 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -45,6 +45,8 @@ The default format returned for queries is JSON: [{"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}] +If the query returns more than one column with the same name, later occurrences are renamed with a numeric suffix - ``select 1 as id, 2 as id`` returns ``[{"id": 1, "id_2": 2}]``. This only applies to JSON output: :ref:`CSV and TSV ` and :ref:`table ` output keep the duplicate column headers unchanged. + .. _cli_query_nl: Newline-delimited JSON diff --git a/docs/python-api.rst b/docs/python-api.rst index 6723efb..7ac3951 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -233,6 +233,17 @@ The SQL query is executed as soon as ``db.query()`` is called. The resulting row ``db.query()`` can only be used with SQL that returns rows. Passing a statement that returns no rows - an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause, for example - will raise a ``ValueError``. The rejected statement is rolled back, so it has no effect on the database. Use :ref:`db.execute() ` for those statements instead. +If a query returns more than one column with the same name - a join between two tables that share column names, for example - later occurrences are renamed with a numeric suffix, so every value is included in the dictionary: + +.. code-block:: python + + row = next(db.query("select 1 as id, 2 as id, 3 as id")) + print(row) + # Outputs: + # {'id': 1, 'id_2': 2, 'id_3': 3} + +A suffix that would collide with another column in the query is skipped - ``select 1 as id, 2 as id, 3 as id_2`` returns ``{'id': 1, 'id_3': 2, 'id_2': 3}``. The same renaming is applied by ``table.rows_where()`` and ``table.search()``. + .. _python_api_execute: db.execute(sql, params) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index a30f79a..d5a4b1e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -34,6 +34,7 @@ from .utils import ( OperationalError, _compile_code, chunks, + dedupe_keys, file_progress, find_spatialite, flatten as _flatten, @@ -3493,6 +3494,10 @@ FILE_COLUMNS = { def output_rows(iterator, headers, nl, arrays, json_cols): + # Duplicate column names would collide as dictionary keys, so rename + # later occurrences id, id -> id, id_2 - CSV and table output keep + # the original duplicate headers since they never build dictionaries + headers = dedupe_keys(headers) # We have to iterate two-at-a-time so we can know if we # should output a trailing comma or if we have reached # the last row. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3f23c7d..d009f35 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,5 +1,6 @@ from .utils import ( chunks, + dedupe_keys, hash_record, sqlite3, OperationalError, @@ -786,7 +787,7 @@ class Database: cursor = self.conn.execute(sql, *args) if cursor.description is None: raise ValueError(message) - keys = [d[0] for d in cursor.description] + keys = dedupe_keys(d[0] for d in cursor.description) return (dict(zip(keys, row)) for row in cursor) # Execute inside a savepoint, so a statement that turns out not to # return rows can be rolled back before the ValueError is raised @@ -796,7 +797,7 @@ class Database: cursor = self.conn.execute(sql, *args) if cursor.description is None: raise ValueError(message) - keys = [d[0] for d in cursor.description] + keys = dedupe_keys(d[0] for d in cursor.description) try: self.conn.execute('RELEASE "sqlite_utils_query"') released = True @@ -1865,7 +1866,7 @@ class Queryable: if offset is not None: sql += " offset {}".format(offset) cursor = self.db.execute(sql, where_args or []) - columns = [c[0] for c in cursor.description] + columns = dedupe_keys(c[0] for c in cursor.description) for row in cursor: yield dict(zip(columns, row)) @@ -3398,7 +3399,7 @@ class Table(Queryable): ), args, ) - columns = [c[0] for c in cursor.description] + columns = dedupe_keys(c[0] for c in cursor.description) for row in cursor: yield dict(zip(columns, row)) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 865ee79..b39b117 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -613,6 +613,37 @@ def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> ).hexdigest() +def dedupe_keys(keys: Iterable[str]) -> List[str]: + """ + Rename duplicates in a list of column names so every name is unique, + by appending ``_2``, ``_3``... to later occurrences - skipping any + suffix that would collide with another column in the list. + + Used when converting SQL query rows to dictionaries, where duplicate + column names would otherwise silently overwrite each other. + + :param keys: List of column names, possibly containing duplicates + """ + keys = list(keys) + taken = set(keys) + if len(taken) == len(keys): + # No duplicates - the common case + return keys + seen: set = set() + result = [] + for key in keys: + if key in seen: + new_key = key + suffix = 2 + while new_key in seen or new_key in taken: + new_key = "{}_{}".format(key, suffix) + suffix += 1 + key = new_key + seen.add(key) + result.append(key) + return result + + def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]: for key, value in d.items(): if isinstance(value, dict): diff --git a/tests/test_cli.py b/tests/test_cli.py index d26e4dd..f19a00c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -746,6 +746,26 @@ def test_query_json_empty(db_path): assert result.output.strip() == "[]" +def test_query_json_duplicate_columns_are_deduped(db_path): + # https://github.com/simonw/sqlite-utils/issues/624 + result = CliRunner().invoke( + cli.cli, + [db_path, "select 1 as id, 2 as id, 'x' as value, 'y' as value"], + ) + assert result.output.strip() == ( + '[{"id": 1, "id_2": 2, "value": "x", "value_2": "y"}]' + ) + + +def test_query_csv_duplicate_columns_are_preserved(db_path): + # CSV output should keep the duplicate headers, not rename them + result = CliRunner().invoke( + cli.cli, + [db_path, "select 1 as id, 2 as id", "--csv"], + ) + assert result.output.replace("\r", "").strip() == "id,id\n1,2" + + def test_query_invalid_function(db_path): result = CliRunner().invoke( cli.cli, [db_path, "select bad()", "--functions", "def invalid_python"] diff --git a/tests/test_fts.py b/tests/test_fts.py index 3f7c5a9..64ec645 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -83,6 +83,20 @@ def test_enable_fts_escape_table_names(fresh_db): assert [] == list(table.search("bar")) +def test_search_duplicate_columns_are_deduped(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/624 + table = fresh_db["t"] + table.insert_all(search_records) + table.enable_fts(["text", "country"], fts_version="FTS4") + rows = list(table.search("tanuki", columns=["text", "text"])) + assert rows == [ + { + "text": "tanuki are running tricksters", + "text_2": "tanuki are running tricksters", + } + ] + + def test_search_limit_offset(fresh_db): table = fresh_db["t"] table.insert_all(search_records) diff --git a/tests/test_query.py b/tests/test_query.py index b9822e1..0b9f2ae 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -218,6 +218,22 @@ def test_query_insert_returning_respects_explicit_transaction(fresh_db): assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] +def test_query_duplicate_column_names_are_deduped(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/624 + fresh_db["one"].insert({"id": 1, "value": "left"}) + fresh_db["two"].insert({"id": 2, "value": "right"}) + rows = list( + fresh_db.query("select one.id, two.id, one.value, two.value from one, two") + ) + assert rows == [{"id": 1, "id_2": 2, "value": "left", "value_2": "right"}] + + +def test_query_deduped_column_avoids_existing_names(fresh_db): + # The renamed duplicate must not overwrite a real column called id_2 + rows = list(fresh_db.query("select 1 as id, 2 as id, 3 as id_2")) + assert rows == [{"id": 1, "id_3": 2, "id_2": 3}] + + def test_execute_returning_dicts(fresh_db): # Like db.query() but returns a list, included for backwards compatibility # see https://github.com/simonw/sqlite-utils/issues/290 diff --git a/tests/test_rows.py b/tests/test_rows.py index a8a4ca0..f050d5a 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -104,3 +104,10 @@ def test_pks_and_rows_where_compound_pk(fresh_db): (("number", 1), {"type": "number", "number": 1, "plusone": 2}), (("number", 2), {"type": "number", "number": 2, "plusone": 3}), ] + + +def test_rows_where_duplicate_select_columns_are_deduped(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/624 + fresh_db["t"].insert({"id": 1, "name": "Cleo"}) + rows = list(fresh_db["t"].rows_where(select="id, id, name")) + assert rows == [{"id": 1, "id_2": 1, "name": "Cleo"}] diff --git a/tests/test_utils.py b/tests/test_utils.py index f728bcd..3de5e94 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -83,3 +83,20 @@ def test_maximize_csv_field_size_limit(): ) def test_flatten(input, expected): assert utils.flatten(input) == expected + + +@pytest.mark.parametrize( + "input,expected", + ( + ([], []), + (["id", "name"], ["id", "name"]), + (["id", "id"], ["id", "id_2"]), + (["id", "id", "id"], ["id", "id_2", "id_3"]), + # A renamed duplicate must not clobber a real column called id_2 + (["id", "id", "id_2"], ["id", "id_3", "id_2"]), + (["id_2", "id", "id"], ["id_2", "id", "id_3"]), + (["id", "id", "id_2", "id_2"], ["id", "id_3", "id_2", "id_2_2"]), + ), +) +def test_dedupe_keys(input, expected): + assert utils.dedupe_keys(input) == expected From 02281f77ed7bfb56ff74a2f09a00aa03298e3268 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 21:42:22 -0700 Subject: [PATCH 022/110] Vendor SQLite version setup action (#775) Vendor https://github.com/asg017/sqlite-versions/tree/71ea0de37ae739c33e447af91ba71dda8fcf22e6 and make it more robust against `sqlite.org` timeouts. Closes #774 --- .../actions/setup-sqlite-version/action.yml | 39 +++++ .../setup-sqlite-version.sh | 144 ++++++++++++++++++ .github/workflows/test-sqlite-support.yml | 6 +- 3 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 .github/actions/setup-sqlite-version/action.yml create mode 100644 .github/actions/setup-sqlite-version/setup-sqlite-version.sh diff --git a/.github/actions/setup-sqlite-version/action.yml b/.github/actions/setup-sqlite-version/action.yml new file mode 100644 index 0000000..fdbc71c --- /dev/null +++ b/.github/actions/setup-sqlite-version/action.yml @@ -0,0 +1,39 @@ +name: "Setup SQLite version" +description: "Build and activate a specific SQLite version from its amalgamation archive" +inputs: + version: + description: "The SQLite version to install" + required: true + cflags: + description: "CFLAGS to use when compiling SQLite" + required: false + default: "" + skip-activate: + description: "Set to true to skip modifying the library path" + required: false + default: "false" + fallback-urls: + description: "Whitespace-separated fallback download URLs to try after sqlite.org" + required: false + default: "" +outputs: + sqlite-location: + description: "Directory containing the compiled SQLite library" + value: ${{ steps.build.outputs.sqlite-location }} +runs: + using: "composite" + steps: + - shell: bash + run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads" + - uses: actions/cache@v6 + with: + path: ${{ runner.temp }}/sqlite-versions/downloads + key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1 + - id: build + shell: bash + run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh" + env: + SQLITE_VERSION: ${{ inputs.version }} + SQLITE_CFLAGS: ${{ inputs.cflags }} + SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }} + SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }} diff --git a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh new file mode 100644 index 0000000..0df7290 --- /dev/null +++ b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}" +cflags="${SQLITE_CFLAGS:-}" +skip_activate="${SQLITE_SKIP_ACTIVATE:-false}" +extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}" + +case "$version_spec" in + 3.46 | 3.46.0) + sqlite_version="3.46.0" + sqlite_year="2024" + amalgamation_id="3460000" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip" + ;; + 3.23.1) + sqlite_version="3.23.1" + sqlite_year="2018" + amalgamation_id="3230100" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3230100.zip" + ;; + *) + echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh." + exit 1 + ;; +esac + +case "$(uname -s)" in + Linux) + library_name="libsqlite3.so.0" + library_path_var="LD_LIBRARY_PATH" + ;; + Darwin) + library_name="libsqlite3.dylib" + library_path_var="DYLD_LIBRARY_PATH" + ;; + *) + echo "::error::Unsupported platform $(uname -s)" + exit 1 + ;; +esac + +runner_temp="${RUNNER_TEMP:-}" +if [ -z "$runner_temp" ]; then + runner_temp="$(mktemp -d)" +fi + +filename="sqlite-amalgamation-${amalgamation_id}" +official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip" +download_dir="${runner_temp}/sqlite-versions/downloads" +source_root="${runner_temp}/sqlite-versions/source" +source_dir="${source_root}/${filename}" +build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}" +archive_path="${download_dir}/${filename}.zip" + +mkdir -p "$download_dir" "$source_root" "$build_dir" + +download_archive() { + local url + local candidate_path="${archive_path}.tmp" + local urls=("$official_url") + + for url in $builtin_fallback_urls $extra_fallback_urls; do + urls+=("$url") + done + + rm -f "$candidate_path" + for url in "${urls[@]}"; do + echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}" + if curl \ + --fail \ + --location \ + --show-error \ + --retry 5 \ + --retry-delay 2 \ + --retry-max-time 180 \ + --retry-all-errors \ + --connect-timeout 20 \ + --max-time 240 \ + --output "$candidate_path" \ + "$url"; then + mv "$candidate_path" "$archive_path" + return 0 + fi + + echo "::warning::Download failed from ${url}" + rm -f "$candidate_path" + done + + echo "::error::Could not download SQLite ${sqlite_version} amalgamation" + return 1 +} + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + if [ ! -f "$archive_path" ]; then + download_archive + fi + + rm -rf "$source_dir" + unzip -q "$archive_path" -d "$source_root" +fi + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}" + exit 1 +fi + +read -r -a cflag_args <<< "$cflags" + +echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}" +gcc \ + -fPIC \ + -shared \ + "${cflag_args[@]}" \ + "${source_dir}/sqlite3.c" \ + "-I${source_dir}" \ + -o "${build_dir}/${library_name}" + +if [ "$library_name" = "libsqlite3.so.0" ]; then + ln -sf "$library_name" "${build_dir}/libsqlite3.so" +fi + +if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT" +else + echo "sqlite-location=${build_dir}" +fi + +case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in + true | 1 | yes) + echo "Skipping ${library_path_var} activation" + ;; + *) + existing_value="${!library_path_var:-}" + if [ -n "${GITHUB_ENV:-}" ]; then + if [ -n "$existing_value" ]; then + echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV" + else + echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV" + fi + fi + echo "Added ${build_dir} to ${library_path_var}" + ;; +esac diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index aa2ab9b..f195cba 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -18,16 +18,16 @@ jobs: "3.23.1", # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml - name: Set up SQLite ${{ matrix.sqlite-version }} - uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 + uses: ./.github/actions/setup-sqlite-version with: version: ${{ matrix.sqlite-version }} cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" From 50938ee6f846ddd792921f5a0353c73134dbbeda Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 21:54:30 -0700 Subject: [PATCH 023/110] Rename ensure_autocommit_off() to ensure_autocommit_on(), closes #705 --- docs/changelog.rst | 1 + docs/upgrading.rst | 2 ++ sqlite_utils/db.py | 20 +++++++++++++------- tests/test_wal.py | 11 +++++++++++ 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7efeb95..0220967 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Breaking changes: - ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) - Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library. +- The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``, because the old name described the opposite of what it did. The method temporarily puts the connection into driver-level autocommit mode - by setting ``isolation_level = None`` - so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. (:issue:`705`) Compound foreign key support: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index e7fd273..09a9e2f 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -77,6 +77,8 @@ Python API changes **table.convert() no longer skips falsey values.** Matching the CLI change above, ``table.convert()`` now converts every value. The ``skip_false`` parameter has been removed - previously it defaulted to ``True``, skipping empty strings and other falsey values. +**ensure_autocommit_off() is now ensure_autocommit_on().** The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``. The old name described the opposite of what the method did: it temporarily puts the connection into driver-level autocommit mode (by setting ``isolation_level = None``), so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. The behavior is unchanged - update any calls to use the new name. + **View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method. **ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained new fields - ``columns``, ``other_columns``, ``is_compound``, ``on_delete`` and ``on_update`` - so that compound (multi-column) foreign keys and foreign key actions can be represented. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d009f35..d984bcf 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -610,16 +610,22 @@ class Database: self.conn.execute("ROLLBACK") @contextlib.contextmanager - def ensure_autocommit_off(self) -> Generator[None, None, None]: + def ensure_autocommit_on(self) -> Generator[None, None, None]: """ - Ensure autocommit is off for this database connection. + Ensure the connection is in driver-level autocommit mode for the + duration of a block of code. + + This temporarily sets ``isolation_level = None`` on the underlying + ``sqlite3`` connection, so the driver does not open implicit + transactions. This is useful for statements such as + ``PRAGMA journal_mode=wal`` which cannot run inside a transaction. Example usage:: - with db.ensure_autocommit_off(): + with db.ensure_autocommit_on(): # do stuff here - This will reset to the previous autocommit state at the end of the block. + The previous ``isolation_level`` is restored at the end of the block. """ old_isolation_level = self.conn.isolation_level try: @@ -783,7 +789,7 @@ class Database: if self.conn.in_transaction: cursor = self.conn.execute(sql, *args) else: - with self.ensure_autocommit_off(): + with self.ensure_autocommit_on(): cursor = self.conn.execute(sql, *args) if cursor.description is None: raise ValueError(message) @@ -1085,7 +1091,7 @@ class Database: """ if self.journal_mode != "wal": self._ensure_no_open_transaction("enable_wal()") - with self.ensure_autocommit_off(): + with self.ensure_autocommit_on(): self.execute("PRAGMA journal_mode=wal;") def disable_wal(self) -> None: @@ -1097,7 +1103,7 @@ class Database: """ if self.journal_mode != "delete": self._ensure_no_open_transaction("disable_wal()") - with self.ensure_autocommit_off(): + with self.ensure_autocommit_on(): self.execute("PRAGMA journal_mode=delete;") def _ensure_no_open_transaction(self, operation: str) -> None: diff --git a/tests/test_wal.py b/tests/test_wal.py index ee7ecf0..c5a9c60 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -49,6 +49,17 @@ def test_disable_wal_inside_transaction_raises(db_path_tmpdir): assert [r["id"] for r in db["test"].rows] == [1] +def test_ensure_autocommit_on(db_path_tmpdir): + db, path, tmpdir = db_path_tmpdir + previous_isolation_level = db.conn.isolation_level + assert previous_isolation_level is not None + with db.ensure_autocommit_on(): + # isolation_level of None means driver-level autocommit mode + assert db.conn.isolation_level is None + # Restored afterwards + assert db.conn.isolation_level == previous_isolation_level + + def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): # Calling enable_wal() when WAL is already enabled is a no-op, # so it is fine inside a transaction From 77d241959c90d34f3b2122c57a5f134ead61f9f0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 22:11:22 -0700 Subject: [PATCH 024/110] Match column names case-insensitively, closes #760 Column names passed to Python API methods are now resolved against the table schema case-insensitively, mirroring how SQLite itself compares identifiers (ASCII-only case folding). Fixes KeyError populating last_pk from insert()/upsert(), silently ignored transform() rename/drop/types options, duplicate column errors from create_table(transform=True), redundant lookup() indexes, and case-sensitive foreign key validation. Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 13 ++ sqlite_utils/db.py | 238 +++++++++++++++++++++++++++++------- tests/test_column_casing.py | 233 +++++++++++++++++++++++++++++++++++ 3 files changed, 442 insertions(+), 42 deletions(-) create mode 100644 tests/test_column_casing.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 0220967..0614dcd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,19 @@ Other foreign key improvements: - Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``. - Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound. +Case-insensitive column matching: + +Column names passed to Python API methods are now matched against the table schema case-insensitively, mirroring how SQLite itself treats identifiers. Previously many methods accepted mixed-case identifiers in the SQL they generated but then failed - or silently did nothing - when performing Python-side comparisons against the schema. (:issue:`760`) Fixes include: + +- ``table.insert()`` and ``table.upsert()`` now populate ``table.last_pk`` correctly when the ``pk=`` argument uses different casing to the table schema or the record keys - previously this raised a ``KeyError`` after the row had already been written. +- Upserts no longer raise or misbehave when the casing of ``pk=`` differs from the casing of the record keys. The primary key columns are correctly excluded from the generated ``DO UPDATE SET`` clause. +- ``table.transform()`` arguments ``types=``, ``rename=``, ``drop=``, ``pk=``, ``not_null=``, ``defaults=``, ``column_order=`` and ``drop_foreign_keys=`` all resolve column names case-insensitively. Previously options like ``rename={"name": "title"}`` against a column called ``Name`` were silently ignored. +- ``db.create_table(..., transform=True)`` now recognizes existing columns that differ only by case, instead of attempting to add them again and failing with ``duplicate column name``. The casing used in the existing schema is preserved. +- ``table.lookup()`` returns the primary key value even if ``pk=`` casing differs from the schema, and recognizes existing unique indexes case-insensitively instead of creating redundant ones. +- ``table.extract()`` and ``table.convert()`` - including ``multi=True`` and ``output=`` - accept column names in any casing. +- Foreign key columns are validated and recorded using the casing of the actual schema columns, in ``foreign_keys=`` when creating tables, ``db.add_foreign_keys()``, ``table.add_foreign_key()`` and ``table.add_column(fk_col=...)``. Duplicate foreign key detection is also case-insensitive. +- ``table.create()`` with ``pk=``, ``not_null=``, ``defaults=`` or ``column_order=`` referencing columns using different casing no longer creates an unwanted extra primary key column or raises a ``ValueError``. + .. _v4_0rc2: 4.0rc2 (2026-07-04) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d984bcf..90d49c6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -85,6 +85,34 @@ def quote_identifier(identifier: str) -> str: return '"{}"'.format(identifier.replace('"', '""')) +_IDENTIFIER_CASEFOLD = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" +) + + +def fold_identifier_case(identifier: str) -> str: + """ + Lowercase an identifier using the same rules SQLite uses - only ASCII + characters are folded, other characters are left unchanged. + """ + return identifier.translate(_IDENTIFIER_CASEFOLD) + + +def resolve_casing(name: str, candidates: Iterable[str]) -> str: + """ + SQLite treats identifiers as case-insensitive. Return the entry in + ``candidates`` that matches ``name`` case-insensitively, preferring an + exact match. If nothing matches, return ``name`` unchanged. + """ + if name in candidates: + return name + folded = fold_identifier_case(name) + for candidate in candidates: + if fold_identifier_case(candidate) == folded: + return candidate + return name + + pd: Any = None try: pd = importlib.import_module("pandas") @@ -1263,6 +1291,48 @@ class Database: ) return fks + def _resolve_foreign_key_casing( + self, fk: ForeignKey, columns: Iterable[str] + ) -> ForeignKey: + """ + Return ``fk`` with its column references resolved to match the casing + of the actual columns. ``columns`` provides the column names of + ``fk.table``, which may be a table that is still being created. + """ + resolved_columns = tuple(resolve_casing(c, columns) for c in fk.columns) + if fk.other_table == fk.table: + other_candidates: Iterable[str] = columns + else: + other_candidates = self[fk.other_table].columns_dict + resolved_other_columns = tuple( + resolve_casing(c, other_candidates) for c in fk.other_columns + ) + if ( + resolved_columns == fk.columns + and resolved_other_columns == fk.other_columns + ): + return fk + if fk.is_compound: + return ForeignKey( + fk.table, + None, + fk.other_table, + None, + columns=resolved_columns, + other_columns=resolved_other_columns, + is_compound=True, + on_delete=fk.on_delete, + on_update=fk.on_update, + ) + return ForeignKey( + fk.table, + resolved_columns[0], + fk.other_table, + resolved_other_columns[0], + on_delete=fk.on_delete, + on_update=fk.on_update, + ) + def create_table_sql( self, name: str, @@ -1296,11 +1366,14 @@ class Database: """ if hash_id_columns and (hash_id is None): hash_id = "id" - foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) + resolved_fks: List[ForeignKey] = [ + self._resolve_foreign_key_casing(fk, columns) + for fk in self.resolve_foreign_keys(name, foreign_keys or []) + ] # Compound foreign keys are rendered as table-level constraints; # single-column ones as inline REFERENCES on their column foreign_keys_by_column = { - fk.column: fk for fk in foreign_keys if not fk.is_compound + fk.column: fk for fk in resolved_fks if not fk.is_compound } # any extracts will be treated as integer columns with a foreign key extracts = resolve_extracts(extracts) @@ -1315,8 +1388,10 @@ class Database: name, extract_column, extract_table, "id" ) # Soundness check not_null, and defaults if provided - not_null = not_null or set() - defaults = defaults or {} + not_null = {resolve_casing(n, columns) for n in not_null or set()} + defaults = {resolve_casing(n, columns): v for n, v in (defaults or {}).items()} + if column_order is not None: + column_order = [resolve_casing(c, columns) for c in column_order] if not columns: raise ValueError("Tables must have at least one column") if not all(n in columns for n in not_null): @@ -1342,7 +1417,7 @@ class Database: column_items.insert(0, (hash_id, str)) pk = hash_id # Soundness check foreign_keys point to existing tables - for fk in foreign_keys: + for fk in resolved_fks: for other_column in fk.other_columns: if fk.other_table == name and columns.get(other_column): continue @@ -1356,12 +1431,14 @@ class Database: column_defs = [] # ensure pk is a tuple single_pk = None - if isinstance(pk, list) and len(pk) == 1 and isinstance(pk[0], str): + if isinstance(pk, (list, tuple)) and len(pk) == 1 and isinstance(pk[0], str): pk = pk[0] if isinstance(pk, str): - single_pk = pk + single_pk = pk = resolve_casing(pk, [c[0] for c in column_items]) if pk not in [c[0] for c in column_items]: column_items.insert(0, (pk, int)) + elif pk: + pk = [resolve_casing(p, [c[0] for c in column_items]) for p in pk] for column_name, column_type in column_items: column_extras = [] if column_name == single_pk: @@ -1402,7 +1479,7 @@ class Database: ) # Compound foreign keys become table-level FOREIGN KEY constraints column_names = [c[0] for c in column_items] - for fk in foreign_keys: + for fk in resolved_fks: if not fk.is_compound: continue missing = [c for c in fk.columns if c not in column_names] @@ -1483,6 +1560,11 @@ class Database: should_transform = False # First add missing columns and figure out columns to drop existing_columns = table.columns_dict + # Match existing columns case-insensitively, the way SQLite does + columns = { + resolve_casing(col_name, existing_columns): col_type + for col_name, col_type in columns.items() + } missing_columns = dict( (col_name, col_type) for col_name, col_type in columns.items() @@ -1506,18 +1588,28 @@ class Database: current_pks = table.pks desired_pk = None if isinstance(pk, str): - desired_pk = [pk] + desired_pk = [resolve_casing(pk, existing_columns)] elif pk: - desired_pk = list(pk) + desired_pk = [resolve_casing(p, existing_columns) for p in pk] if desired_pk and current_pks != desired_pk: should_transform = True # Any not-null changes? current_not_null = {c.name for c in table.columns if c.notnull} - desired_not_null = set(not_null) if not_null else set() + desired_not_null = ( + {resolve_casing(n, existing_columns) for n in not_null} + if not_null + else set() + ) if current_not_null != desired_not_null: should_transform = True # How about defaults? - if defaults and defaults != table.default_values: + if ( + defaults + and { + resolve_casing(c, existing_columns): v for c, v in defaults.items() + } + != table.default_values + ): should_transform = True # Only run .transform() if there is something to do if should_transform: @@ -1671,12 +1763,15 @@ class Database: is_compound=True, ) table = fk_object.table - columns = fk_object.columns other_table = fk_object.other_table - other_columns = fk_object.other_columns if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) table_obj = self.table(table) + fk_object = self._resolve_foreign_key_casing( + fk_object, table_obj.columns_dict + ) + columns = fk_object.columns + other_columns = fk_object.other_columns for column in columns: if column not in table_obj.columns_dict: raise AlterError("No such column: {} in {}".format(column, table)) @@ -1693,12 +1788,16 @@ class Database: ) ) # We will silently skip foreign keys that exist already + columns_folded = tuple(fold_identifier_case(c) for c in columns) + other_columns_folded = tuple(fold_identifier_case(c) for c in other_columns) if not any( fk for fk in table_obj.foreign_keys - if fk.columns == columns - and fk.other_table == other_table - and fk.other_columns == other_columns + if tuple(fold_identifier_case(c) for c in fk.columns) == columns_folded + and fold_identifier_case(fk.other_table) + == fold_identifier_case(other_table) + and tuple(fold_identifier_case(c) for c in fk.other_columns) + == other_columns_folded ): foreign_keys_to_create.append(fk_object) @@ -2438,6 +2537,31 @@ class Table(Queryable): rename = rename or {} drop = drop or set() + # Resolve column references against the existing schema, matching + # case-insensitively the way SQLite does + existing_columns = self.columns_dict + types = {resolve_casing(c, existing_columns): t for c, t in types.items()} + rename = {resolve_casing(c, existing_columns): v for c, v in rename.items()} + drop = {resolve_casing(c, existing_columns) for c in drop} + if pk is not DEFAULT and pk is not None: + if isinstance(pk, str): + pk = resolve_casing(pk, existing_columns) + else: + pk = [resolve_casing(p, existing_columns) for p in pk] + if isinstance(not_null, dict): + not_null = { + resolve_casing(c, existing_columns): v + for c, v in cast(Dict[str, Any], not_null).items() + } + elif isinstance(not_null, set): + not_null = {resolve_casing(c, existing_columns) for c in not_null} + if defaults is not None: + defaults = { + resolve_casing(c, existing_columns): v for c, v in defaults.items() + } + if column_order is not None: + column_order = [resolve_casing(c, existing_columns) for c in column_order] + create_table_foreign_keys: List[ForeignKeyIndicator] = [] if foreign_keys is not None: @@ -2452,28 +2576,37 @@ class Table(Queryable): create_table_foreign_keys.extend(foreign_keys) else: # Construct foreign_keys from current, plus add_foreign_keys, minus drop_foreign_keys - # Bind fresh names here - type checkers do not narrow captured - # variables inside nested functions, so the closures would - # otherwise see the Optional declared types of drop and rename - dropped_columns = drop - renamed_columns = rename + # The casing of columns in a foreign key definition can differ + # from the casing of the columns themselves, so these comparisons + # are all case-folded + dropped_columns_folded = {fold_identifier_case(c) for c in drop} + renamed_columns_folded = { + fold_identifier_case(k): v for k, v in rename.items() + } def fk_should_be_dropped(fk: ForeignKey) -> bool: + fk_columns_folded = tuple(fold_identifier_case(c) for c in fk.columns) if drop_foreign_keys is not None: for spec in drop_foreign_keys: if isinstance(spec, str): # A column name matches any foreign key it participates in - if spec in fk.columns: + if fold_identifier_case(spec) in fk_columns_folded: return True - elif tuple(spec) == fk.columns: + elif ( + tuple(fold_identifier_case(s) for s in spec) + == fk_columns_folded + ): # A tuple/list must match a compound key's columns exactly return True # Dropping any of a foreign key's columns drops the whole key - return any(column in dropped_columns for column in fk.columns) + return any( + column in dropped_columns_folded for column in fk_columns_folded + ) def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: columns = tuple( - renamed_columns.get(column) or column for column in fk.columns + renamed_columns_folded.get(fold_identifier_case(column)) or column + for column in fk.columns ) if fk.is_compound: return ForeignKey( @@ -2662,6 +2795,8 @@ class Table(Queryable): rename = rename or {} if isinstance(columns, str): columns = [columns] + columns = [resolve_casing(c, self.columns_dict) for c in columns] + rename = {resolve_casing(k, self.columns_dict): v for k, v in rename.items()} if not set(columns).issubset(self.columns_dict.keys()): raise InvalidColumns( "Invalid columns {} for table with columns {}".format( @@ -2853,6 +2988,7 @@ class Table(Queryable): raise AlterError("table '{}' does not exist".format(fk)) # if fk_col specified, must be a valid column if fk_col is not None: + fk_col = resolve_casing(fk_col, self.db[fk].columns_dict) if fk_col not in self.db[fk].columns_dict: raise AlterError("table '{}' has no column {}".format(fk, fk_col)) else: @@ -2958,6 +3094,7 @@ class Table(Queryable): :param on_update: ``ON UPDATE`` action for the foreign key. """ columns = (column,) if isinstance(column, str) else tuple(column) + columns = tuple(resolve_casing(c, self.columns_dict) for c in columns) # Ensure columns exist for col in columns: if col not in self.columns_dict: @@ -2979,6 +3116,9 @@ class Table(Queryable): other_columns = (other_column,) else: other_columns = tuple(other_column) + other_columns = tuple( + resolve_casing(c, self.db[other_table].columns_dict) for c in other_columns + ) if len(columns) != len(other_columns): raise ValueError( "Compound foreign key must have the same number of columns " @@ -2996,9 +3136,12 @@ class Table(Queryable): if any( fk for fk in self.foreign_keys - if fk.columns == columns - and fk.other_table == other_table - and fk.other_columns == other_columns + if tuple(fold_identifier_case(c) for c in fk.columns) + == tuple(fold_identifier_case(c) for c in columns) + and fold_identifier_case(fk.other_table) + == fold_identifier_case(other_table) + and tuple(fold_identifier_case(c) for c in fk.other_columns) + == tuple(fold_identifier_case(c) for c in other_columns) ): if ignore: return self @@ -3547,6 +3690,7 @@ class Table(Queryable): """ if isinstance(columns, str): columns = [columns] + columns = [resolve_casing(c, self.columns_dict) for c in columns] if multi: return self._convert_multi( @@ -3561,6 +3705,7 @@ class Table(Queryable): if output is not None: if len(columns) != 1: raise ValueError("output= can only be used with a single column") + output = resolve_casing(output, self.columns_dict) if output not in self.columns_dict: self.add_column(output, output_type or "text") @@ -3760,6 +3905,8 @@ class Table(Queryable): # Everything from here on is for upsert=True pk_cols = [pk] if isinstance(pk, str) else list(pk) + # The records may use different casing for the pk columns than pk= + pk_cols = [resolve_casing(c, all_columns) for c in pk_cols] # Every record must provide a value for every primary key column - a # NULL primary key never matches ON CONFLICT, so the record would be # inserted as a brand new row instead of upserted @@ -3810,10 +3957,7 @@ class Table(Queryable): # At this point we need compatibility UPSERT for SQLite < 3.24.0 # (INSERT OR IGNORE + second UPDATE stage) queries_and_params = [] - if isinstance(pk, str): - pks = [pk] - else: - pks = pk + pks = pk_cols self.last_pk = None for record_values in values: record = dict(zip(all_columns, record_values)) @@ -4226,9 +4370,9 @@ class Table(Queryable): if hash_id: self.last_pk = row[hash_id] elif isinstance(pk, str): - self.last_pk = row[pk] + self.last_pk = row[resolve_casing(pk, row)] else: - self.last_pk = tuple(row[p] for p in pk) + self.last_pk = tuple(row[resolve_casing(p, row)] for p in pk) else: self.last_pk = self.last_rowid else: @@ -4240,11 +4384,14 @@ class Table(Queryable): # hash_id not supported in list mode for last_pk pass elif isinstance(pk, str): - pk_index = column_names.index(pk) + pk_index = column_names.index(resolve_casing(pk, column_names)) self.last_pk = first_record_list[pk_index] else: self.last_pk = tuple( - first_record_list[column_names.index(p)] for p in pk + first_record_list[ + column_names.index(resolve_casing(p, column_names)) + ] + for p in pk ) else: first_record_dict = cast(Dict[str, Any], first_record) @@ -4252,9 +4399,12 @@ class Table(Queryable): self.last_pk = hash_record(first_record_dict, hash_id_columns) else: self.last_pk = ( - first_record_dict[pk] + first_record_dict[resolve_casing(pk, first_record_dict)] if isinstance(pk, str) - else tuple(first_record_dict[p] for p in pk) + else tuple( + first_record_dict[resolve_casing(p, first_record_dict)] + for p in pk + ) ) if analyze: @@ -4399,8 +4549,12 @@ class Table(Queryable): combined_values.update(extra_values) if self.exists(): self.add_missing_columns([combined_values]) - unique_column_sets = [set(i.columns) for i in self.indexes] - if set(lookup_values.keys()) not in unique_column_sets: + unique_column_sets = [ + {fold_identifier_case(c) for c in i.columns} for i in self.indexes + ] + if { + fold_identifier_case(c) for c in lookup_values + } not in unique_column_sets: self.create_index(lookup_values.keys(), unique=True) wheres = [ "{} = ?".format(quote_identifier(column)) for column in lookup_values @@ -4411,7 +4565,7 @@ class Table(Queryable): ) ) try: - return rows[0][pk] + return rows[0][resolve_casing(pk, rows[0])] except IndexError: return self.insert( combined_values, diff --git a/tests/test_column_casing.py b/tests/test_column_casing.py new file mode 100644 index 0000000..ce11345 --- /dev/null +++ b/tests/test_column_casing.py @@ -0,0 +1,233 @@ +""" +SQLite treats column names as case-insensitive. These tests exercise the +places where sqlite-utils performs Python-side lookups of column names +provided by the caller, which should match the schema case-insensitively. + +https://github.com/simonw/sqlite-utils/issues/760 +""" + +import pytest + +from sqlite_utils import Database +from sqlite_utils.db import ForeignKey + + +def test_insert_populates_last_pk_case_insensitively(fresh_db): + books = fresh_db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.insert({"Id": 1, "Title": "One"}, pk="id") + assert books.last_pk == 1 + + +def test_insert_populates_last_pk_compound_pk_case_insensitively(fresh_db): + books = fresh_db["books"] + books.create({"Author": str, "Position": int, "Title": str}) + books.insert( + {"Author": "Sue", "Position": 1, "Title": "One"}, pk=("author", "position") + ) + assert books.last_pk == ("Sue", 1) + + +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_pk_case_differs_from_schema(use_old_upsert): + db = Database(memory=True, use_old_upsert=use_old_upsert) + books = db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.insert({"Id": 1, "Title": "One"}) + books.upsert({"id": 1, "title": "Won"}, pk="id") + assert list(books.rows) == [{"Id": 1, "Title": "Won"}] + assert books.last_pk == 1 + + +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_record_key_case_differs_from_pk(use_old_upsert): + # all_columns comes from the record keys, pk= from the caller + db = Database(memory=True, use_old_upsert=use_old_upsert) + books = db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.upsert({"ID": 1, "Title": "One"}, pk="id") + assert list(books.rows) == [{"Id": 1, "Title": "One"}] + assert books.last_pk == 1 + + +def test_upsert_inferred_pk_case_differs_from_record_keys(fresh_db): + # pk is inferred from the existing schema as "Id", records use "id" + books = fresh_db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.upsert({"id": 1, "title": "One"}) + assert list(books.rows) == [{"Id": 1, "Title": "One"}] + assert books.last_pk == 1 + + +def test_upsert_list_mode_pk_case_insensitive(fresh_db): + books = fresh_db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.upsert_all([["id", "title"], [1, "One"]], pk="Id") + assert list(books.rows) == [{"Id": 1, "Title": "One"}] + assert books.last_pk == 1 + + +def test_lookup_pk_case_insensitive(fresh_db): + fresh_db["species"].create({"ID": int, "Name": str}, pk="ID") + fresh_db["species"].insert({"ID": 5, "Name": "Palm"}) + fresh_db["species"].create_index(["Name"], unique=True) + assert fresh_db["species"].lookup({"Name": "Palm"}, pk="id") == 5 + + +def test_lookup_does_not_create_redundant_index(fresh_db): + fresh_db["species"].create({"id": int, "Name": str}, pk="id") + fresh_db["species"].create_index(["Name"], unique=True) + fresh_db["species"].lookup({"name": "Palm"}) + assert len(fresh_db["species"].indexes) == 1 + + +def test_create_table_transform_same_columns_different_case(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db["t"].insert({"Name": "Cleo", "Age": 5}) + fresh_db.create_table("t", {"name": str, "age": int}, transform=True) + # Schema casing is preserved - SQLite considers these the same columns + assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} + assert list(fresh_db["t"].rows) == [{"Name": "Cleo", "Age": 5}] + + +def test_create_table_transform_case_insensitive_with_changes(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db.create_table("t", {"name": str, "age": str, "size": int}, transform=True) + # age changed type, size added, Name untouched + assert fresh_db["t"].columns_dict == {"Name": str, "Age": str, "size": int} + + +def test_transform_types_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str, "Age": str}) + fresh_db["t"].transform(types={"age": int}) + assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} + + +def test_transform_rename_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str}) + fresh_db["t"].transform(rename={"name": "title"}) + assert fresh_db["t"].columns_dict == {"title": str} + + +def test_transform_drop_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db["t"].transform(drop=["name"]) + assert fresh_db["t"].columns_dict == {"Age": int} + + +def test_transform_not_null_and_defaults_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db["t"].transform(not_null={"name"}, defaults={"age": 3}) + columns = {c.name: c for c in fresh_db["t"].columns} + assert columns["Name"].notnull + assert fresh_db["t"].default_values == {"Age": 3} + + +def test_transform_pk_case_insensitive(fresh_db): + fresh_db["t"].create({"Id": int, "Name": str}) + fresh_db["t"].transform(pk="id") + assert fresh_db["t"].pks == ["Id"] + assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} + + +def test_transform_drop_foreign_keys_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create( + {"id": int, "Parent_ID": int}, + pk="id", + foreign_keys=[("Parent_ID", "parent", "Id")], + ) + fresh_db["child"].transform(drop_foreign_keys=["parent_id"]) + assert fresh_db["child"].foreign_keys == [] + + +def test_add_foreign_key_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") + fresh_db["child"].add_foreign_key("parent_id", "parent", "id") + fks = fresh_db["child"].foreign_keys + assert len(fks) == 1 + # The foreign key should use the schema casing of the columns + assert fks[0].column == "Parent_ID" + assert fks[0].other_column == "Id" + + +def test_add_foreign_keys_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") + fresh_db.add_foreign_keys([("child", "parent_id", "parent", "id")]) + fks = fresh_db["child"].foreign_keys + assert len(fks) == 1 + assert fks[0].column == "Parent_ID" + assert fks[0].other_column == "Id" + + +def test_add_foreign_key_detects_existing_case_insensitively(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create( + {"id": int, "Parent_ID": int}, + pk="id", + foreign_keys=[("Parent_ID", "parent", "Id")], + ) + # ignore=True should treat this as already existing, not add a duplicate + fresh_db["child"].add_foreign_key("parent_id", "parent", "id", ignore=True) + assert len(fresh_db["child"].foreign_keys) == 1 + + +def test_add_column_fk_col_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create({"id": int}, pk="id") + fresh_db["child"].add_column("parent_id", int, fk="parent", fk_col="id") + fks = fresh_db["child"].foreign_keys + assert len(fks) == 1 + assert fks[0].other_column == "Id" + + +def test_extract_case_insensitive(fresh_db): + fresh_db["trees"].insert({"id": 1, "Species": "Palm"}, pk="id") + fresh_db["trees"].extract("species") + assert fresh_db["trees"].columns_dict == {"id": int, "Species_id": int} + assert list(fresh_db["Species"].rows) == [{"id": 1, "Species": "Palm"}] + + +def test_convert_multi_case_insensitive(fresh_db): + fresh_db["t"].insert({"id": 1, "Name": "Cleo"}, pk="id") + fresh_db["t"].convert("name", lambda v: {"upper": v.upper()}, multi=True) + assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "upper": "CLEO"}] + + +def test_convert_output_case_insensitive(fresh_db): + fresh_db["t"].insert({"id": 1, "Name": "Cleo", "Upper": None}, pk="id") + fresh_db["t"].convert("name", lambda v: v.upper(), output="upper") + assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "Upper": "CLEO"}] + + +def test_create_table_sql_pk_case_insensitive(fresh_db): + fresh_db["t"].create({"Id": int, "Name": str}, pk="id") + # Should not have created an extra lowercase "id" column + assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} + assert fresh_db["t"].pks == ["Id"] + + +def test_create_table_not_null_and_defaults_case_insensitive(fresh_db): + fresh_db["t"].create( + {"Name": str, "Age": int}, not_null={"name"}, defaults={"age": 1} + ) + columns = {c.name: c for c in fresh_db["t"].columns} + assert columns["Name"].notnull + assert fresh_db["t"].default_values == {"Age": 1} + + +def test_create_table_foreign_keys_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create( + {"id": int, "Parent_ID": int}, + pk="id", + foreign_keys=[("parent_id", "parent", "id")], + ) + fks = fresh_db["child"].foreign_keys + assert fks == [ + ForeignKey( + table="child", column="Parent_ID", other_table="parent", other_column="Id" + ) + ] From a4acc3958c456298bb017cbdea515fe4d5374ed3 Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Mon, 6 Jul 2026 07:33:55 +0200 Subject: [PATCH 025/110] Fix transform() corrupting TRUE/FALSE/NULL column defaults into strings * Fix transform() corrupting TRUE/FALSE/NULL column defaults quote_default_value() passed keyword-literal defaults (TRUE, FALSE, NULL) through self.quote(), wrapping them in quotes. So transform() rebuilt a column declared "INTEGER DEFAULT TRUE" as "INTEGER DEFAULT 'TRUE'", and a later default insert stored the text 'TRUE' instead of the integer 1 (likewise 'FALSE' for 0 and 'NULL' for null) - silent value corruption. Return these keyword literals unquoted, as already done for the CURRENT_TIME/DATE/TIMESTAMP literals. PR #764 --- docs/changelog.rst | 4 ++++ sqlite_utils/db.py | 5 +++++ tests/test_default_value.py | 5 +++++ tests/test_transform.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0614dcd..43a4653 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -42,6 +42,10 @@ Column names passed to Python API methods are now matched against the table sche - Foreign key columns are validated and recorded using the casing of the actual schema columns, in ``foreign_keys=`` when creating tables, ``db.add_foreign_keys()``, ``table.add_foreign_key()`` and ``table.add_column(fk_col=...)``. Duplicate foreign key detection is also case-insensitive. - ``table.create()`` with ``pk=``, ``not_null=``, ``defaults=`` or ``column_order=`` referencing columns using different casing no longer creates an unwanted extra primary key column or raises a ``ValueError``. +Everything else: + +- Fixed a bug where ``table.transform()`` could convert ``DEFAULT TRUE``, ``DEFAULT FALSE`` and ``DEFAULT NULL`` column defaults into quoted string defaults when rebuilding a table. Thanks, `Vincent Gao `__. (`#764 `__) + .. _v4_0rc2: 4.0rc2 (2026-07-04) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 90d49c6..0807892 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -983,6 +983,11 @@ class Database: if str(value).upper() in ("CURRENT_TIME", "CURRENT_DATE", "CURRENT_TIMESTAMP"): return value + if isinstance(value, str) and value.upper() in ("TRUE", "FALSE", "NULL"): + # Keyword literals must stay unquoted; quoting them would turn the + # default into a string ('TRUE' instead of 1, 'NULL' instead of null). + return value + if str(value).endswith(")"): # Expr return "({})".format(value) diff --git a/tests/test_default_value.py b/tests/test_default_value.py index c5e4b17..3724d99 100644 --- a/tests/test_default_value.py +++ b/tests/test_default_value.py @@ -21,6 +21,11 @@ EXAMPLES = [ # Strings ("TEXT DEFAULT 'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'"), ('TEXT DEFAULT "CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"'), + # Boolean and null keyword literals must stay unquoted + ("INTEGER DEFAULT TRUE", "TRUE", "TRUE"), + ("INTEGER DEFAULT FALSE", "FALSE", "FALSE"), + ("INTEGER DEFAULT true", "true", "true"), + ("TEXT DEFAULT NULL", "NULL", "NULL"), ] diff --git a/tests/test_transform.py b/tests/test_transform.py index 5eb501d..71518be 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -224,6 +224,40 @@ def test_transform_rename_pk(fresh_db): ) +def test_transform_preserves_keyword_literal_defaults(fresh_db): + # transform() used to requote keyword-literal defaults (DEFAULT TRUE became + # DEFAULT 'TRUE'), so a default insert stored the text 'TRUE' instead of the + # integer 1 -- silent value corruption on every rebuilt table. + fresh_db.execute( + "CREATE TABLE t (" + " id INTEGER PRIMARY KEY," + " is_active INTEGER DEFAULT TRUE," + " flag INTEGER DEFAULT FALSE," + " note TEXT DEFAULT NULL" + ")" + ) + table = fresh_db["t"] + table.insert({"id": 1}) + before = fresh_db.execute("SELECT is_active, flag, note FROM t").fetchone() + assert before == (1, 0, None) + + # Rebuild the table via an unrelated change. + table.transform(rename={"note": "note2"}) + + # The keyword literals stay unquoted in the schema ... + assert "DEFAULT TRUE" in table.schema + assert "DEFAULT FALSE" in table.schema + assert "DEFAULT NULL" in table.schema + assert "'TRUE'" not in table.schema + + # ... and a fresh default insert still yields 1 / 0 / NULL, not strings. + table.insert({"id": 2}) + after = fresh_db.execute( + "SELECT is_active, flag, note2 FROM t WHERE id = 2" + ).fetchone() + assert after == (1, 0, None) + + def test_transform_not_null(fresh_db): dogs = fresh_db["dogs"] dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") From 3e8b7403a286e0f9ae4cfb9b8ff2e010c3edb2de Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 22:35:35 -0700 Subject: [PATCH 026/110] Release 4.0rc3 Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4889420844 Refs #530, #594, #624, #705, #760, #764, #769, #770, #771, #772, #774, #775 --- docs/changelog.rst | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 43a4653..4b48a40 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,10 +4,10 @@ Changelog =========== -.. _v_unreleased: +.. _v4_0rc3: -Unreleased ----------- +4.0rc3 (2026-07-05) +------------------- Breaking changes: diff --git a/pyproject.toml b/pyproject.toml index 9d1f041..bff9390 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.0rc2" +version = "4.0rc3" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From d1f5e06816f2fdaa2f626c8cd6814a209bf51a7b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 23:12:54 -0700 Subject: [PATCH 027/110] Add subheadings to 4.0rc3 release notes So I can link to those sections. --- docs/changelog.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4b48a40..6412101 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,27 +9,31 @@ 4.0rc3 (2026-07-05) ------------------- -Breaking changes: +Breaking changes +~~~~~~~~~~~~~~~~ - ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) - Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library. - The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``, because the old name described the opposite of what it did. The method temporarily puts the connection into driver-level autocommit mode - by setting ``isolation_level = None`` - so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. (:issue:`705`) -Compound foreign key support: +Compound foreign key support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Tables can now be created with compound foreign keys, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. See :ref:`python_api_compound_foreign_keys`. - ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely. - ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept tuples of column names to add a compound foreign key to an existing table. - ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key. -Other foreign key improvements: +Other foreign key improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - ``ForeignKey`` now exposes ``on_delete`` and ``on_update`` fields reflecting the foreign key's ``ON DELETE``/``ON UPDATE`` actions, and ``table.transform()`` preserves those actions. Previously a transform silently stripped clauses such as ``ON DELETE CASCADE`` from the table schema. - ``table.add_foreign_key()`` accepts new ``on_delete=`` and ``on_update=`` parameters for creating foreign keys with actions, e.g. ``table.add_foreign_key("author_id", "authors", "id", on_delete="CASCADE")``. (:issue:`530`) - Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``. - Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound. -Case-insensitive column matching: +Case-insensitive column matching +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Column names passed to Python API methods are now matched against the table schema case-insensitively, mirroring how SQLite itself treats identifiers. Previously many methods accepted mixed-case identifiers in the SQL they generated but then failed - or silently did nothing - when performing Python-side comparisons against the schema. (:issue:`760`) Fixes include: @@ -42,7 +46,8 @@ Column names passed to Python API methods are now matched against the table sche - Foreign key columns are validated and recorded using the casing of the actual schema columns, in ``foreign_keys=`` when creating tables, ``db.add_foreign_keys()``, ``table.add_foreign_key()`` and ``table.add_column(fk_col=...)``. Duplicate foreign key detection is also case-insensitive. - ``table.create()`` with ``pk=``, ``not_null=``, ``defaults=`` or ``column_order=`` referencing columns using different casing no longer creates an unwanted extra primary key column or raises a ``ValueError``. -Everything else: +Everything else +~~~~~~~~~~~~~~~ - Fixed a bug where ``table.transform()`` could convert ``DEFAULT TRUE``, ``DEFAULT FALSE`` and ``DEFAULT NULL`` column defaults into quoted string defaults when rebuilding a table. Thanks, `Vincent Gao `__. (`#764 `__) From af3894a096ed44433ca4d40bf7bf701a71f4f097 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 23:19:00 -0700 Subject: [PATCH 028/110] Changelog headings for 4.0rc3 --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6412101..085d9d5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,14 +12,14 @@ Breaking changes ~~~~~~~~~~~~~~~~ -- ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) +- :ref:`table.foreign_keys ` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) - Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library. - The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``, because the old name described the opposite of what it did. The method temporarily puts the connection into driver-level autocommit mode - by setting ``isolation_level = None`` - so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. (:issue:`705`) Compound foreign key support ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Tables can now be created with compound foreign keys, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. See :ref:`python_api_compound_foreign_keys`. +- Tables can now be created with :ref:`compound foreign keys `, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. - ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely. - ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept tuples of column names to add a compound foreign key to an existing table. - ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key. From 5f81752cf5e2a9a3f6ef5bff481887582b71b03f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 23:19:40 -0700 Subject: [PATCH 029/110] Fix for frustrating error in 'just docs' locally fatal: refusing to fetch into branch 'refs/heads/main' checked out at '/Users/simon/Dropbox/dev/sqlite-utils' I'm not going to pretend to understand this fix, it works. --- docs/conf.py | 58 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 04a2301..4f29b39 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from subprocess import Popen, PIPE -from beanbag_docutils.sphinx.ext.github import github_linkcode_resolve +import inspect +from pathlib import Path +from subprocess import Popen, PIPE, check_output +import sys # This file is execfile()d with the current directory set to its # containing dir. @@ -45,14 +47,52 @@ extlinks = { } +def _linkcode_git_ref(): + try: + return check_output(["git", "rev-parse", "HEAD"]).decode("utf8").strip() + except Exception: + return "main" + + def linkcode_resolve(domain, info): - return github_linkcode_resolve( - domain=domain, - info=info, - allowed_module_names=["sqlite_utils"], - github_org_id="simonw", - github_repo_id="sqlite-utils", - branch="main", + if domain != "py": + return None + + module_name = info.get("module") + if not module_name or module_name.split(".")[0] != "sqlite_utils": + return None + + module = sys.modules.get(module_name) + if module is None: + return None + + obj = module + for part in info.get("fullname", "").split("."): + obj = getattr(obj, part, None) + if obj is None: + return None + + if isinstance(obj, property): + obj = obj.fget + + try: + obj = inspect.unwrap(obj) + source_file = inspect.getsourcefile(obj) + _, line_number = inspect.getsourcelines(obj) + except Exception: + return None + + if source_file is None: + return None + + try: + filename = Path(source_file).resolve().relative_to(Path(__file__).parent.parent) + except ValueError: + return None + + return ( + "https://github.com/simonw/sqlite-utils/blob/" + f"{_linkcode_git_ref()}/{filename}#L{line_number}" ) From d516e585433792c2bc2e99f2bc6eaa3227c462ae Mon Sep 17 00:00:00 2001 From: Johnson K C Date: Mon, 6 Jul 2026 09:54:17 -0700 Subject: [PATCH 030/110] Honor --no-headers for --fmt and --table output (#566) (#751) Co-authored-by: Claude Opus 4.8 (1M context) --- docs/cli-reference.rst | 16 +++++++-------- sqlite_utils/cli.py | 18 ++++++++++++++--- tests/test_cli.py | 44 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index cbac48b..1eeaabf 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -116,7 +116,7 @@ See :ref:`cli_query`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, @@ -185,7 +185,7 @@ See :ref:`cli_memory`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, @@ -422,7 +422,7 @@ See :ref:`cli_search`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -688,7 +688,7 @@ See :ref:`cli_tables`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -730,7 +730,7 @@ See :ref:`cli_views`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -777,7 +777,7 @@ See :ref:`cli_rows`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, @@ -817,7 +817,7 @@ See :ref:`cli_triggers`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -857,7 +857,7 @@ See :ref:`cli_indexes`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index d5a4b1e..8918439 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -111,7 +111,11 @@ def output_options(fn): ), click.option("--csv", is_flag=True, help="Output CSV"), click.option("--tsv", is_flag=True, help="Output TSV"), - click.option("--no-headers", is_flag=True, help="Omit CSV headers"), + click.option( + "--no-headers", + is_flag=True, + help="Omit headers from CSV/TSV and table/--fmt output", + ), click.option( "-t", "--table", is_flag=True, help="Output as a formatted table" ), @@ -240,7 +244,13 @@ def tables( yield row if table or fmt: - print(tabulate.tabulate(_iter(), headers=headers, tablefmt=fmt or "simple")) + print( + tabulate.tabulate( + _iter(), + headers=() if no_headers else headers, + tablefmt=fmt or "simple", + ) + ) elif csv or tsv: writer = csv_std.writer(sys.stdout, dialect="excel-tab" if tsv else "excel") if not no_headers: @@ -2145,7 +2155,9 @@ def _execute_query( elif fmt or table: print( tabulate.tabulate( - list(cursor), headers=headers, tablefmt=fmt or "simple" + list(cursor), + headers=() if no_headers else headers, + tablefmt=fmt or "simple", ) ) elif csv or tsv: diff --git a/tests/test_cli.py b/tests/test_cli.py index f19a00c..d11c42f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -195,6 +195,50 @@ def test_output_table(db_path, options, expected): assert expected == result.output.strip() +@pytest.mark.parametrize( + "fmt_option", [["--fmt", "simple"], ["-t"], ["--fmt", "github"]] +) +def test_output_table_no_headers(db_path, fmt_option): + # --no-headers should omit the header row from --fmt/--table output too, not + # just from --csv/--tsv (#566). Previously the flag was silently ignored for + # tabulate formats and the column names were always printed. + db = Database(db_path) + with db.conn: + db["dogs"].insert_all( + [ + {"id": 1, "name": "Cleo", "age": 4}, + {"id": 2, "name": "Pancakes", "age": 2}, + ] + ) + sql = "select id, name, age from dogs order by id" + + with_headers = CliRunner().invoke(cli.cli, ["query", db_path, sql] + fmt_option) + without_headers = CliRunner().invoke( + cli.cli, ["query", db_path, sql] + fmt_option + ["--no-headers"] + ) + assert with_headers.exit_code == 0 + assert without_headers.exit_code == 0 + + # The column names appear when headers are shown, and must not appear at all + # once --no-headers is passed. + assert "name" in with_headers.output + for header in ("id", "name", "age"): + assert ( + header not in without_headers.output + ), f"header {header!r} leaked into --no-headers output" + # The data is still all present. + for value in ("Cleo", "Pancakes", "1", "2", "4"): + assert value in without_headers.output + + # The rows command shares the same code path. + rows_no_headers = CliRunner().invoke( + cli.cli, ["rows", db_path, "dogs"] + fmt_option + ["--no-headers"] + ) + assert rows_no_headers.exit_code == 0 + assert "name" not in rows_no_headers.output + assert "Cleo" in rows_no_headers.output + + def test_create_index(db_path): db = Database(db_path) assert [] == db["Gosh"].indexes From 815b6a7d3dcd262b9ef415482d197a2141b60427 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 11:10:07 -0700 Subject: [PATCH 031/110] JSON output no longer escapes non-ASCII characters, new --ascii option (#777) Closes #625 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JaHan1NhaTRAxJ9LQtSLf9 --- docs/changelog.rst | 7 +++++++ docs/cli-reference.rst | 11 +++++++++++ docs/cli.rst | 27 +++++++++++++++++++++++++++ sqlite_utils/cli.py | 35 +++++++++++++++++++++++++++++------ tests/test_cli.py | 28 ++++++++++++++++++++++++++++ tests/test_cli_convert.py | 19 +++++++++++++++++++ 6 files changed, 121 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 085d9d5..556e497 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _unreleased: + +Unreleased +---------- + +- JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) + .. _v4_0rc3: 4.0rc3 (2026-07-05) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 1eeaabf..49eba52 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -129,6 +129,8 @@ See :ref:`cli_query`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX -r, --raw Raw output, first column of first row --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query @@ -198,6 +200,8 @@ See :ref:`cli_memory`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX -r, --raw Raw output, first column of first row --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query @@ -435,6 +439,7 @@ See :ref:`cli_search`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -701,6 +706,7 @@ See :ref:`cli_tables`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --columns Include list of columns for each table --schema Include schema for each table --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -743,6 +749,7 @@ See :ref:`cli_views`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --columns Include list of columns for each view --schema Include schema for each view --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -790,6 +797,8 @@ See :ref:`cli_rows`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -830,6 +839,7 @@ See :ref:`cli_triggers`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -870,6 +880,7 @@ See :ref:`cli_indexes`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. diff --git a/docs/cli.rst b/docs/cli.rst index 84a65d9..957ee64 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -111,6 +111,33 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +.. _cli_query_json_ascii: + +Unicode characters in JSON +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +JSON output includes unicode characters directly, without escaping them: + +.. code-block:: bash + + sqlite-utils dogs.db "select '日本語' as text" + +.. code-block:: output + + [{"text": "日本語"}] + +Use ``--ascii`` to escape non-ASCII characters as ``\uXXXX`` sequences instead: + +.. code-block:: bash + + sqlite-utils dogs.db "select '日本語' as text" --ascii + +.. code-block:: output + + [{"text": "\u65e5\u672c\u8a9e"}] + +The ``--ascii`` option can help on systems that cannot display or process UTF-8, such as Windows consoles using a legacy code page. On Windows, setting the ``PYTHONUTF8=1`` environment variable is an alternative fix for ``UnicodeEncodeError`` crashes when redirecting output to a file. + .. _cli_query_binary_json: Binary data in JSON diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8918439..8ee2091 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -131,6 +131,13 @@ def output_options(fn): is_flag=True, default=False, ), + click.option( + "--ascii", + "ascii_", + help="Escape non-ASCII characters in JSON output as \\uXXXX", + is_flag=True, + default=False, + ), ) ): fn = decorator(fn) @@ -199,6 +206,7 @@ def tables( table, fmt, json_cols, + ascii_, columns, schema, load_extension, @@ -258,7 +266,7 @@ def tables( for row in _iter(): writer.writerow(row) else: - for line in output_rows(_iter(), headers, nl, arrays, json_cols): + for line in output_rows(_iter(), headers, nl, arrays, json_cols, ascii_): click.echo(line) @@ -296,6 +304,7 @@ def views( table, fmt, json_cols, + ascii_, columns, schema, load_extension, @@ -321,6 +330,7 @@ def views( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, columns=columns, schema=schema, load_extension=load_extension, @@ -1857,6 +1867,7 @@ def query( table, fmt, json_cols, + ascii_, raw, raw_lines, param, @@ -1895,6 +1906,7 @@ def query( nl, arrays, json_cols, + ascii_, ) @@ -1969,6 +1981,7 @@ def memory( table, fmt, json_cols, + ascii_, raw, raw_lines, param, @@ -2108,6 +2121,7 @@ def memory( nl, arrays, json_cols, + ascii_, ) @@ -2125,6 +2139,7 @@ def _execute_query( nl, arrays, json_cols, + ascii_, ): with db.conn: try: @@ -2167,7 +2182,7 @@ def _execute_query( for row in cursor: writer.writerow(row) else: - for line in output_rows(cursor, headers, nl, arrays, json_cols): + for line in output_rows(cursor, headers, nl, arrays, json_cols, ascii_): click.echo(line) @@ -2211,6 +2226,7 @@ def search( table, fmt, json_cols, + ascii_, load_extension, ): """Execute a full-text search against this table @@ -2257,6 +2273,7 @@ def search( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, param=[("query", q)], load_extension=load_extension, ) @@ -2317,6 +2334,7 @@ def rows( table, fmt, json_cols, + ascii_, load_extension, ): """Output all rows in the specified table @@ -2351,6 +2369,7 @@ def rows( fmt=fmt, param=param, json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -2377,6 +2396,7 @@ def triggers( table, fmt, json_cols, + ascii_, load_extension, ): """Show triggers configured in this database @@ -2406,6 +2426,7 @@ def triggers( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -2434,6 +2455,7 @@ def indexes( table, fmt, json_cols, + ascii_, load_extension, ): """Show indexes for the whole database or specific tables @@ -2475,6 +2497,7 @@ def indexes( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -3112,7 +3135,7 @@ def convert( if multi: def preview(v): - return json.dumps(fn(v), default=repr) if v else v + return json.dumps(fn(v), default=repr, ensure_ascii=False) if v else v else: @@ -3458,7 +3481,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose): @cli.command(name="plugins") def plugins_list(): "List installed plugins" - click.echo(json.dumps(get_plugins(), indent=2)) + click.echo(json.dumps(get_plugins(), indent=2, ensure_ascii=False)) ensure_plugins_loaded() @@ -3505,7 +3528,7 @@ FILE_COLUMNS = { } -def output_rows(iterator, headers, nl, arrays, json_cols): +def output_rows(iterator, headers, nl, arrays, json_cols, ascii_=False): # Duplicate column names would collide as dictionary keys, so rename # later occurrences id, id -> id, id_2 - CSV and table output keep # the original duplicate headers since they never build dictionaries @@ -3526,7 +3549,7 @@ def output_rows(iterator, headers, nl, arrays, json_cols): data = dict(zip(headers, data)) line = "{firstchar}{serialized}{maybecomma}{lastchar}".format( firstchar=("[" if first else " ") if not nl else "", - serialized=json.dumps(data, default=json_binary), + serialized=json.dumps(data, default=json_binary, ensure_ascii=ascii_), maybecomma="," if (not nl and not is_last) else "", lastchar="]" if (is_last and not nl) else "", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index d11c42f..9e17969 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1047,6 +1047,34 @@ def test_query_json_with_json_cols(db_path): assert expected == result_rows.output.strip() +def test_query_json_unicode_not_escaped_by_default(db_path): + db = Database(db_path) + with db.conn: + db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text"]) + assert result.exit_code == 0 + assert result.output.strip() == '[{"id": 1, "text": "Japanese 日本語"}]' + # Same for --nl + result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text", "--nl"]) + assert result.exit_code == 0 + assert result.output.strip() == '{"id": 1, "text": "Japanese 日本語"}' + + +@pytest.mark.parametrize("command", ["query", "rows"]) +def test_query_json_ascii_option(db_path, command): + db = Database(db_path) + with db.conn: + db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + if command == "query": + args = [db_path, "select id, text from text", "--ascii"] + else: + args = ["rows", db_path, "text", "--ascii"] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 0 + expected = '[{"id": 1, "text": "Japanese ' + "\\u65e5\\u672c\\u8a9e" + '"}]' + assert result.output.strip() == expected + + @pytest.mark.parametrize( "content,is_binary", [(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)], diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 443e72c..6c3f5c5 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -215,6 +215,25 @@ def test_convert_multi_dryrun(test_db_and_path): ) +def test_convert_multi_dryrun_unicode_not_escaped(test_db_and_path): + db_path = test_db_and_path[1] + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "{'text': 'Japanese 日本語'}", + "--dry-run", + "--multi", + ], + ) + assert result.exit_code == 0 + # Preview should match what jsonify_if_needed() would actually store + assert '{"text": "Japanese 日本語"}' in result.output + + @pytest.mark.parametrize("drop", (True, False)) def test_convert_output_column(test_db_and_path, drop): db, db_path = test_db_and_path From 6225eba5c8daf97a5e459e9fcce2e46bcd022f50 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 20:13:52 -0700 Subject: [PATCH 032/110] Raise InvalidColumns on insert(pk="invalid") Closes #732 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 18 ++++++++++++++++++ tests/test_create.py | 16 ++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 556e497..a68a9b6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) +- ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) .. _v4_0rc3: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0807892..68aee3e 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -4229,6 +4229,24 @@ class Table(Queryable): if hash_id: pk = hash_id + if pk and not hash_id and self.exists(): + pk_cols = [pk] if isinstance(pk, str) else list(pk) + existing_columns = self.columns_dict + missing_pk_cols = [ + col + for col in pk_cols + if resolve_casing(col, existing_columns) not in existing_columns + ] + if missing_pk_cols: + raise InvalidColumns( + "Invalid primary key column{} {} for table {} with columns {}".format( + "s" if len(missing_pk_cols) > 1 else "", + missing_pk_cols, + self.name, + list(existing_columns), + ) + ) + if ignore and replace: raise ValueError("Use either ignore=True or replace=True, not both") all_columns = [] diff --git a/tests/test_create.py b/tests/test_create.py index decefcf..7fbd7d5 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -3,6 +3,7 @@ from sqlite_utils.db import ( Database, DescIndex, AlterError, + InvalidColumns, NoObviousTable, OperationalError, ForeignKey, @@ -925,6 +926,21 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db assert rows == [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] +@pytest.mark.parametrize("num_rows", (0, 1, 2, 3, 10)) +def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): + # https://github.com/simonw/sqlite-utils/issues/732 + fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") + rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + + with pytest.raises(InvalidColumns) as ex: + fresh_db["t"].insert_all(rows, pk="not_a_column", alter=True) + + assert ex.value.args == ( + "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", + ) + assert fresh_db["t"].count == 0 + + def test_insert_ignore(fresh_db): fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") # Should raise an error if we try this again From 221774f25a53c76874b28af1c2dd170c76bbd9f1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 20:22:08 -0700 Subject: [PATCH 033/110] Fix for table.insert(..., pk=..., ignore=True), closes #554 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 56 ++++++++++++++++++++++++++++++++++++-------- tests/test_create.py | 18 ++++++++++++++ 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a68a9b6..2b2e5cf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased - JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) - ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) +- Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) .. _v4_0rc3: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 68aee3e..916d2f0 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -4386,18 +4386,54 @@ class Table(Queryable): if num_records_processed == 1: # For an insert we need to use result.lastrowid if not upsert and result is not None: - self.last_rowid = result.lastrowid - if (hash_id or pk) and self.last_rowid: - # Set self.last_pk to the pk(s) for that rowid - row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] - if hash_id: - self.last_pk = row[hash_id] - elif isinstance(pk, str): - self.last_pk = row[resolve_casing(pk, row)] + ignored_insert = ignore and result.rowcount == 0 + if ignored_insert: + if list_mode: + first_record_list = cast(Sequence[Any], first_record) + if hash_id: + pass + elif isinstance(pk, str): + pk_index = column_names.index( + resolve_casing(pk, column_names) + ) + self.last_pk = first_record_list[pk_index] + elif pk: + self.last_pk = tuple( + first_record_list[ + column_names.index(resolve_casing(p, column_names)) + ] + for p in pk + ) else: - self.last_pk = tuple(row[resolve_casing(p, row)] for p in pk) + first_record_dict = cast(Dict[str, Any], first_record) + if hash_id: + self.last_pk = hash_record( + first_record_dict, hash_id_columns + ) + elif isinstance(pk, str): + self.last_pk = first_record_dict[ + resolve_casing(pk, first_record_dict) + ] + elif pk: + self.last_pk = tuple( + first_record_dict[resolve_casing(p, first_record_dict)] + for p in pk + ) else: - self.last_pk = self.last_rowid + self.last_rowid = result.lastrowid + if (hash_id or pk) and self.last_rowid: + # Set self.last_pk to the pk(s) for that rowid + row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] + if hash_id: + self.last_pk = row[hash_id] + elif isinstance(pk, str): + self.last_pk = row[resolve_casing(pk, row)] + else: + self.last_pk = tuple( + row[resolve_casing(p, row)] for p in pk + ) + else: + self.last_pk = self.last_rowid else: # For an upsert use first_record from earlier if list_mode: diff --git a/tests/test_create.py b/tests/test_create.py index 7fbd7d5..2788920 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -953,6 +953,24 @@ def test_insert_ignore(fresh_db): assert rows == [{"id": 1, "bar": 2}] +def test_insert_ignore_with_pk_after_other_table_insert(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/554 + user = {"id": "abc", "name": "david"} + + fresh_db["users"].insert(user, pk="id") + fresh_db["comments"].insert_all( + [ + {"id": "def", "text": "ok"}, + {"id": "ghi", "text": "great"}, + ], + ) + + table = fresh_db["users"].insert(user, pk="id", ignore=True) + + assert table.last_pk == "abc" + assert list(fresh_db["users"].rows) == [user] + + def test_insert_hash_id(fresh_db): dogs = fresh_db["dogs"] id = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk From b8aa1368571b09f765af37783579224f74812bf3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 20:37:02 -0700 Subject: [PATCH 034/110] Remove beanbag-docutils - we stopped needing that in 5f81752 --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bff9390..01f1735 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,6 @@ dev = [ "tabulate>=0.10.0", ] docs = [ - "beanbag-docutils>=2.0", "codespell", "furo", "pygments-csv-lexer", From 2616dec7957a366da3260cf87eb79c78bae39d14 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 20:39:29 -0700 Subject: [PATCH 035/110] .extract() and .lookup() no longer extract null values, closes #186 - table.extract() and the sqlite-utils extract command now skip rows where every extracted column is null: the new foreign key column is left null and no all-null record is added to the lookup table. Rows with at least one non-null extracted column are extracted as before. - The extracts= option to insert() and friends keeps None values as null instead of creating a lookup record for them - previously each insert batch added a duplicate null row to the lookup table. - table.lookup() compares values using IS rather than = so lookup values containing None match existing rows correctly, instead of inserting a duplicate row on every call. Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 3 ++ docs/cli.rst | 2 ++ docs/python-api.rst | 6 ++++ docs/upgrading.rst | 2 ++ sqlite_utils/db.py | 18 +++++++--- tests/test_extract.py | 80 ++++++++++++++++++++++++++++++++++++++++-- tests/test_extracts.py | 48 +++++++++++++++++++++++++ tests/test_lookup.py | 24 +++++++++++++ 8 files changed, 176 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2b2e5cf..e773b11 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,9 @@ Unreleased ---------- +- **Breaking change**: ``table.extract()`` - and the ``sqlite-utils extract`` command - no longer extract rows where every extracted column is ``null``. Those rows now keep a ``null`` value in the new foreign key column instead of pointing at an all-``null`` record in the lookup table. When extracting multiple columns, rows are still extracted if at least one of the columns has a value. (:issue:`186`) +- The ``extracts=`` option to ``table.insert()`` and friends no longer creates a lookup table record for ``None`` values - the column value stays ``null``. Previously every batch of inserted rows containing a ``None`` value would add a duplicate ``null`` record to the lookup table. +- Fixed a bug where ``table.lookup()`` inserted a duplicate row on every call if any of the lookup values were ``None``. Lookup values are now compared using ``IS`` so that ``None`` values match existing rows correctly. - JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) - ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) - Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) diff --git a/docs/cli.rst b/docs/cli.rst index 957ee64..1f95bbd 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2286,6 +2286,8 @@ The ``sqlite-utils extract`` command can be used to extract specified columns in Take a look at the Python API documentation for :ref:`python_api_extract` for a detailed description of how this works, including examples of table schemas before and after running an extraction operation. +Rows where every extracted column is ``null`` are not extracted - those rows get a ``null`` value in their new foreign key column and no record is created for them in the lookup table. + The command takes a database, table and one or more columns that should be extracted. To extract the ``species`` column from the ``trees`` table you would run: .. code-block:: bash diff --git a/docs/python-api.rst b/docs/python-api.rst index 7ac3951..0e61cd4 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1279,6 +1279,8 @@ To create a species record with a note on when it was first seen, you can use th The first time this is called the record will be created for ``name="Palm"``. Any subsequent calls with that name will ignore the second argument, even if it includes different values. +``None`` values are matched correctly: calling ``.lookup()`` a second time with the same values will return the primary key of the existing row even if some of those values are ``None``. + ``.lookup()`` also accepts keyword arguments, which are passed through to the :ref:`insert() method ` and can be used to influence the shape of the created table. Supported parameters are: - ``pk`` - which defaults to ``id`` @@ -1324,6 +1326,8 @@ To extract the ``species`` column out to a separate ``Species`` table, you can d "species": "Common Juniper" }, extracts={"species": "Species"}) +``None`` values are not extracted: no record is created for them in the lookup table and the column value stays ``null``. + .. _python_api_m2m: Working with many-to-many relationships @@ -2022,6 +2026,8 @@ This produces a lookup table like so: "latin" TEXT ) +Rows where every extracted column is ``null`` are not extracted: no record is created for them in the lookup table and their foreign key column is left as ``null``. When extracting multiple columns, rows where at least one of the extracted columns has a value will be extracted as usual. + .. _python_api_hash: Setting an ID based on the hash of the row contents diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 09a9e2f..88e987d 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -77,6 +77,8 @@ Python API changes **table.convert() no longer skips falsey values.** Matching the CLI change above, ``table.convert()`` now converts every value. The ``skip_false`` parameter has been removed - previously it defaulted to ``True``, skipping empty strings and other falsey values. +**Null values are no longer extracted into lookup tables.** ``table.extract()`` and the ``sqlite-utils extract`` command leave rows alone if every extracted column is ``null`` - the new foreign key column is left as ``null`` instead of pointing at an all-``null`` record in the lookup table. The ``extracts=`` insert option similarly keeps ``None`` values as ``null``. Relatedly, ``table.lookup()`` now compares values using ``IS`` so that looking up a value containing ``None`` returns the existing matching row - previously it inserted a duplicate row on every call. + **ensure_autocommit_off() is now ensure_autocommit_on().** The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``. The old name described the opposite of what the method did: it temporarily puts the connection into driver-level autocommit mode (by setting ``isolation_level = None``), so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. The behavior is unchanged - update any calls to use the new name. **View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 916d2f0..0f2924c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2841,14 +2841,20 @@ class Table(Queryable): ) lookup_columns = [(rename.get(col) or col) for col in columns] lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True) + # Rows where every extracted column is null are left alone - they + # get a null foreign key and no lookup table record, see #186 + all_columns_are_null = " AND ".join( + "{} IS NULL".format(quote_identifier(c)) for c in columns + ) self.db.execute( - "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {}".format( + "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {} WHERE NOT ({all_null})".format( quote_identifier(table), quote_identifier(self.name), lookup_columns=", ".join( quote_identifier(c) for c in lookup_columns ), table_cols=", ".join(quote_identifier(c) for c in columns), + all_null=all_columns_are_null, ) ) @@ -2857,7 +2863,7 @@ class Table(Queryable): # And populate it self.db.execute( - "UPDATE {} SET {} = (SELECT id FROM {} WHERE {where})".format( + "UPDATE {} SET {} = (SELECT id FROM {} WHERE {where}) WHERE NOT ({all_null})".format( quote_identifier(self.name), quote_identifier(magic_lookup_column), quote_identifier(table), @@ -2870,6 +2876,7 @@ class Table(Queryable): ) for column in columns ), + all_null=all_columns_are_null, ) ) # Figure out the right column order @@ -3858,7 +3865,7 @@ class Table(Queryable): # Only process extracts if there are any if has_extracts: for i, key in enumerate(all_columns): - if key in extracts: + if key in extracts and record_values[i] is not None: record_values[i] = self.db.table(extracts[key]).lookup( {"value": record_values[i]} ) @@ -3878,7 +3885,7 @@ class Table(Queryable): ), ) ) - if key in extracts: + if key in extracts and value is not None: extract_table = extracts[key] value = self.db.table(extract_table).lookup({"value": value}) record_values.append(value) @@ -4615,8 +4622,9 @@ class Table(Queryable): fold_identifier_case(c) for c in lookup_values } not in unique_column_sets: self.create_index(lookup_values.keys(), unique=True) + # IS rather than = so that null values are matched correctly wheres = [ - "{} = ?".format(quote_identifier(column)) for column in lookup_values + "{} IS ?".format(quote_identifier(column)) for column in lookup_values ] rows = list( self.rows_where( diff --git a/tests/test_extract.py b/tests/test_extract.py index d24c597..1c0fa01 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -189,9 +189,85 @@ def test_extract_works_with_null_values(fresh_db): ) assert list(fresh_db["listens"].rows) == [ {"id": 1, "track_title": "foo", "album_id": 1}, - {"id": 2, "track_title": "baz", "album_id": 2}, + {"id": 2, "track_title": "baz", "album_id": None}, ] assert list(fresh_db["albums"].rows) == [ {"id": 1, "album_title": "bar"}, - {"id": 2, "album_title": None}, + ] + + +def test_extract_null_values_single_column(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/186 + fresh_db["species"].insert({"id": 1, "species": "Wolf"}, pk="id") + fresh_db["individuals"].insert_all( + [ + {"id": 10, "name": "Terriana", "species": "Fox"}, + {"id": 11, "name": "Spenidorm", "species": None}, + {"id": 12, "name": "Grantheim", "species": "Wolf"}, + {"id": 13, "name": "Turnutopia", "species": None}, + {"id": 14, "name": "Wargal", "species": "Wolf"}, + ], + pk="id", + ) + fresh_db["individuals"].extract("species") + # No null row should have been added to species + assert list(fresh_db["species"].rows) == [ + {"id": 1, "species": "Wolf"}, + {"id": 2, "species": "Fox"}, + ] + assert list(fresh_db["individuals"].rows) == [ + {"id": 10, "name": "Terriana", "species_id": 2}, + {"id": 11, "name": "Spenidorm", "species_id": None}, + {"id": 12, "name": "Grantheim", "species_id": 1}, + {"id": 13, "name": "Turnutopia", "species_id": None}, + {"id": 14, "name": "Wargal", "species_id": 1}, + ] + + +def test_extract_null_values_multiple_columns(fresh_db): + # A row should be extracted if at least one column is not null - + # only rows where ALL extracted columns are null are left alone + fresh_db["circulation"].insert_all( + [ + {"id": 1, "title": "title one", "creator": "creator one", "year": 2018}, + {"id": 2, "title": "title two", "creator": None, "year": 2019}, + {"id": 3, "title": None, "creator": None, "year": 2020}, + {"id": 4, "title": None, "creator": None, "year": 2021}, + ], + pk="id", + ) + fresh_db["circulation"].extract( + ["title", "creator"], table="books", fk_column="book_id" + ) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "title one", "creator": "creator one"}, + {"id": 2, "title": "title two", "creator": None}, + ] + assert list(fresh_db["circulation"].rows) == [ + {"id": 1, "book_id": 1, "year": 2018}, + {"id": 2, "book_id": 2, "year": 2019}, + {"id": 3, "book_id": None, "year": 2020}, + {"id": 4, "book_id": None, "year": 2021}, + ] + + +def test_extract_null_values_existing_lookup_table_with_null_row(fresh_db): + # Even if the lookup table already contains an all-null row, rows where + # every extracted column is null should keep a null foreign key + fresh_db["species"].insert({"id": 1, "species": None}, pk="id") + fresh_db["individuals"].insert_all( + [ + {"id": 10, "name": "Terriana", "species": "Fox"}, + {"id": 11, "name": "Spenidorm", "species": None}, + ], + pk="id", + ) + fresh_db["individuals"].extract("species") + assert list(fresh_db["species"].rows) == [ + {"id": 1, "species": None}, + {"id": 2, "species": "Fox"}, + ] + assert list(fresh_db["individuals"].rows) == [ + {"id": 10, "name": "Terriana", "species_id": 2}, + {"id": 11, "name": "Spenidorm", "species_id": None}, ] diff --git a/tests/test_extracts.py b/tests/test_extracts.py index eb4f37e..7add79a 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -67,3 +67,51 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): {"id": 2, "species_id": 1}, {"id": 3, "species_id": 2}, ] == list(fresh_db["Trees"].rows) + + +def test_extracts_null_values(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/186 + # Null values should stay null, not be extracted into the lookup table + fresh_db["Trees"].insert_all( + [ + {"id": 1, "species_id": "Oak"}, + {"id": 2, "species_id": None}, + {"id": 3, "species_id": "Palm"}, + {"id": 4, "species_id": None}, + ], + extracts={"species_id": "Species"}, + ) + assert list(fresh_db["Species"].rows) == [ + {"id": 1, "value": "Oak"}, + {"id": 2, "value": "Palm"}, + ] + assert list(fresh_db["Trees"].rows) == [ + {"id": 1, "species_id": 1}, + {"id": 2, "species_id": None}, + {"id": 3, "species_id": 2}, + {"id": 4, "species_id": None}, + ] + + +def test_extracts_null_values_list_mode(fresh_db): + # Same as test_extracts_null_values but for list-based records + fresh_db["Trees"].insert_all( + [ + ["id", "species_id"], + [1, "Oak"], + [2, None], + [3, "Palm"], + [4, None], + ], + extracts={"species_id": "Species"}, + ) + assert list(fresh_db["Species"].rows) == [ + {"id": 1, "value": "Oak"}, + {"id": 2, "value": "Palm"}, + ] + assert list(fresh_db["Trees"].rows) == [ + {"id": 1, "species_id": 1}, + {"id": 2, "species_id": None}, + {"id": 3, "species_id": 2}, + {"id": 4, "species_id": None}, + ] diff --git a/tests/test_lookup.py b/tests/test_lookup.py index a36b464..da4f18b 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -157,3 +157,27 @@ def test_lookup_with_extra_insert_parameters(fresh_db): def test_lookup_new_table_strict(fresh_db, strict): fresh_db["species"].lookup({"name": "Palm"}, strict=strict) assert fresh_db["species"].strict == strict or not fresh_db.supports_strict + + +def test_lookup_null_value_idempotent(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/186 + # Repeated lookups of a null value should return the same row, + # not insert a duplicate row each time + species = fresh_db["species"] + first_id = species.lookup({"name": None}) + second_id = species.lookup({"name": None}) + assert first_id == second_id + assert list(species.rows) == [{"id": first_id, "name": None}] + + +def test_lookup_compound_key_with_null_idempotent(fresh_db): + species = fresh_db["species"] + palm_id = species.lookup({"name": "Palm", "type": None}) + oak_id = species.lookup({"name": "Oak", "type": "Tree"}) + assert palm_id == species.lookup({"name": "Palm", "type": None}) + assert oak_id == species.lookup({"name": "Oak", "type": "Tree"}) + assert palm_id != oak_id + assert list(species.rows) == [ + {"id": palm_id, "name": "Palm", "type": None}, + {"id": oak_id, "name": "Oak", "type": "Tree"}, + ] From f2fbcf60d8bbc95ca996fd078b42fc60831c4f6c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:03:15 -0700 Subject: [PATCH 036/110] --no-headers fix in changelog, refs #566 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e773b11..14e26a1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Unreleased - The ``extracts=`` option to ``table.insert()`` and friends no longer creates a lookup table record for ``None`` values - the column value stays ``null``. Previously every batch of inserted rows containing a ``None`` value would add a duplicate ``null`` record to the lookup table. - Fixed a bug where ``table.lookup()`` inserted a duplicate row on every call if any of the lookup values were ``None``. Lookup values are now compared using ``IS`` so that ``None`` values match existing rows correctly. - JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) +- ``--no-headers`` now omits the header row from ``--fmt`` and ``--table`` output, not just CSV and TSV output. (:issue:`566`) - ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) - Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) From 7d861181687e88194931c52ebfd57eb31f34c8d8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:19:20 -0700 Subject: [PATCH 037/110] Fix failed db.execute() write leaves a phantom transaction open Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 16 ++++++++++++---- tests/test_atomic.py | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 14e26a1..3bdab36 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,7 @@ Unreleased - ``--no-headers`` now omits the header row from ``--fmt`` and ``--table`` output, not just CSV and TSV output. (:issue:`566`) - ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) - Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) +- Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. .. _v4_0rc3: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0f2924c..534f145 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -870,10 +870,18 @@ class Database: if self._tracer: self._tracer(sql, parameters) was_in_transaction = self.conn.in_transaction - if parameters is not None: - cursor = self.conn.execute(sql, parameters) - else: - cursor = self.conn.execute(sql) + try: + if parameters is not None: + cursor = self.conn.execute(sql, parameters) + else: + cursor = self.conn.execute(sql) + except Exception: + if not was_in_transaction and self.conn.in_transaction: + # The failed statement opened an implicit transaction that + # nothing would ever commit - roll it back, otherwise it + # would capture every subsequent write + self.conn.execute("ROLLBACK") + raise if ( not was_in_transaction and self.conn.in_transaction diff --git a/tests/test_atomic.py b/tests/test_atomic.py index f75c4ee..1a4b4ae 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -258,6 +258,47 @@ def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): assert [r["id"] for r in fresh_db["t"].rows] == [1] +def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir): + # A failed write must not leave the driver's implicit transaction open - + # that would silently disable auto-commit for every subsequent write + path = str(tmpdir / "test.db") + db = Database(path) + db["t"].insert({"id": 1}, pk="id") + with pytest.raises(sqlite3.IntegrityError): + db.execute("insert into t (id) values (1)") + assert not db.conn.in_transaction + # Subsequent writes commit as normal and survive closing the connection + db["other"].insert({"id": 2}) + db.close() + db2 = Database(path) + assert db2["other"].exists() + db2.close() + + +def test_execute_failed_write_preserves_explicit_transaction(fresh_db): + # A failed write inside an explicit transaction must not roll back + # the caller's earlier work - only the caller decides that + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.begin() + fresh_db.execute("insert into t (id) values (2)") + with pytest.raises(sqlite3.IntegrityError): + fresh_db.execute("insert into t (id) values (1)") + assert fresh_db.conn.in_transaction + fresh_db.commit() + assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] + + +def test_execute_failed_write_inside_atomic_preserves_block(fresh_db): + # A caught failure inside an atomic() block must leave the block's + # transaction open so its other work still commits + fresh_db["t"].insert({"id": 1}, pk="id") + with fresh_db.atomic(): + fresh_db.execute("insert into t (id) values (2)") + with pytest.raises(sqlite3.IntegrityError): + fresh_db.execute("insert into t (id) values (1)") + assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] + + def test_query_returning_commits_after_iteration(tmpdir): if sqlite3.sqlite_version_info < (3, 35, 0): import pytest as _pytest From adc10df98102c76c86c77108615792ba238a0ae3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:22:29 -0700 Subject: [PATCH 038/110] Fix for db.query("; COMMIT") bypasses the first-token scanner Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 11 ++++++----- tests/test_atomic.py | 13 +++++++++++++ tests/test_query.py | 27 +++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3bdab36..30ad989 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,7 @@ Unreleased - ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) - Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. +- Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. .. _v4_0rc3: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 534f145..bd023c5 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -436,14 +436,15 @@ _QUERY_REJECTED_KEYWORDS = _TRANSACTION_CONTROL_KEYWORDS | { def _first_keyword(sql: str) -> str: """ - Return the first keyword of a SQL statement, uppercased, skipping any - leading whitespace and ``--`` or ``/* ... */`` comments - the only - things SQLite's tokenizer allows before the first token. Returns an - empty string if there is no leading keyword. + Return the first keyword of a SQL statement, uppercased, skipping + everything the sqlite3 driver tolerates before the first real token: + whitespace, ``--`` or ``/* ... */`` comments, empty statements + (bare ``;``) and a UTF-8 byte order mark. Returns an empty string if + there is no leading keyword. """ i, n = 0, len(sql) while i < n: - if sql[i].isspace(): + if sql[i].isspace() or sql[i] in (";", "\ufeff"): i += 1 elif sql.startswith("--", i): newline = sql.find("\n", i) diff --git a/tests/test_atomic.py b/tests/test_atomic.py index 1a4b4ae..0d25b84 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -258,6 +258,19 @@ def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): assert [r["id"] for r in fresh_db["t"].rows] == [1] +@pytest.mark.parametrize("begin_sql", ["; begin", "\ufeffbegin"]) +def test_execute_prefixed_begin_leaves_transaction_open(fresh_db, begin_sql): + # sqlite3 tolerates empty statements and a UTF-8 BOM before the first + # real token, so a BEGIN behind either must not be auto-committed + # out from under the caller + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.execute(begin_sql) + assert fresh_db.conn.in_transaction + fresh_db.execute("insert into t (id) values (2)") + fresh_db.rollback() + assert [r["id"] for r in fresh_db["t"].rows] == [1] + + def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir): # A failed write must not leave the driver's implicit transaction open - # that would silently disable auto-commit for every subsequent write diff --git a/tests/test_query.py b/tests/test_query.py index 0b9f2ae..c2f6731 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -59,6 +59,10 @@ def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db): "-- comment\nbegin", "/* multi\nline */ -- and another\n vacuum", "\t /* a */ /* b */ savepoint s1", + "; commit", + ";;\n ; rollback", + "; /* comment */ vacuum", + "\ufeffbegin", ], ) def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql): @@ -83,6 +87,23 @@ def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db): assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] +@pytest.mark.parametrize("sql", ["; COMMIT", "\ufeffCOMMIT"]) +def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql): + # sqlite3 tolerates empty statements and a UTF-8 BOM before the first + # real token, so the keyword scanner must skip them too - previously + # '; COMMIT' slipped past the check and committed the caller's open + # transaction before raising OperationalError + fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.begin() + fresh_db.execute("insert into dogs (name) values ('Pancakes')") + with pytest.raises(ValueError): + fresh_db.query(sql) + # The explicit transaction is still open and can still be rolled back + assert fresh_db.conn.in_transaction + fresh_db.rollback() + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + + def test_query_error_leaves_no_transaction_open(fresh_db): with pytest.raises(sqlite3.OperationalError): fresh_db.query("select * from missing_table") @@ -137,6 +158,12 @@ def test_query_comment_prefixed_pragma_inside_transaction(fresh_db): ("", ""), (" ", ""), ("123", ""), + ("; commit", "COMMIT"), + (";;\n ; rollback", "ROLLBACK"), + ("; -- comment\n begin", "BEGIN"), + ("\ufeffcommit", "COMMIT"), + ("\ufeff ; select 1", "SELECT"), + (";", ""), ], ) def test_first_keyword(sql, expected): From 66934918c689238b19a74b2f09794002cc985094 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:30:47 -0700 Subject: [PATCH 039/110] Document that rejected write PRAGMAs in db.query() still take effect db.query() promises that a statement rejected with ValueError is rolled back and has no effect. PRAGMA statements are the exception: some of them refuse to run inside a transaction, so they execute outside the savepoint guard - a row-less PRAGMA such as "PRAGMA user_version = 5" therefore takes effect despite the ValueError. Documenting this as a known limitation rather than fixing it, since a fix would need a hardcoded list of row-returning PRAGMAs. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/python-api.rst | 2 ++ sqlite_utils/db.py | 5 ++++- tests/test_query.py | 12 ++++++++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 30ad989..027765c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,7 @@ Unreleased - Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. +- Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. .. _v4_0rc3: diff --git a/docs/python-api.rst b/docs/python-api.rst index 0e61cd4..516e2aa 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -233,6 +233,8 @@ The SQL query is executed as soon as ``db.query()`` is called. The resulting row ``db.query()`` can only be used with SQL that returns rows. Passing a statement that returns no rows - an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause, for example - will raise a ``ValueError``. The rejected statement is rolled back, so it has no effect on the database. Use :ref:`db.execute() ` for those statements instead. +There is one exception to the rolled-back guarantee: a ``PRAGMA`` statement that returns no rows, such as ``PRAGMA user_version = 5``, still raises a ``ValueError`` but will already have taken effect. Some PRAGMA statements refuse to run inside a transaction, so PRAGMAs are executed outside the savepoint that is used to roll back other rejected statements. Use ``db.execute()`` for PRAGMA statements that do not return rows. + If a query returns more than one column with the same name - a join between two tables that share column names, for example - later occurrences are renamed with a numeric suffix, so every value is included in the dictionary: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index bd023c5..f6153cd 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -797,7 +797,10 @@ class Database: parameters, or a dictionary for ``where id = :id`` :raises ValueError: if the SQL statement does not return rows - use :meth:`execute` for those statements instead. The rejected statement - is rolled back, so it has no effect on the database + is rolled back, so it has no effect on the database. One exception: + a row-less ``PRAGMA`` statement takes effect despite the + ``ValueError``, because PRAGMAs run outside the savepoint guard - + some of them refuse to run inside a transaction """ message = ( "query() can only be used with SQL that returns rows - " diff --git a/tests/test_query.py b/tests/test_query.py index c2f6731..f4aa336 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -122,6 +122,18 @@ def test_query_pragma(tmpdir): db.close() +def test_query_rejected_pragma_still_takes_effect(fresh_db): + # Documented limitation: PRAGMAs run outside the savepoint guard, + # because some of them refuse to run inside a transaction - so a + # row-less PRAGMA takes effect even though it raises ValueError. + # If this test starts failing because the pragma was rolled back, + # the limitation has been fixed - update the docs in python-api.rst + # and the query() docstring to remove the carve-out + with pytest.raises(ValueError): + fresh_db.query("pragma user_version = 5") + assert fresh_db.execute("pragma user_version").fetchone()[0] == 5 + + def test_query_comment_prefixed_pragma(tmpdir): from sqlite_utils import Database From 8e015d024c34f2a5e5059e0f10fcdf41410613fe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:34:02 -0700 Subject: [PATCH 040/110] Compound primary keys now resolve in PRIMARY KEY declaration order PRAGMA table_info sets is_pk to the 1-based position of each column within the PRIMARY KEY, which can differ from table column order. table.pks previously returned table column order, so an implicit compound FOREIGN KEY ... REFERENCES other was introspected with its referenced columns inverted, and transform() baked that inverted order into the rewritten schema - failing with IntegrityError on valid data, or silently reversing the constraint with foreign keys off. table.pks, compound foreign key guessing (create, add_foreign_key) and transform() now all use declaration order, and transform() no longer reorders a compound PRIMARY KEY (b, a) into table column order. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 31 +++++++++++++++------- tests/test_foreign_keys.py | 54 ++++++++++++++++++++++++++++++++++++++ tests/test_introspect.py | 15 +++++++++++ 4 files changed, 91 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 027765c..e7c582e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed a bug where compound primary key columns were returned in table column order instead of ``PRIMARY KEY`` declaration order. For a table declared as ``CREATE TABLE other (b TEXT, a TEXT, PRIMARY KEY (a, b))`` an implicit ``FOREIGN KEY (x, y) REFERENCES other`` was introspected as referencing ``(b, a)`` when SQLite resolves it as ``(a, b)`` - running ``transform()`` on such a table then rewrote the schema with the inverted column order, silently reversing the meaning of the constraint and causing foreign key errors on valid data. ``table.pks``, compound foreign key guessing and ``transform()`` now all use the primary key declaration order, and ``transform()`` no longer reorders a compound ``PRIMARY KEY (b, a)`` into table column order. .. _v4_0rc3: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f6153cd..1e10b11 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2012,10 +2012,9 @@ class Queryable: :param offset: Integer for SQL offset """ column_names = [column.name for column in self.columns] - pks = [column.name for column in self.columns if column.is_pk] - if not pks: + pks = self.pks + if self.use_rowid: column_names.insert(0, "rowid") - pks = ["rowid"] select = ",".join(quote_identifier(column_name) for column_name in column_names) for row in self.rows_where( select=select, @@ -2148,8 +2147,18 @@ class Table(Queryable): @property def pks(self) -> List[str]: - "Primary key columns for this table." - names = [column.name for column in self.columns if column.is_pk] + """ + Primary key columns for this table, in PRIMARY KEY declaration order - + ``PRAGMA table_info`` sets ``is_pk`` to the 1-based position of each + column within the primary key, which can differ from the order of the + columns in the table. SQLite uses the declaration order to resolve + implicit foreign key references, so this order matters. + """ + pk_columns = sorted( + (column for column in self.columns if column.is_pk), + key=lambda column: column.is_pk, + ) + names = [column.name for column in pk_columns] if not names: names = ["rowid"] return names @@ -2673,7 +2682,8 @@ class Table(Queryable): if pk is DEFAULT: pks_renamed = tuple( - rename.get(p.name) or p.name for p in self.columns if p.is_pk + rename.get(pk_name) or pk_name + for pk_name in (self.pks if not self.use_rowid else []) ) if len(pks_renamed) == 1: pk = pks_renamed[0] @@ -3017,7 +3027,10 @@ class Table(Queryable): raise AlterError("table '{}' has no column {}".format(fk, fk_col)) else: # automatically set fk_col to first primary_key of fk table - pks = [c for c in self.db[fk].columns if c.is_pk] + pks = sorted( + (c for c in self.db[fk].columns if c.is_pk), + key=lambda c: c.is_pk, + ) if pks: fk_col = pks[0].name fk_col_type = pks[0].type @@ -3770,9 +3783,7 @@ class Table(Queryable): # First we execute the function pk_to_values = {} new_column_types: Dict[str, Set[type]] = {} - pks = [column.name for column in self.columns if column.is_pk] - if not pks: - pks = ["rowid"] + pks = self.pks with progressbar( length=self.count, silent=not show_progress, label="1: Evaluating" diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 8950189..1672069 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -524,3 +524,57 @@ def test_add_compound_foreign_key_on_delete(courses_db): assert fk.is_compound is True assert fk.on_delete == "SET NULL" assert "ON DELETE SET NULL" in courses_db["courses"].schema + + +def test_implicit_compound_foreign_key_resolves_pk_declaration_order(fresh_db): + # The other table's PRIMARY KEY declares its columns in a different + # order to the table's column order. SQLite resolves the implicit + # "REFERENCES other" using PRIMARY KEY declaration order, so the + # introspected other_columns must too + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db.execute( + "create table child (x text, y text, foreign key (x, y) references other)" + ) + fk = fresh_db["child"].foreign_keys[0] + assert fk.other_columns == ("a", "b") + + +def test_transform_implicit_compound_foreign_key_stays_valid(fresh_db): + # transform() rewrites the implicit FK with explicit columns - they + # must be in PRIMARY KEY declaration order or valid data fails the + # foreign key check with an IntegrityError + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db.execute( + "create table child (x text, y text, foreign key (x, y) references other)" + ) + fresh_db.execute("PRAGMA foreign_keys = ON") + fresh_db["other"].insert({"a": "A", "b": "B"}) + fresh_db["child"].insert({"x": "A", "y": "B"}) + fresh_db["child"].transform(types={"x": str}) + assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + # The constraint still points the right way around + fresh_db["child"].insert({"x": "A", "y": "B"}) + with pytest.raises(sqlite3.IntegrityError): + fresh_db["child"].insert({"x": "B", "y": "A"}) + + +def test_create_compound_foreign_key_guesses_pk_declaration_order(fresh_db): + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db["other"].insert({"a": "A", "b": "B"}) + fresh_db["child"].create( + {"id": int, "x": str, "y": str}, + pk="id", + foreign_keys=[(("x", "y"), "other")], + ) + assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + fresh_db.execute("PRAGMA foreign_keys = ON") + fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}) + with pytest.raises(sqlite3.IntegrityError): + fresh_db["child"].insert({"id": 2, "x": "B", "y": "A"}) + + +def test_add_compound_foreign_key_guesses_pk_declaration_order(fresh_db): + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}, pk="id") + fresh_db["child"].add_foreign_key(("x", "y"), "other") + assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 4ff3f77..8b6765d 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -321,3 +321,18 @@ def test_table_default_values(fresh_db, value): ) default_values = fresh_db["default_values"].default_values assert default_values == {"value": value} + + +def test_pks_use_primary_key_declaration_order(fresh_db): + # PRIMARY KEY (a, b) declared against columns stored in order (b, a) - + # pks must follow the declaration order, which is what SQLite uses to + # resolve implicit foreign key references and compound pk lookups + fresh_db.execute("create table t (b text, a text, primary key (a, b))") + assert fresh_db["t"].pks == ["a", "b"] + + +def test_transform_preserves_compound_pk_declaration_order(fresh_db): + fresh_db.execute("create table t (a text, b text, c text, primary key (b, a))") + fresh_db["t"].transform(drop={"c"}) + assert fresh_db["t"].pks == ["b", "a"] + assert 'PRIMARY KEY ("b", "a")' in fresh_db["t"].schema From 404e935b6348c57e507fba36eb06daf462257af8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:35:45 -0700 Subject: [PATCH 041/110] ForeignKey is now a frozen dataclass, restoring hashability The namedtuple-to-dataclass change made ForeignKey unhashable, breaking set(table.foreign_keys) and dict-key usage that worked in 3.x. frozen=True restores immutability and hashability. Equality and hashing cover all compared fields including on_delete/on_update - two foreign keys differing only in their actions are different constraints. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/upgrading.rst | 2 +- sqlite_utils/db.py | 24 +++++++++++++++++------- tests/test_foreign_keys.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e7c582e..b1c6108 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``ForeignKey`` objects are hashable again. The 4.0 change from ``namedtuple`` to dataclass accidentally made them unhashable, breaking patterns like ``set(table.foreign_keys)`` that worked in 3.x. ``ForeignKey`` is now a frozen dataclass - immutable and hashable, like the namedtuple was. - Fixed a bug where compound primary key columns were returned in table column order instead of ``PRIMARY KEY`` declaration order. For a table declared as ``CREATE TABLE other (b TEXT, a TEXT, PRIMARY KEY (a, b))`` an implicit ``FOREIGN KEY (x, y) REFERENCES other`` was introspected as referencing ``(b, a)`` when SQLite resolves it as ``(a, b)`` - running ``transform()`` on such a table then rewrote the schema with the inverted column order, silently reversing the meaning of the constraint and causing foreign key errors on valid data. ``table.pks``, compound foreign key guessing and ``transform()`` now all use the primary key declaration order, and ``transform()`` no longer reorders a compound ``PRIMARY KEY (b, a)`` into table column order. .. _v4_0rc3: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 88e987d..92b582a 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -95,7 +95,7 @@ Python API changes for fk in db["courses"].foreign_keys: fk.table, fk.column, fk.other_table, fk.other_column -Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. +Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. Like the old namedtuple, ``ForeignKey`` instances are immutable and hashable - they can be collected into sets and used as dictionary keys. Note that equality now includes the ``on_delete`` and ``on_update`` actions: a ``ForeignKey`` with ``ON DELETE CASCADE`` is not equal to one without. Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` tuples instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are one-item tuples. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1e10b11..e94434a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -193,7 +193,7 @@ Summary information about a column, see :ref:`python_api_analyze_column`. """ -@dataclass(order=True) +@dataclass(order=True, frozen=True) class ForeignKey: """ A foreign key defined on a table. @@ -208,6 +208,11 @@ class ForeignKey: ``on_delete`` and ``on_update`` hold the foreign key actions, e.g. ``"CASCADE"`` - ``"NO ACTION"`` if not set. + Instances are immutable and hashable, so they can be collected into + sets and used as dictionary keys. Equality covers every compared field, + including ``on_delete`` and ``on_update`` - two foreign keys differing + only in their actions are different constraints. + Prior to sqlite-utils 4.0 this was a ``namedtuple`` and could be unpacked or indexed as ``(table, column, other_table, other_column)``. It is now a dataclass - access its fields by name instead. @@ -227,16 +232,21 @@ class ForeignKey: def __post_init__(self): # Populate columns/other_columns for single-column foreign keys, - # normalizing any lists to tuples + # normalizing any lists to tuples. object.__setattr__ because the + # dataclass is frozen if self.columns: - self.columns = tuple(self.columns) + object.__setattr__(self, "columns", tuple(self.columns)) else: - self.columns = (self.column,) if self.column is not None else () + object.__setattr__( + self, "columns", (self.column,) if self.column is not None else () + ) if self.other_columns: - self.other_columns = tuple(self.other_columns) + object.__setattr__(self, "other_columns", tuple(self.other_columns)) else: - self.other_columns = ( - (self.other_column,) if self.other_column is not None else () + object.__setattr__( + self, + "other_columns", + (self.other_column,) if self.other_column is not None else (), ) diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 1672069..49d3352 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -578,3 +578,35 @@ def test_add_compound_foreign_key_guesses_pk_declaration_order(fresh_db): fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}, pk="id") fresh_db["child"].add_foreign_key(("x", "y"), "other") assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + + +def test_foreign_keys_are_hashable(fresh_db): + # set() over foreign_keys worked with the 3.x namedtuple and must + # keep working with the dataclass + fresh_db["p"].insert({"id": 1}, pk="id") + fresh_db["c"].insert( + {"id": 1, "pid": 1}, pk="id", foreign_keys=[("pid", "p", "id")] + ) + fks = set(fresh_db["c"].foreign_keys) + assert len(fks) == 1 + assert ForeignKey("c", "pid", "p", "id") in fks + # Usable as dict keys too + assert {fk: True for fk in fks} + + +def test_foreign_key_is_immutable(): + import dataclasses + + fk = ForeignKey("c", "pid", "p", "id") + with pytest.raises(dataclasses.FrozenInstanceError): + fk.table = "other" + + +def test_foreign_key_equality_and_hash_include_actions(): + # Two foreign keys differing only in ON DELETE behavior are different + # constraints - they compare unequal and hash separately + plain = ForeignKey("c", "pid", "p", "id") + cascade = ForeignKey("c", "pid", "p", "id", on_delete="CASCADE") + assert plain != cascade + assert len({plain, cascade}) == 2 + assert plain == ForeignKey("c", "pid", "p", "id") From 29ca9d27e24be818a26fff41450664ef771a03a0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:37:56 -0700 Subject: [PATCH 042/110] foreign_keys= accepts mixed ForeignKey objects, tuples and strings again resolve_foreign_keys() used all-or-nothing isinstance checks, so mixing ForeignKey objects with tuples raised "foreign_keys= should be a list of tuples" - a regression from 3.x, where ForeignKey was a namedtuple and passed the tuple check. Each item is now normalized individually, which also allows bare column-name strings in a mixed list. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 30 ++++++++++++++++-------------- tests/test_foreign_keys.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b1c6108..6e6fa30 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- The ``foreign_keys=`` argument to ``create()`` and ``insert()`` accepts a mixed list of ``ForeignKey`` objects, tuples and column name strings again. In 4.0 pre-releases mixing ``ForeignKey`` objects with tuples raised a ``ValueError`` - a regression from 3.x, where ``ForeignKey`` was a ``namedtuple`` and passed the tuple checks. - ``ForeignKey`` objects are hashable again. The 4.0 change from ``namedtuple`` to dataclass accidentally made them unhashable, breaking patterns like ``set(table.foreign_keys)`` that worked in 3.x. ``ForeignKey`` is now a frozen dataclass - immutable and hashable, like the namedtuple was. - Fixed a bug where compound primary key columns were returned in table column order instead of ``PRIMARY KEY`` declaration order. For a table declared as ``CREATE TABLE other (b TEXT, a TEXT, PRIMARY KEY (a, b))`` an implicit ``FOREIGN KEY (x, y) REFERENCES other`` was introspected as referencing ``(b, a)`` when SQLite resolves it as ``(a, b)`` - running ``transform()`` on such a table then rewrote the schema with the inverted column order, silently reversing the meaning of the constraint and causing foreign key errors on valid data. ``table.pks``, compound foreign key guessing and ``transform()`` now all use the primary key declaration order, and ``transform()`` no longer reorders a compound ``PRIMARY KEY (b, a)`` into table column order. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e94434a..6105a64 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1238,21 +1238,23 @@ class Database: (("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")) """ table = self.table(name) - if all(isinstance(fk, ForeignKey) for fk in foreign_keys): - return cast(List[ForeignKey], foreign_keys) - if all(isinstance(fk, str) for fk in foreign_keys): - # It's a list of columns - fks = [] - for column in foreign_keys: - column = cast(str, column) - other_table = table.guess_foreign_table(column) - other_column = table.guess_foreign_column(other_table) - fks.append(ForeignKey(name, column, other_table, other_column)) - return fks - if not all(isinstance(fk, (tuple, list)) for fk in foreign_keys): - raise ValueError("foreign_keys= should be a list of tuples") fks = [] - for tuple_or_list in cast(Iterable[Sequence[Any]], foreign_keys): + for fk in foreign_keys: + if isinstance(fk, ForeignKey): + fks.append(fk) + continue + if isinstance(fk, str): + # A bare column name - guess the other table and column + other_table = table.guess_foreign_table(fk) + other_column = table.guess_foreign_column(other_table) + fks.append(ForeignKey(name, fk, other_table, other_column)) + continue + if not isinstance(fk, (tuple, list)): + raise ValueError( + "foreign_keys= should be a list of tuples, " + "ForeignKey objects or column name strings" + ) + tuple_or_list = cast(Sequence[Any], fk) if len(tuple_or_list) == 4: if tuple_or_list[0] != name: raise ValueError( diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 49d3352..d7c20f6 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -610,3 +610,36 @@ def test_foreign_key_equality_and_hash_include_actions(): assert plain != cascade assert len({plain, cascade}) == 2 assert plain == ForeignKey("c", "pid", "p", "id") + + +def test_create_table_mixed_foreign_keys_list(fresh_db): + # 3.x accepted a mix of ForeignKey objects, tuples and bare column + # strings in foreign_keys= (ForeignKey was a namedtuple, so it passed + # the tuple check) - keep accepting the mix + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["publishers"].insert({"id": 1}, pk="id") + fresh_db["books"].create( + {"id": int, "author_id": int, "publisher_id": int}, + pk="id", + foreign_keys=[ + ForeignKey("books", "author_id", "authors", "id"), + ("publisher_id", "publishers", "id"), + ], + ) + fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} + assert fks == {"author_id": "authors", "publisher_id": "publishers"} + + +def test_create_table_mixed_foreign_keys_with_string(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["publishers"].insert({"id": 1}, pk="id") + fresh_db["books"].create( + {"id": int, "author_id": int, "publisher_id": int}, + pk="id", + foreign_keys=[ + "author_id", # bare column, table and column guessed + ("publisher_id", "publishers", "id"), + ], + ) + fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} + assert fks == {"author_id": "authors", "publisher_id": "publishers"} From 1ed95e4ad2676b7f2f0725919bba9c4b051456c4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:42:02 -0700 Subject: [PATCH 043/110] Fix pks_and_rows_where() on views The previous compound-pk-ordering commit switched pks_and_rows_where() to Table-only properties, but the method is defined on Queryable and views exposed it too - calling it on a View raised AttributeError. Restored Queryable-safe logic, and stopped double-quoting the synthesized rowid column: SQLite turns a double-quoted identifier that does not resolve into a string literal, so on a view the generated select "rowid" silently produced the string 'rowid' and a confusing KeyError, where 3.x's [rowid] quoting raised OperationalError cleanly. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 21 ++++++++++++++++----- tests/test_rows.py | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6e6fa30..5d6c901 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed ``pks_and_rows_where()`` raising ``AttributeError`` when called on a view, and no longer double-quotes the synthesized ``rowid`` column in its generated SQL - SQLite turns a double-quoted identifier that does not resolve into a string literal, which on a view produced a confusing ``KeyError`` instead of the ``OperationalError`` raised in 3.x. Compound primary keys returned by this method now follow ``PRIMARY KEY`` declaration order. - The ``foreign_keys=`` argument to ``create()`` and ``insert()`` accepts a mixed list of ``ForeignKey`` objects, tuples and column name strings again. In 4.0 pre-releases mixing ``ForeignKey`` objects with tuples raised a ``ValueError`` - a regression from 3.x, where ``ForeignKey`` was a ``namedtuple`` and passed the tuple checks. - ``ForeignKey`` objects are hashable again. The 4.0 change from ``namedtuple`` to dataclass accidentally made them unhashable, breaking patterns like ``set(table.foreign_keys)`` that worked in 3.x. ``ForeignKey`` is now a frozen dataclass - immutable and hashable, like the namedtuple was. - Fixed a bug where compound primary key columns were returned in table column order instead of ``PRIMARY KEY`` declaration order. For a table declared as ``CREATE TABLE other (b TEXT, a TEXT, PRIMARY KEY (a, b))`` an implicit ``FOREIGN KEY (x, y) REFERENCES other`` was introspected as referencing ``(b, a)`` when SQLite resolves it as ``(a, b)`` - running ``transform()`` on such a table then rewrote the schema with the inverted column order, silently reversing the meaning of the constraint and causing foreign key errors on valid data. ``table.pks``, compound foreign key guessing and ``transform()`` now all use the primary key declaration order, and ``transform()`` no longer reorders a compound ``PRIMARY KEY (b, a)`` into table column order. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6105a64..4c7ef52 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2023,11 +2023,22 @@ class Queryable: :param limit: Integer number of rows to limit to :param offset: Integer for SQL offset """ - column_names = [column.name for column in self.columns] - pks = self.pks - if self.use_rowid: - column_names.insert(0, "rowid") - select = ",".join(quote_identifier(column_name) for column_name in column_names) + # This method is defined on Queryable so it serves views too, which + # have no pks property - sort pk columns into declaration order here + pk_columns = sorted( + (column for column in self.columns if column.is_pk), + key=lambda column: column.is_pk, + ) + pks = [column.name for column in pk_columns] + select_parts = [quote_identifier(column.name) for column in self.columns] + if not pks: + # rowid is left unquoted: it is not a real column, and SQLite + # turns a double-quoted identifier that does not resolve into a + # string literal - on a view that would silently select the + # string 'rowid' instead of raising an error + select_parts.insert(0, "rowid") + pks = ["rowid"] + select = ",".join(select_parts) for row in self.rows_where( select=select, where=where, diff --git a/tests/test_rows.py b/tests/test_rows.py index f050d5a..46417ef 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -111,3 +111,24 @@ def test_rows_where_duplicate_select_columns_are_deduped(fresh_db): fresh_db["t"].insert({"id": 1, "name": "Cleo"}) rows = list(fresh_db["t"].rows_where(select="id, id, name")) assert rows == [{"id": 1, "id_2": 1, "name": "Cleo"}] + + +def test_pks_and_rows_where_view(fresh_db): + # pks_and_rows_where() lives on Queryable so views expose it, but + # SQLite views have no rowid - it has always failed with an + # OperationalError from the generated SQL. Guard against it failing + # earlier with an AttributeError from View lacking Table properties + from sqlite_utils.utils import sqlite3 + + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.create_view("dog_names", "select name from dogs") + with pytest.raises(sqlite3.OperationalError): + list(fresh_db["dog_names"].pks_and_rows_where()) + + +def test_pks_and_rows_where_compound_pk_declaration_order(fresh_db): + # Compound pks are returned in PRIMARY KEY declaration order + fresh_db.execute("create table t (b text, a text, primary key (a, b))") + fresh_db["t"].insert({"a": "A", "b": "B"}) + pks_and_rows = list(fresh_db["t"].pks_and_rows_where()) + assert pks_and_rows == [(("A", "B"), {"b": "B", "a": "A"})] From 884574685fde886863dcadb0d297960c88ba5446 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:44:56 -0700 Subject: [PATCH 044/110] insert/upsert --csv no longer rewrites column types of existing tables Type detection is the 4.0 default for CSV/TSV data, and the detected-type transform ran even when the target table already existed - inserting a CSV into a table with a TEXT zip column converted the column to INTEGER, corrupting values with leading zeros ('01234' became 1234) with no warning. Detected types now only apply to tables the command created. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/cli.rst | 2 ++ sqlite_utils/cli.py | 12 +++++++++- tests/test_cli_insert.py | 48 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d6c901..80bcfb7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed a bug where inserting CSV or TSV data into an existing table rewrote that table's column types to match the incoming file. Type detection is the default in 4.0, so ``sqlite-utils insert data.db places places.csv --csv`` against a table with a ``TEXT`` zip code column would convert the column to ``INTEGER`` and corrupt values with leading zeros - ``"01234"`` became ``1234``. Detected types are now only applied when the ``insert`` or ``upsert`` command creates the table. - Fixed ``pks_and_rows_where()`` raising ``AttributeError`` when called on a view, and no longer double-quotes the synthesized ``rowid`` column in its generated SQL - SQLite turns a double-quoted identifier that does not resolve into a string literal, which on a view produced a confusing ``KeyError`` instead of the ``OperationalError`` raised in 3.x. Compound primary keys returned by this method now follow ``PRIMARY KEY`` declaration order. - The ``foreign_keys=`` argument to ``create()`` and ``insert()`` accepts a mixed list of ``ForeignKey`` objects, tuples and column name strings again. In 4.0 pre-releases mixing ``ForeignKey`` objects with tuples raised a ``ValueError`` - a regression from 3.x, where ``ForeignKey`` was a ``namedtuple`` and passed the tuple checks. - ``ForeignKey`` objects are hashable again. The 4.0 change from ``namedtuple`` to dataclass accidentally made them unhashable, breaking patterns like ``set(table.foreign_keys)`` that worked in 3.x. ``ForeignKey`` is now a frozen dataclass - immutable and hashable, like the namedtuple was. diff --git a/docs/cli.rst b/docs/cli.rst index 1f95bbd..063e80f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1326,6 +1326,8 @@ A progress bar is displayed when inserting data from a file. You can hide the pr By default, column types are automatically detected for CSV or TSV files - resulting in a mix of ``TEXT``, ``INTEGER`` and ``REAL`` columns. To disable type detection and treat all columns as ``TEXT``, use the ``--no-detect-types`` option. +Detected types are only applied when the table is created by the command. Inserting CSV or TSV data into a table that already exists leaves the existing column types unchanged - values are inserted using the table's existing schema. + For example, given a ``creatures.csv`` file containing this: .. code-block:: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8ee2091..0ef7cc4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1165,6 +1165,9 @@ def insert_upsert_implementation( db.conn.cursor().executemany(bulk_sql, doc_chunk) return + # table_names() rather than db.table(), which raises NoTable for + # views before the error handling below can deal with them + table_existed_before_insert = table in db.table_names() try: db.table(table).insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs @@ -1194,7 +1197,14 @@ def insert_upsert_implementation( ) else: raise - if tracker is not None and db.table(table).exists(): + # Apply detected types only to a table this command created - + # transforming a pre-existing table would rewrite its column types + # and corrupt values such as TEXT zip codes with leading zeros + if ( + tracker is not None + and not table_existed_before_insert + and db.table(table).exists() + ): db.table(table).transform(types=tracker.types) # Clean up open file-like objects diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 2df1e0c..196590e 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -628,3 +628,51 @@ def test_insert_into_view_errors(tmpdir): ) assert result.exit_code == 1 assert result.output.strip() == "Error: Table v is actually a view" + + +def test_insert_csv_detect_types_leaves_existing_table_alone(db_path): + # Type detection is the default for CSV/TSV inserts, but it must only + # apply to tables created by this command - transforming a pre-existing + # table would rewrite its column types and corrupt data such as + # TEXT zip codes with leading zeros + db = Database(db_path) + db["places"].insert({"name": "Boston", "zip": "01234"}) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "places", "-", "--csv"], + catch_exceptions=False, + input="name,zip\nSF,94107", + ) + assert result.exit_code == 0, result.output + assert db["places"].columns_dict["zip"] is str + assert list(db["places"].rows) == [ + {"name": "Boston", "zip": "01234"}, + {"name": "SF", "zip": "94107"}, + ] + + +def test_insert_csv_detect_types_new_table(db_path): + # A table created by the insert still gets detected types + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "data", "-", "--csv"], + catch_exceptions=False, + input="name,age,weight\nCleo,5,12.5", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert db["data"].columns_dict == {"name": str, "age": int, "weight": float} + + +def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): + db = Database(db_path) + db["places"].insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id") + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "places", "-", "--csv", "--pk", "id"], + catch_exceptions=False, + input="id,name,zip\n2,SF,94107", + ) + assert result.exit_code == 0, result.output + assert db["places"].columns_dict["zip"] is str + assert db["places"].get(1)["zip"] == "01234" From 3de8507c6b7d28288d220d5627d4efe4f21b7abc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:47:59 -0700 Subject: [PATCH 045/110] insert(pk=, alter=True) can add the pk column from records again The InvalidColumns check added for #732 fired before alter=True had a chance to add the missing pk column - a regression from 3.x, where insert({"id": 5, "a": 2}, pk="id", alter=True) against a table without an id column worked. With alter=True the check is now deferred until the first batch of record keys is known: a pk column supplied by the records passes, one found in neither the table nor the records still raises InvalidColumns before anything is inserted. An empty record iterator with alter=True returns without error, matching the 3.x no-op. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 19 ++++++++++++++++++- tests/test_create.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 80bcfb7..df41884 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed a regression where ``table.insert(..., pk=..., alter=True)`` raised ``InvalidColumns`` if the primary key column did not exist in the table yet. With ``alter=True`` the check now waits until the record keys are known, so a pk column supplied by the records is added by the alter as it was in 3.x. A pk column found in neither the table nor the records still raises ``InvalidColumns``. - Fixed a bug where inserting CSV or TSV data into an existing table rewrote that table's column types to match the incoming file. Type detection is the default in 4.0, so ``sqlite-utils insert data.db places places.csv --csv`` against a table with a ``TEXT`` zip code column would convert the column to ``INTEGER`` and corrupt values with leading zeros - ``"01234"`` became ``1234``. Detected types are now only applied when the ``insert`` or ``upsert`` command creates the table. - Fixed ``pks_and_rows_where()`` raising ``AttributeError`` when called on a view, and no longer double-quotes the synthesized ``rowid`` column in its generated SQL - SQLite turns a double-quoted identifier that does not resolve into a string literal, which on a view produced a confusing ``KeyError`` instead of the ``OperationalError`` raised in 3.x. Compound primary keys returned by this method now follow ``PRIMARY KEY`` declaration order. - The ``foreign_keys=`` argument to ``create()`` and ``insert()`` accepts a mixed list of ``ForeignKey`` objects, tuples and column name strings again. In 4.0 pre-releases mixing ``ForeignKey`` objects with tuples raised a ``ValueError`` - a regression from 3.x, where ``ForeignKey`` was a ``namedtuple`` and passed the tuple checks. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 4c7ef52..faf588f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -4282,6 +4282,10 @@ class Table(Queryable): if hash_id: pk = hash_id + # pk columns missing from an existing table are an error - unless + # alter=True, where a pk column supplied by the records will be + # added, so validation waits until the record keys are known + deferred_invalid_pk_check = None if pk and not hash_id and self.exists(): pk_cols = [pk] if isinstance(pk, str) else list(pk) existing_columns = self.columns_dict @@ -4291,7 +4295,7 @@ class Table(Queryable): if resolve_casing(col, existing_columns) not in existing_columns ] if missing_pk_cols: - raise InvalidColumns( + invalid_pk_error = InvalidColumns( "Invalid primary key column{} {} for table {} with columns {}".format( "s" if len(missing_pk_cols) > 1 else "", missing_pk_cols, @@ -4299,6 +4303,9 @@ class Table(Queryable): list(existing_columns), ) ) + if not alter: + raise invalid_pk_error + deferred_invalid_pk_check = (missing_pk_cols, invalid_pk_error) if ignore and replace: raise ValueError("Use either ignore=True or replace=True, not both") @@ -4409,6 +4416,16 @@ class Table(Queryable): all_columns = list(sorted(all_columns_set)) if hash_id: all_columns.insert(0, hash_id) + if deferred_invalid_pk_check is not None: + # alter=True - pk columns the table lacks are valid if + # the records supply them, otherwise raise the error + missing_pk_cols, invalid_pk_error = deferred_invalid_pk_check + record_columns = {column: True for column in all_columns} + if any( + resolve_casing(col, record_columns) not in record_columns + for col in missing_pk_cols + ): + raise invalid_pk_error else: if not list_mode: for record in cast(List[Dict[str, Any]], chunk): diff --git a/tests/test_create.py b/tests/test_create.py index 2788920..42fa1c4 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -932,6 +932,23 @@ def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + with pytest.raises(InvalidColumns) as ex: + fresh_db["t"].insert_all(rows, pk="not_a_column") + + assert ex.value.args == ( + "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", + ) + assert fresh_db["t"].count == 0 + + +@pytest.mark.parametrize("num_rows", (1, 2, 3, 10)) +def test_insert_all_pk_not_in_records_alter_raises(fresh_db, num_rows): + # With alter=True the check is deferred until the record keys are + # known - a pk column that is in neither the table nor the records + # still raises + fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") + rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + with pytest.raises(InvalidColumns) as ex: fresh_db["t"].insert_all(rows, pk="not_a_column", alter=True) @@ -941,6 +958,26 @@ def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): assert fresh_db["t"].count == 0 +def test_insert_pk_in_records_with_alter_adds_column(fresh_db): + # 3.x allowed insert(pk=..., alter=True) to add the pk column from the + # records - the InvalidColumns check must not fire in that case + fresh_db["t"].insert({"a": 1}) + fresh_db["t"].insert({"id": 5, "a": 2}, pk="id", alter=True) + assert fresh_db["t"].columns_dict.keys() == {"a", "id"} + assert list(fresh_db.query("select * from t order by a")) == [ + {"a": 1, "id": None}, + {"a": 2, "id": 5}, + ] + + +def test_insert_all_invalid_pk_alter_empty_records_is_noop(fresh_db): + # With alter=True the pk check needs record keys, so an empty iterator + # returns without error - matching the 3.x no-op for empty inserts + fresh_db.conn.execute("CREATE TABLE t (a TEXT)") + fresh_db["t"].insert_all([], pk="not_a_column", alter=True) + assert fresh_db["t"].count == 0 + + def test_insert_ignore(fresh_db): fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") # Should raise an error if we try this again From b3aa3f47b717bba2c166f11e59a35bcd2f8e6b09 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:50:05 -0700 Subject: [PATCH 046/110] migrate --stop-before an already-applied migration is now an error The CLI validated --stop-before names against both pending and applied migrations, but Migrations.apply() only looked for the stop name among pending ones - naming an applied migration passed validation and then silently applied every migration after it, the exact outcome the option exists to prevent. apply() now raises ValueError before applying anything if a stop_before name matches an applied migration in its set; names not in the set are still ignored, since unqualified CLI values are offered to every set. The migrate command reports it as a clean error. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/migrations.rst | 2 +- sqlite_utils/cli.py | 5 ++++- sqlite_utils/migrations.py | 22 +++++++++++++++++++++- tests/test_cli_migrate.py | 22 ++++++++++++++++++++++ tests/test_migrations.py | 30 ++++++++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index df41884..4339e81 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``sqlite-utils migrate --stop-before`` now exits with an error if the named migration has already been applied. Previously the name passed validation but was only checked against pending migrations, so every migration after it was silently applied - the exact outcome ``--stop-before`` exists to prevent. ``Migrations.apply(db, stop_before=...)`` raises ``ValueError`` in the same situation, before applying anything. - Fixed a regression where ``table.insert(..., pk=..., alter=True)`` raised ``InvalidColumns`` if the primary key column did not exist in the table yet. With ``alter=True`` the check now waits until the record keys are known, so a pk column supplied by the records is added by the alter as it was in 3.x. A pk column found in neither the table nor the records still raises ``InvalidColumns``. - Fixed a bug where inserting CSV or TSV data into an existing table rewrote that table's column types to match the incoming file. Type detection is the default in 4.0, so ``sqlite-utils insert data.db places places.csv --csv`` against a table with a ``TEXT`` zip code column would convert the column to ``INTEGER`` and corrupt values with leading zeros - ``"01234"`` became ``1234``. Detected types are now only applied when the ``insert`` or ``upsert`` command creates the table. - Fixed ``pks_and_rows_where()`` raising ``AttributeError`` when called on a view, and no longer double-quotes the synthesized ``rowid`` column in its generated SQL - SQLite turns a double-quoted identifier that does not resolve into a string literal, which on a view produced a confusing ``KeyError`` instead of the ``OperationalError`` raised in 3.x. Compound primary keys returned by this method now follow ``PRIMARY KEY`` declaration order. diff --git a/docs/migrations.rst b/docs/migrations.rst index e685936..cfdbf13 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -157,7 +157,7 @@ You can also target a specific migration set using ``migration_set:migration_nam The ``--stop-before`` option can be passed more than once. -If a ``--stop-before`` value does not match any known migration the command exits with an error, rather than silently applying everything. +If a ``--stop-before`` value does not match any known migration the command exits with an error, rather than silently applying everything. Naming a migration that has already been applied is also an error - stopping before it is impossible to honor - and no pending migrations are applied. Verbose output ============== diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0ef7cc4..188eff6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3462,7 +3462,10 @@ def migrate(db_path, migrations, stop_before, list_, verbose): for migration_set in migration_sets: matches = _stop_before_for_migration_set(stop_before, migration_set.name) if isinstance(migration_set, sqlite_utils.Migrations): - migration_set.apply(db, stop_before=matches) + try: + migration_set.apply(db, stop_before=matches) + except ValueError as e: + raise click.ClickException(str(e)) else: # Legacy sqlite-migrate Migrations objects take a single string # for stop_before, not a list diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py index 36e053d..00d0fa5 100644 --- a/sqlite_utils/migrations.py +++ b/sqlite_utils/migrations.py @@ -96,14 +96,34 @@ class Migrations: changes are rolled back, no record is written and the migration stays pending. Migrations registered with ``transactional=False`` run outside of a transaction. + + :raises ValueError: if a ``stop_before`` name matches a migration in + this set that has already been applied - stopping before it is + impossible to honor, and no pending migrations are applied """ - self.ensure_migrations_table(db) if stop_before is None: stop_before_names = set() elif isinstance(stop_before, str): stop_before_names = {stop_before} else: stop_before_names = set(stop_before) + # A stop_before naming an already-applied migration cannot be + # honored - error rather than applying everything after it. Names + # not in this set at all are ignored, because unqualified CLI + # values are offered to every migration set + already_applied = stop_before_names.intersection( + migration.name for migration in self.applied(db) + ) + if already_applied: + raise ValueError( + "Cannot stop before migration{} {} in set '{}' - already " + "been applied".format( + "s" if len(already_applied) > 1 else "", + ", ".join(sorted(already_applied)), + self.name, + ) + ) + self.ensure_migrations_table(db) for migration in self.pending(db): name = migration.name if name in stop_before_names: diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index 1cdf8e7..0f1c7ea 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -463,3 +463,25 @@ def test_list_does_not_upgrade_legacy_migrations_table(two_migrations): db2 = sqlite_utils.Database(db_path) assert db2["_sqlite_migrations"].pks == ["migration_set", "name"] db2.close() + + +def test_stop_before_applied_migration_errors(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + migrations_path = str(path / "foo" / "migrations.py") + # Apply everything first + first = CliRunner().invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, migrations_path, "--stop-before", "bar"], + ) + assert first.exit_code == 0 + # foo is now applied - stopping before it is an error, and bar + # must not be applied as a side effect + result = CliRunner().invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, migrations_path, "--stop-before", "foo"], + ) + assert result.exit_code != 0 + assert "already been applied" in result.output + db = sqlite_utils.Database(db_path) + assert not db["bar"].exists() diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 5634d79..04185fc 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -214,3 +214,33 @@ def test_duplicate_migration_name_errors(): pass assert "m001" in str(ex.value) + + +def test_stop_before_applied_migration_errors(migrations): + # Stopping before a migration that has already been applied is + # impossible to honor - previously the stop name was only checked + # against pending migrations, so everything after it was applied + db = sqlite_utils.Database(memory=True) + migrations.apply(db, stop_before="m002") # applies m001 only + with pytest.raises(ValueError) as ex: + migrations.apply(db, stop_before="m001") + assert "m001" in str(ex.value) + assert "already been applied" in str(ex.value) + # Nothing else was applied + assert not db["cats"].exists() + + +def test_stop_before_applied_migration_errors_before_any_apply(migrations): + # The error fires before any pending migration runs, even those that + # come before the already-applied stop target in registration order + db = sqlite_utils.Database(memory=True) + only_second = Migrations("test") + + @only_second() + def m002(db): + db["cats"].create({"name": str}) + + only_second.apply(db) # m002 applied, m001 still pending + with pytest.raises(ValueError): + migrations.apply(db, stop_before="m002") + assert not db["dogs"].exists() From a0387791e511a17eb96e0ed667da222550a00412 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:51:31 -0700 Subject: [PATCH 047/110] ensure_autocommit_on() raises TransactionError inside a transaction Assigning conn.isolation_level commits any pending transaction as a side effect, so entering the context manager with a transaction open silently committed the caller's work and made a later rollback() a no-op. All internal callers already ensure no transaction is open; the public API now enforces it, matching enable_wal() and disable_wal(). Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 9 +++++++++ tests/test_wal.py | 17 +++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4339e81..87a1ab7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``db.ensure_autocommit_on()`` now raises ``TransactionError`` if called while a transaction is open. Assigning ``isolation_level`` commits any pending transaction as a side effect, so entering the block silently committed the caller's open transaction and made a later ``rollback()`` a no-op. - ``sqlite-utils migrate --stop-before`` now exits with an error if the named migration has already been applied. Previously the name passed validation but was only checked against pending migrations, so every migration after it was silently applied - the exact outcome ``--stop-before`` exists to prevent. ``Migrations.apply(db, stop_before=...)`` raises ``ValueError`` in the same situation, before applying anything. - Fixed a regression where ``table.insert(..., pk=..., alter=True)`` raised ``InvalidColumns`` if the primary key column did not exist in the table yet. With ``alter=True`` the check now waits until the record keys are known, so a pk column supplied by the records is added by the alter as it was in 3.x. A pk column found in neither the table nor the records still raises ``InvalidColumns``. - Fixed a bug where inserting CSV or TSV data into an existing table rewrote that table's column types to match the incoming file. Type detection is the default in 4.0, so ``sqlite-utils insert data.db places places.csv --csv`` against a table with a ``TEXT`` zip code column would convert the column to ``INTEGER`` and corrupt values with leading zeros - ``"01234"`` became ``1234``. Detected types are now only applied when the ``insert`` or ``upsert`` command creates the table. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index faf588f..90e7a83 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -665,7 +665,16 @@ class Database: # do stuff here The previous ``isolation_level`` is restored at the end of the block. + + :raises TransactionError: if a transaction is open - assigning + ``isolation_level`` would commit it as a side effect, silently + breaking the caller's ability to roll back """ + if self.conn.in_transaction: + raise TransactionError( + "ensure_autocommit_on() cannot be used inside a transaction - " + "changing isolation_level would commit the open transaction" + ) old_isolation_level = self.conn.isolation_level try: self.conn.isolation_level = None diff --git a/tests/test_wal.py b/tests/test_wal.py index c5a9c60..2ddcf54 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -69,3 +69,20 @@ def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): db["test"].insert({"id": 1}, pk="id") db.enable_wal() assert [r["id"] for r in db["test"].rows] == [1] + + +def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): + # Setting isolation_level commits any pending transaction as a side + # effect, silently breaking the caller's rollback guarantee - so + # entering autocommit mode with a transaction open is an error + db, path, tmpdir = db_path_tmpdir + db["test"].insert({"id": 1}, pk="id") + db.begin() + db.execute("insert into test (id) values (2)") + with pytest.raises(TransactionError): + with db.ensure_autocommit_on(): + pass + # The transaction is still open and can still be rolled back + assert db.conn.in_transaction + db.rollback() + assert [r["id"] for r in db["test"].rows] == [1] From 16bbfb582d6acf33a8c05317b64d93c85e184a5c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:53:48 -0700 Subject: [PATCH 048/110] add_foreign_keys() errors instead of silently dropping action changes The existing-foreign-key dedup compared only columns and other_table, so requesting an FK that already exists with different ON DELETE/ON UPDATE actions was a silent no-op - and with add_foreign_key() raising "already exists", there was no signal that the requested actions were dropped. An exact match (including actions) is still skipped for idempotency; a mismatch now raises AlterError pointing at table.transform(). Also validates that compound foreign keys passed as 4-tuples have the same number of columns on both sides - extra other-columns were being silently discarded, e.g. ("t", ("a",), "other", ("a", "b")) created a single-column key referencing just "a". Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/python-api.rst | 2 ++ sqlite_utils/db.py | 25 ++++++++++++++++++--- tests/test_foreign_keys.py | 46 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 87a1ab7..ff78738 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``db.add_foreign_keys()`` no longer silently ignores requested ``ON DELETE``/``ON UPDATE`` actions when a foreign key with the same columns already exists - it raises ``AlterError`` suggesting ``table.transform()``, since the actions of an existing foreign key cannot be changed in place. Exact duplicates, including actions, are still skipped so repeated calls stay idempotent. The method also now validates that compound foreign keys have the same number of columns on both sides, instead of silently discarding the extra columns. - ``db.ensure_autocommit_on()`` now raises ``TransactionError`` if called while a transaction is open. Assigning ``isolation_level`` commits any pending transaction as a side effect, so entering the block silently committed the caller's open transaction and made a later ``rollback()`` a no-op. - ``sqlite-utils migrate --stop-before`` now exits with an error if the named migration has already been applied. Previously the name passed validation but was only checked against pending migrations, so every migration after it was silently applied - the exact outcome ``--stop-before`` exists to prevent. ``Migrations.apply(db, stop_before=...)`` raises ``ValueError`` in the same situation, before applying anything. - Fixed a regression where ``table.insert(..., pk=..., alter=True)`` raised ``InvalidColumns`` if the primary key column did not exist in the table yet. With ``alter=True`` the check now waits until the record keys are known, so a pk column supplied by the records is added by the alter as it was in 3.x. A pk column found in neither the table nor the records still raises ``InvalidColumns``. diff --git a/docs/python-api.rst b/docs/python-api.rst index 516e2aa..11af1f4 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1636,6 +1636,8 @@ Here's an example adding two foreign keys at once: This method runs the same checks as ``.add_foreign_keys()`` and will raise ``sqlite_utils.db.AlterError`` if those checks fail. +Foreign keys that already exist are silently skipped, so repeated calls are idempotent - but only if they match exactly. Requesting a foreign key that exists with different ``ON DELETE``/``ON UPDATE`` actions raises ``AlterError``: use ``table.transform()`` to change the actions of an existing foreign key. + .. _python_api_index_foreign_keys: Adding indexes for all foreign keys diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 90e7a83..eb59c50 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1786,6 +1786,11 @@ class Database: if isinstance(other_column_or_columns, str) else tuple(other_column_or_columns) ) + if len(columns) != len(other_columns): + raise ValueError( + "Compound foreign key must have the same number of " + "columns on both sides" + ) if len(columns) == 1: fk_object = ForeignKey( table, columns[0], other_table, other_columns[0] @@ -1825,10 +1830,11 @@ class Database: other_column, other_table ) ) - # We will silently skip foreign keys that exist already + # Silently skip foreign keys that exist already - but only if + # they match exactly, including ON DELETE/ON UPDATE actions columns_folded = tuple(fold_identifier_case(c) for c in columns) other_columns_folded = tuple(fold_identifier_case(c) for c in other_columns) - if not any( + existing = [ fk for fk in table_obj.foreign_keys if tuple(fold_identifier_case(c) for c in fk.columns) == columns_folded @@ -1836,8 +1842,21 @@ class Database: == fold_identifier_case(other_table) and tuple(fold_identifier_case(c) for c in fk.other_columns) == other_columns_folded - ): + ] + if not existing: foreign_keys_to_create.append(fk_object) + elif any( + fk.on_delete != fk_object.on_delete + or fk.on_update != fk_object.on_update + for fk in existing + ): + raise AlterError( + "Foreign key already exists for {} => {}.{} but with " + "different ON DELETE/ON UPDATE actions - use " + "table.transform() to change them".format( + ", ".join(columns), other_table, ", ".join(other_columns) + ) + ) # Group them by table by_table: Dict[str, List[ForeignKey]] = {} diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index d7c20f6..b37d374 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -643,3 +643,49 @@ def test_create_table_mixed_foreign_keys_with_string(fresh_db): ) fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} assert fks == {"author_id": "authors", "publisher_id": "publishers"} + + +def test_add_foreign_keys_existing_with_different_actions_errors(fresh_db): + # Requesting an existing foreign key with different ON DELETE/ON UPDATE + # actions was silently skipped, dropping the requested change + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert( + {"id": 1, "author_id": 1}, + pk="id", + foreign_keys=[("author_id", "authors", "id")], + ) + with pytest.raises(AlterError) as ex: + fresh_db.add_foreign_keys( + [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] + ) + assert "ON DELETE" in str(ex.value) + assert fresh_db["books"].foreign_keys[0].on_delete == "NO ACTION" + + +def test_add_foreign_keys_identical_existing_is_noop(fresh_db): + # An exact match, including actions, is silently skipped so repeated + # calls stay idempotent + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db["books"].add_foreign_key("author_id", "authors", "id", on_delete="CASCADE") + fresh_db.add_foreign_keys( + [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] + ) + fks = fresh_db["books"].foreign_keys + assert len(fks) == 1 + assert fks[0].on_delete == "CASCADE" + + +def test_add_foreign_keys_compound_column_count_mismatch_errors(fresh_db): + # Previously the extra other-column was silently discarded, creating + # a single-column foreign key to just ("id") + fresh_db["departments"].insert( + {"campus": "north", "code": "cs"}, pk=("campus", "code") + ) + fresh_db["courses"].insert({"id": 1, "campus": "north"}, pk="id") + with pytest.raises(ValueError) as ex: + fresh_db.add_foreign_keys( + [("courses", ("campus",), "departments", ("campus", "code"))] + ) + assert "same number of columns" in str(ex.value) + assert fresh_db["courses"].foreign_keys == [] From 8572d1e39c3c807bc0643249411fb33bd0881eb6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:55:21 -0700 Subject: [PATCH 049/110] extract() no longer duplicates NULL-containing rows in shared lookups The lookup-table insert relied on INSERT OR IGNORE and the unique index to dedupe against existing rows, but SQLite unique indexes treat NULLs as distinct - extracting a second table into the same lookup table re-inserted every NULL-containing value, growing orphan rows on each extract. The insert now also has an IS-based NOT EXISTS guard, matching how the foreign keys themselves are resolved. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 18 +++++++++++++++++- tests/test_extract.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ff78738..d2b8a67 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed a bug where running ``table.extract()`` more than once against the same lookup table inserted duplicate rows for values containing ``null`` - SQLite unique indexes treat ``NULL`` values as distinct, so ``INSERT OR IGNORE`` alone could not dedupe them. Each repeat extract added another copy that nothing referenced. The insert now uses an ``IS``-based ``NOT EXISTS`` guard so ``null``-containing rows match existing lookup rows. - ``db.add_foreign_keys()`` no longer silently ignores requested ``ON DELETE``/``ON UPDATE`` actions when a foreign key with the same columns already exists - it raises ``AlterError`` suggesting ``table.transform()``, since the actions of an existing foreign key cannot be changed in place. Exact duplicates, including actions, are still skipped so repeated calls stay idempotent. The method also now validates that compound foreign keys have the same number of columns on both sides, instead of silently discarding the extra columns. - ``db.ensure_autocommit_on()`` now raises ``TransactionError`` if called while a transaction is open. Assigning ``isolation_level`` commits any pending transaction as a side effect, so entering the block silently committed the caller's open transaction and made a later ``rollback()`` a no-op. - ``sqlite-utils migrate --stop-before`` now exits with an error if the named migration has already been applied. Previously the name passed validation but was only checked against pending migrations, so every migration after it was silently applied - the exact outcome ``--stop-before`` exists to prevent. ``Migrations.apply(db, stop_before=...)`` raises ``ValueError`` in the same situation, before applying anything. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index eb59c50..e69f996 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2919,8 +2919,22 @@ class Table(Queryable): all_columns_are_null = " AND ".join( "{} IS NULL".format(quote_identifier(c)) for c in columns ) + # INSERT OR IGNORE dedupes against the unique index, but unique + # indexes treat NULLs as distinct - the NOT EXISTS guard uses IS + # comparison so NULL-containing rows match existing lookup rows + # instead of being inserted again + already_in_lookup = " AND ".join( + "{lookup}.{lookup_col} IS {source}.{source_col}".format( + lookup=quote_identifier(table), + lookup_col=quote_identifier(rename.get(column) or column), + source=quote_identifier(self.name), + source_col=quote_identifier(column), + ) + for column in columns + ) self.db.execute( - "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {} WHERE NOT ({all_null})".format( + "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {} " + "WHERE NOT ({all_null}) AND NOT EXISTS (SELECT 1 FROM {lookup} WHERE {already_in_lookup})".format( quote_identifier(table), quote_identifier(self.name), lookup_columns=", ".join( @@ -2928,6 +2942,8 @@ class Table(Queryable): ), table_cols=", ".join(quote_identifier(c) for c in columns), all_null=all_columns_are_null, + lookup=quote_identifier(table), + already_in_lookup=already_in_lookup, ) ) diff --git a/tests/test_extract.py b/tests/test_extract.py index 1c0fa01..c73ee7a 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -271,3 +271,34 @@ def test_extract_null_values_existing_lookup_table_with_null_row(fresh_db): {"id": 10, "name": "Terriana", "species_id": 2}, {"id": 11, "name": "Spenidorm", "species_id": None}, ] + + +def test_extract_repeated_into_shared_lookup_with_nulls(fresh_db): + # Unique indexes treat NULLs as distinct, so INSERT OR IGNORE alone + # cannot dedupe NULL-containing rows against the existing lookup + # table - extracting a second table into the same lookup previously + # inserted duplicate rows that nothing pointed to + fresh_db["t1"].insert_all( + [ + {"id": 1, "species": None, "common": "X"}, + {"id": 2, "species": "Oak", "common": "Oak"}, + ], + pk="id", + ) + fresh_db["t2"].insert_all([{"id": 1, "species": None, "common": "X"}], pk="id") + fresh_db["t1"].extract(["species", "common"], table="lk") + fresh_db["t2"].extract(["species", "common"], table="lk") + assert fresh_db["lk"].count == 2 + # Both tables point at the same lookup row + t1_fk = fresh_db.execute("select lk_id from t1 where id = 1").fetchone()[0] + t2_fk = fresh_db.execute("select lk_id from t2 where id = 1").fetchone()[0] + assert t1_fk == t2_fk + + +def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db): + # Non-NULL rows were already deduped by the unique index - keep it so + fresh_db["t1"].insert_all([{"id": 1, "species": "Oak"}], pk="id") + fresh_db["t2"].insert_all([{"id": 1, "species": "Oak"}], pk="id") + fresh_db["t1"].extract(["species"], table="lk") + fresh_db["t2"].extract(["species"], table="lk") + assert fresh_db["lk"].count == 1 From 548a886ca1d2ba6c259676da2c8e00f022bacc2c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:57:14 -0700 Subject: [PATCH 050/110] Clean CLI errors for InvalidColumns from insert --pk and extract sqlite-utils insert db t - --pk badcol and sqlite-utils extract db t nosuchcol dumped raw InvalidColumns tracebacks - the insert error handling caught NoTable and OperationalError but not the InvalidColumns introduced for #732, and the extract command had no handling at all (including for NoTable when pointed at a view). Both now exit with click-style Error: messages. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/cli.py | 8 ++++++-- tests/test_cli.py | 19 +++++++++++++++++++ tests/test_cli_insert.py | 15 +++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d2b8a67..8c5b3e6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``sqlite-utils insert ... --pk `` and ``sqlite-utils extract `` now show a clean ``Error:`` message instead of a raw Python traceback. The ``extract`` command also shows a clean error when pointed at a view. - Fixed a bug where running ``table.extract()`` more than once against the same lookup table inserted duplicate rows for values containing ``null`` - SQLite unique indexes treat ``NULL`` values as distinct, so ``INSERT OR IGNORE`` alone could not dedupe them. Each repeat extract added another copy that nothing referenced. The insert now uses an ``IS``-based ``NOT EXISTS`` guard so ``null``-containing rows match existing lookup rows. - ``db.add_foreign_keys()`` no longer silently ignores requested ``ON DELETE``/``ON UPDATE`` actions when a foreign key with the same columns already exists - it raises ``AlterError`` suggesting ``table.transform()``, since the actions of an existing foreign key cannot be changed in place. Exact duplicates, including actions, are still skipped so repeated calls stay idempotent. The method also now validates that compound foreign keys have the same number of columns on both sides, instead of silently discarding the extra columns. - ``db.ensure_autocommit_on()`` now raises ``TransactionError`` if called while a transaction is open. Assigning ``isolation_level`` commits any pending transaction as a side effect, so entering the block silently committed the caller's open transaction and made a later ``rollback()`` a no-op. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 188eff6..29dcb9d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -13,6 +13,7 @@ from sqlite_utils.db import ( BadMultiValues, DEFAULT, DescIndex, + InvalidColumns, NoTable, NoView, quote_identifier, @@ -1172,7 +1173,7 @@ def insert_upsert_implementation( db.table(table).insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs ) - except NoTable as e: + except (NoTable, InvalidColumns) as e: raise click.ClickException(str(e)) except Exception as e: if ( @@ -2734,7 +2735,10 @@ def extract( fk_column=fk_column, rename=dict(rename), ) - db.table(table).extract(**kwargs) + try: + db.table(table).extract(**kwargs) + except (NoTable, InvalidColumns) as e: + raise click.ClickException(str(e)) @cli.command(name="insert-files") diff --git a/tests/test_cli.py b/tests/test_cli.py index 9e17969..06e14ea 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2758,3 +2758,22 @@ def test_insert_upsert_strict(tmpdir, method, strict): assert result.exit_code == 0 db = Database(db_path) assert db["items"].strict == strict or not db.supports_strict + + +def test_extract_bad_column_clean_error(db_path): + db = Database(db_path) + db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") + result = CliRunner().invoke(cli.cli, ["extract", db_path, "trees", "nope"]) + assert result.exit_code == 1 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.startswith("Error: Invalid columns") + + +def test_extract_view_clean_error(db_path): + db = Database(db_path) + db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") + db.create_view("v", "select * from trees") + result = CliRunner().invoke(cli.cli, ["extract", db_path, "v", "species"]) + assert result.exit_code == 1 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.startswith("Error:") diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 196590e..5af8a2f 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -676,3 +676,18 @@ def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): assert result.exit_code == 0, result.output assert db["places"].columns_dict["zip"] is str assert db["places"].get(1)["zip"] == "01234" + + +def test_insert_invalid_pk_clean_error(db_path): + # An invalid --pk against an existing table should be a clean CLI + # error, not a raw InvalidColumns traceback + db = Database(db_path) + db["t"].insert({"a": 1}) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "t", "-", "--pk", "badcol"], + input='{"a": 2}', + ) + assert result.exit_code == 1 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.startswith("Error: Invalid primary key column") From 93640a7ddedf008036427a3e5bbd033d17cbe9df Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:58:52 -0700 Subject: [PATCH 051/110] migrate --list is read-only with legacy sqlite-migrate classes too The docs promise --list will not create the database file or the _sqlite_migrations table, but legacy sqlite_migrate.Migrations classes create the table (in the legacy schema) from their pending()/applied() methods. The listing now runs inside a transaction that is rolled back, keeping --list read-only regardless of what the migration class does. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/cli.py | 9 ++++++++- tests/test_cli_migrate.py | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8c5b3e6..881423c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``sqlite-utils migrate --list`` is now read-only even when the migrations file uses the legacy ``sqlite_migrate.Migrations`` class, whose listing methods create the ``_sqlite_migrations`` table as a side effect. The listing now runs inside a transaction that is rolled back. - ``sqlite-utils insert ... --pk `` and ``sqlite-utils extract `` now show a clean ``Error:`` message instead of a raw Python traceback. The ``extract`` command also shows a clean error when pointed at a view. - Fixed a bug where running ``table.extract()`` more than once against the same lookup table inserted duplicate rows for values containing ``null`` - SQLite unique indexes treat ``NULL`` values as distinct, so ``INSERT OR IGNORE`` alone could not dedupe them. Each repeat extract added another copy that nothing referenced. The insert now uses an ``IS``-based ``NOT EXISTS`` guard so ``null``-containing rows match existing lookup rows. - ``db.add_foreign_keys()`` no longer silently ignores requested ``ON DELETE``/``ON UPDATE`` actions when a foreign key with the same columns already exists - it raises ``AlterError`` suggesting ``table.transform()``, since the actions of an existing foreign key cannot be changed in place. Exact duplicates, including actions, are still skipped so repeated calls stay idempotent. The method also now validates that compound foreign keys have the same number of columns on both sides, instead of silently discarding the extra columns. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 29dcb9d..fe7dbe4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3434,7 +3434,14 @@ def migrate(db_path, migrations, stop_before, list_, verbose): # Listing is read-only - don't create the database file db = sqlite_utils.Database(memory=True) _register_db_for_cleanup(db) - _display_migration_list(db, migration_sets) + # Legacy sqlite-migrate classes create the migrations table from + # their pending()/applied() methods - run the listing inside a + # transaction and roll it back so --list stays read-only + db.begin() + try: + _display_migration_list(db, migration_sets) + finally: + db.rollback() return db = sqlite_utils.Database(db_path) diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index 0f1c7ea..0f29e36 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -485,3 +485,23 @@ def test_stop_before_applied_migration_errors(two_migrations): assert "already been applied" in result.output db = sqlite_utils.Database(db_path) assert not db["bar"].exists() + + +def test_list_with_legacy_class_is_read_only(tmpdir): + # Legacy sqlite-migrate classes create the _sqlite_migrations table + # from their pending()/applied() methods - --list must roll that + # back so it stays a read-only operation as documented + path = pathlib.Path(tmpdir) + (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8") + db_path = str(path / "test.db") + db = sqlite_utils.Database(db_path) + db["existing"].insert({"id": 1}) + db.close() + result = CliRunner().invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"] + ) + assert result.exit_code == 0, result.output + assert "first" in result.output + db2 = sqlite_utils.Database(db_path) + assert "_sqlite_migrations" not in db2.table_names() + db2.close() From c2a17744095553c7fcc4706cc5da1d20045a299e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:02:41 -0700 Subject: [PATCH 052/110] Fix two tests that assumed modern SQLite behavior SQLite 3.23.1 rejects a UTF-8 byte order mark before the first token, so the BOM variant of the execute()-prefixed-BEGIN test now skips when the SQLite version does not accept a leading BOM. And versions before 3.36 allowed selecting rowid from a view, returning NULL, rather than raising an error - the pks_and_rows_where() view test now accepts either behavior. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- tests/test_atomic.py | 53 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_rows.py | 16 ++++++++----- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/tests/test_atomic.py b/tests/test_atomic.py index 0d25b84..c3fd02f 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -258,11 +258,21 @@ def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): assert [r["id"] for r in fresh_db["t"].rows] == [1] +def _sqlite_accepts_bom(): + try: + sqlite3.connect(":memory:").execute("\ufeffselect 1") + return True + except sqlite3.OperationalError: + return False + + @pytest.mark.parametrize("begin_sql", ["; begin", "\ufeffbegin"]) def test_execute_prefixed_begin_leaves_transaction_open(fresh_db, begin_sql): # sqlite3 tolerates empty statements and a UTF-8 BOM before the first # real token, so a BEGIN behind either must not be auto-committed # out from under the caller + if begin_sql.startswith("\ufeff") and not _sqlite_accepts_bom(): + pytest.skip("This SQLite version rejects a leading byte order mark") fresh_db["t"].insert({"id": 1}, pk="id") fresh_db.execute(begin_sql) assert fresh_db.conn.in_transaction @@ -327,3 +337,46 @@ def test_query_returning_commits_after_iteration(tmpdir): assert other.execute("select count(*) from t").fetchone()[0] == 2 other.close() db.close() + + +TRIGGER_SQL = """ +create trigger no_bad before insert on t +when new.v = 'bad' +begin + select raise(rollback, 'trigger says no'); +end +""" + + +def test_atomic_preserves_error_from_transaction_destroying_trigger(fresh_db): + # RAISE(ROLLBACK) rolls back the whole transaction and destroys every + # savepoint - atomic()'s cleanup must not mask the IntegrityError + # with "cannot rollback - no transaction is active" + fresh_db.execute("create table t (id integer primary key, v text)") + fresh_db.execute(TRIGGER_SQL) + with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): + with fresh_db.atomic(): + fresh_db.execute("insert into t (v) values ('bad')") + assert not fresh_db.conn.in_transaction + + +def test_nested_atomic_preserves_error_from_transaction_destroying_trigger( + fresh_db, +): + # The nested savepoint branch previously raised + # "no such savepoint" from ROLLBACK TO SAVEPOINT + fresh_db.execute("create table t (id integer primary key, v text)") + fresh_db.execute(TRIGGER_SQL) + with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): + with fresh_db.atomic(): + with fresh_db.atomic(): + fresh_db.execute("insert into t (v) values ('bad')") + assert not fresh_db.conn.in_transaction + + +def test_atomic_preserves_error_from_insert_or_rollback(fresh_db): + fresh_db["t"].insert({"id": 1}, pk="id") + with pytest.raises(sqlite3.IntegrityError): + with fresh_db.atomic(): + fresh_db.execute("insert or rollback into t (id) values (1)") + assert not fresh_db.conn.in_transaction diff --git a/tests/test_rows.py b/tests/test_rows.py index 46417ef..46d4f53 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -115,15 +115,21 @@ def test_rows_where_duplicate_select_columns_are_deduped(fresh_db): def test_pks_and_rows_where_view(fresh_db): # pks_and_rows_where() lives on Queryable so views expose it, but - # SQLite views have no rowid - it has always failed with an - # OperationalError from the generated SQL. Guard against it failing - # earlier with an AttributeError from View lacking Table properties + # SQLite views have no rowid. Modern SQLite (3.36+) raises an + # OperationalError from the generated SQL; older versions returned + # NULL for a view's rowid. Either way it must not fail earlier with + # an AttributeError from View lacking Table-only properties from sqlite_utils.utils import sqlite3 fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.create_view("dog_names", "select name from dogs") - with pytest.raises(sqlite3.OperationalError): - list(fresh_db["dog_names"].pks_and_rows_where()) + try: + result = list(fresh_db["dog_names"].pks_and_rows_where()) + except sqlite3.OperationalError: + pass # SQLite 3.36+: no such column: rowid + else: + # Older SQLite returns NULL rowids for views + assert result == [(None, {"rowid": None, "name": "Cleo"})] def test_pks_and_rows_where_compound_pk_declaration_order(fresh_db): From d9a0fd26e09a7e1c07b7356d0cd25a22695f89c5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:04:00 -0700 Subject: [PATCH 053/110] Transaction cleanup no longer masks transaction-destroying errors A RAISE(ROLLBACK) trigger or INSERT OR ROLLBACK conflict rolls back the entire transaction and destroys every savepoint. The cleanup paths in atomic() and query() then raised OperationalError ("no such savepoint" / "cannot rollback - no transaction is active"), masking the original IntegrityError - breaking user code that catches sqlite3.IntegrityError. Cleanup now checks conn.in_transaction first: if the error already destroyed the transaction there is nothing left to undo, and the original exception propagates. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 22 ++++++++++++++++------ tests/test_query.py | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 881423c..9228fb6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed exception masking when a statement destroys the enclosing transaction. An error such as a ``RAISE(ROLLBACK)`` trigger or ``INSERT OR ROLLBACK`` conflict rolls back the whole transaction, destroying every savepoint - the cleanup in ``db.atomic()`` and ``db.query()`` then failed with ``OperationalError: no such savepoint`` (or ``cannot rollback - no transaction is active``), hiding the original ``IntegrityError`` from code that tried to catch it. Cleanup now checks whether a transaction is still open first, so the original exception propagates. - ``sqlite-utils migrate --list`` is now read-only even when the migrations file uses the legacy ``sqlite_migrate.Migrations`` class, whose listing methods create the ``_sqlite_migrations`` table as a side effect. The listing now runs inside a transaction that is rolled back. - ``sqlite-utils insert ... --pk `` and ``sqlite-utils extract `` now show a clean ``Error:`` message instead of a raw Python traceback. The ``extract`` command also shows a clean error when pointed at a view. - Fixed a bug where running ``table.extract()`` more than once against the same lookup table inserted duplicate rows for values containing ``null`` - SQLite unique indexes treat ``NULL`` values as distinct, so ``INSERT OR IGNORE`` alone could not dedupe them. Each repeat extract added another copy that nothing referenced. The insert now uses an ``IS``-based ``NOT EXISTS`` guard so ``null``-containing rows match existing lookup rows. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e69f996..bec68fc 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -602,8 +602,13 @@ class Database: try: yield self except BaseException: - self.conn.execute("ROLLBACK TO SAVEPOINT {};".format(savepoint)) - self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) + # An error such as a RAISE(ROLLBACK) trigger can destroy + # the whole transaction, savepoints included - cleaning up + # anyway would mask the original exception with + # "no such savepoint" + if self.conn.in_transaction: + self.conn.execute("ROLLBACK TO SAVEPOINT {};".format(savepoint)) + self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) raise else: self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) @@ -612,13 +617,15 @@ class Database: try: yield self except BaseException: - self.conn.execute("ROLLBACK") + # rollback() is a no-op if the error already destroyed the + # transaction, so the original exception propagates + self.rollback() raise else: try: self.conn.execute("COMMIT") except BaseException: - self.conn.execute("ROLLBACK") + self.rollback() raise def begin(self) -> None: @@ -870,8 +877,11 @@ class Database: return (dict(zip(keys, row)) for row in fetched) return (dict(zip(keys, row)) for row in cursor) finally: - if not released: - # An error occurred - undo anything the statement changed + if not released and self.conn.in_transaction: + # An error occurred - undo anything the statement changed. + # If the error itself destroyed the transaction (such as a + # RAISE(ROLLBACK) trigger) the savepoint is already gone + # and there is nothing left to undo self.conn.execute('ROLLBACK TO "sqlite_utils_query"') self.conn.execute('RELEASE "sqlite_utils_query"') diff --git a/tests/test_query.py b/tests/test_query.py index f4aa336..aac5a29 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -280,3 +280,20 @@ def test_execute_returning_dicts(fresh_db): assert fresh_db.execute_returning_dicts("select * from test") == [ {"id": 1, "bar": 2} ] + + +def test_query_preserves_error_from_transaction_destroying_trigger(fresh_db): + # RAISE(ROLLBACK) destroys the savepoint guard - the original + # IntegrityError must propagate, not "no such savepoint" + fresh_db.execute("create table t (id integer primary key, v text)") + fresh_db.execute(""" + create trigger no_bad before insert on t + when new.v = 'bad' + begin + select raise(rollback, 'trigger says no'); + end + """) + with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): + fresh_db.query("insert into t (id, v) values (1, 'bad') returning id") + assert not fresh_db.conn.in_transaction + assert fresh_db.execute("select count(*) from t").fetchone()[0] == 0 From 25824467846ca75db3bd737cb19b5a8e0d01b1b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:06:51 -0700 Subject: [PATCH 054/110] Skip RETURNING-based trigger test on SQLite older than 3.35 The query() exception-masking test uses INSERT ... RETURNING, which is a syntax error on SQLite 3.23.1 - skip it there like the other RETURNING tests. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- tests/test_query.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_query.py b/tests/test_query.py index aac5a29..06847da 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -282,6 +282,10 @@ def test_execute_returning_dicts(fresh_db): ] +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 35, 0), + reason="RETURNING requires SQLite 3.35.0 or higher", +) def test_query_preserves_error_from_transaction_destroying_trigger(fresh_db): # RAISE(ROLLBACK) destroys the savepoint guard - the original # IntegrityError must propagate, not "no such savepoint" From 9a2c58246528109c63afa48af3c0c5b5f943f2b0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:26:58 -0700 Subject: [PATCH 055/110] README tweak ready for v4, refs #769 --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e425461..c444c64 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,12 @@ Python CLI utility and library for manipulating SQLite databases. - [Configure SQLite full-text search](https://sqlite-utils.datasette.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance - Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as changing the type of a column - [Extract columns](https://sqlite-utils.datasette.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data +- [Manage database migrations](https://sqlite-utils.datasette.io/en/stable/migrations.html) using Python migration files and the `sqlite-utils migrate` command - [Install plugins](https://sqlite-utils.datasette.io/en/stable/plugins.html) to add custom SQL functions and additional features -Read more on my blog, in this series of posts on [New features in sqlite-utils](https://simonwillison.net/series/sqlite-utils-features/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/). +Upgrading from sqlite-utils 3.x? See the [4.0 upgrade guide](https://sqlite-utils.datasette.io/en/stable/upgrading.html#upgrading-from-3-x-to-4-0). + +Read more on my blog, in this series of posts on [New features in sqlite-utils](https://simonwillison.net/series/sqlite-utils-features/) and other [entries tagged sqlite-utils](https://simonwillison.net/tags/sqlite-utils/). ## Installation From d34f1bea0b34b7004a56b75b5dd083f89f8cc970 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:33:50 -0700 Subject: [PATCH 056/110] Release 4.0rc4 Refs #186, #554, #566, #625, #732, #769 --- docs/changelog.rst | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9228fb6..0943dca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,10 +4,10 @@ Changelog =========== -.. _unreleased: +.. _v4_0rc4: -Unreleased ----------- +4.0rc4 (2026-07-06) +------------------- - **Breaking change**: ``table.extract()`` - and the ``sqlite-utils extract`` command - no longer extract rows where every extracted column is ``null``. Those rows now keep a ``null`` value in the new foreign key column instead of pointing at an all-``null`` record in the lookup table. When extracting multiple columns, rows are still extracted if at least one of the columns has a value. (:issue:`186`) - The ``extracts=`` option to ``table.insert()`` and friends no longer creates a lookup table record for ``None`` values - the column value stays ``null``. Previously every batch of inserted rows containing a ``None`` value would add a duplicate ``null`` record to the lookup table. diff --git a/pyproject.toml b/pyproject.toml index 01f1735..c1024c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.0rc3" +version = "4.0rc4" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From d314d04215f7337d42c847214861ec7ffe0bf757 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:40:37 -0700 Subject: [PATCH 057/110] Bump GitHub Actions versions --- .github/workflows/publish.yml | 8 ++++---- .github/workflows/test-coverage.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 396e2b8..23b23bd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,9 +12,9 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] os: [ubuntu-latest, windows-latest, macos-latest] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip @@ -29,9 +29,9 @@ jobs: runs-on: ubuntu-latest needs: [test] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.14' cache: pip diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 1f55f4e..7668f1b 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -12,9 +12,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" cache: pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d85cbf8..923de2e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,9 +14,9 @@ jobs: numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest, macos-14] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true From 60811e730509667f702bc08f9bf5fc3fe13b7f45 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 08:24:24 -0700 Subject: [PATCH 058/110] Fix rowid pk and last_rowid regressions in insert/upsert Closes #781, #783 Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900618685 * Fix rowid pk and last_rowid regressions in insert/upsert Two behaviour regressions in the 4.0 insert/upsert rewrite broke callers (notably Datasette's write API) that operate on tables without an explicit primary key. Both are fixed here with regression tests. 1. rowid (and its aliases _rowid_/oid) were rejected as a primary key. Table.pks already reports ["rowid"] for a rowid table, but the new pk validation raised InvalidColumns because rowid is not listed among the table's columns, and the insert success path then raised KeyError when looking up the pk value. rowid aliases are now accepted for rowid tables and resolve directly to the rowid. 2. An ignored insert (INSERT OR IGNORE that matched an existing row) no longer populated last_rowid, and only set last_pk when an explicit pk= was passed. It now locates the existing conflicting row by its primary key values and reports that row's rowid and pk, rather than relying on the connection's last inserted rowid. Add a shared ROWID_ALIASES constant for the rowid alias names. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01E7af8SxFZqiCerJB6MqKnY --- sqlite_utils/db.py | 97 +++++++++++++++++++++++++++++++------------- tests/test_create.py | 90 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 28 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index bec68fc..3033a36 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -53,6 +53,11 @@ except ImportError: SQLITE_MAX_VARS = 999 +# Names that refer to a rowid table's implicit integer primary key. These are +# valid primary key targets even though they are not listed among a table's +# columns. See https://www.sqlite.org/lang_createtable.html#rowid +ROWID_ALIASES = frozenset({"rowid", "_rowid_", "oid"}) + _quote_fts_re = re.compile(r'\s+|(".*?")') _virtual_table_using_re = re.compile( @@ -4343,10 +4348,14 @@ class Table(Queryable): if pk and not hash_id and self.exists(): pk_cols = [pk] if isinstance(pk, str) else list(pk) existing_columns = self.columns_dict + # rowid and its aliases are valid primary keys for a rowid table + # even though they are not listed among the table's columns + rowid_aliases = ROWID_ALIASES if self.use_rowid else frozenset() missing_pk_cols = [ col for col in pk_cols - if resolve_casing(col, existing_columns) not in existing_columns + if col.lower() not in rowid_aliases + and resolve_casing(col, existing_columns) not in existing_columns ] if missing_pk_cols: invalid_pk_error = InvalidColumns( @@ -4512,40 +4521,72 @@ class Table(Queryable): if not upsert and result is not None: ignored_insert = ignore and result.rowcount == 0 if ignored_insert: + # The row was not inserted because it conflicts with an + # existing row. Point last_pk / last_rowid at that existing + # row when we can identify it from the record's primary key + # values, rather than leaving them stale or unset. if list_mode: - first_record_list = cast(Sequence[Any], first_record) - if hash_id: - pass - elif isinstance(pk, str): - pk_index = column_names.index( - resolve_casing(pk, column_names) - ) - self.last_pk = first_record_list[pk_index] - elif pk: - self.last_pk = tuple( - first_record_list[ - column_names.index(resolve_casing(p, column_names)) - ] - for p in pk - ) + first_record_dict = dict( + zip(column_names, cast(Sequence[Any], first_record)) + ) else: first_record_dict = cast(Dict[str, Any], first_record) - if hash_id: - self.last_pk = hash_record( - first_record_dict, hash_id_columns - ) - elif isinstance(pk, str): - self.last_pk = first_record_dict[ - resolve_casing(pk, first_record_dict) + if hash_id: + self.last_pk = hash_record(first_record_dict, hash_id_columns) + elif isinstance(pk, str): + self.last_pk = first_record_dict[ + resolve_casing(pk, first_record_dict) + ] + elif pk: + self.last_pk = tuple( + first_record_dict[resolve_casing(p, first_record_dict)] + for p in pk + ) + # Locate the existing conflicting row using its primary key + # columns so we can report its rowid (and pk if not already + # known). Falls back to leaving them unset if the conflict + # cannot be resolved to a pk lookup (e.g. a UNIQUE column). + key_cols: Optional[List[str]] = None + if isinstance(pk, str): + key_cols = [pk] + elif pk: + key_cols = list(pk) + elif not hash_id and not self.use_rowid: + key_cols = self.pks + if key_cols: + try: + key_values = [ + first_record_dict[resolve_casing(c, first_record_dict)] + for c in key_cols ] - elif pk: - self.last_pk = tuple( - first_record_dict[resolve_casing(p, first_record_dict)] - for p in pk + except KeyError: + key_values = None + if key_values is not None: + where = " and ".join( + "{} = ?".format(quote_identifier(c)) for c in key_cols ) + existing = self.db.execute( + "select rowid from {} where {} limit 1".format( + quote_identifier(self.name), where + ), + key_values, + ).fetchone() + if existing is not None: + self.last_rowid = existing[0] + # On a primary key conflict the record's pk + # values identify the existing row + if self.last_pk is None: + self.last_pk = ( + key_values[0] + if len(key_cols) == 1 + else tuple(key_values) + ) else: self.last_rowid = result.lastrowid - if (hash_id or pk) and self.last_rowid: + # A rowid-alias pk resolves directly to the rowid, so there + # is no separate pk column to look up + rowid_pk = isinstance(pk, str) and pk.lower() in ROWID_ALIASES + if (hash_id or (pk and not rowid_pk)) and self.last_rowid: # Set self.last_pk to the pk(s) for that rowid row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] if hash_id: diff --git a/tests/test_create.py b/tests/test_create.py index 42fa1c4..7fab5e6 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -990,6 +990,96 @@ def test_insert_ignore(fresh_db): assert rows == [{"id": 1, "bar": 2}] +def test_insert_ignore_reports_existing_row(fresh_db): + # An ignored insert (row already exists) should point last_rowid and + # last_pk at the existing conflicting row - see the Datasette insert API + fresh_db["docs"].insert({"id": 1, "title": "Exists"}, pk="id") + # Insert a conflicting row with ignore=True and no explicit pk= + table = fresh_db["docs"].insert({"id": 1, "title": "One"}, ignore=True) + assert table.last_rowid == 1 + assert table.last_pk == 1 + assert list(fresh_db["docs"].rows_where("rowid = ?", [table.last_rowid])) == [ + {"id": 1, "title": "Exists"} + ] + + +@pytest.mark.parametrize("rowid_alias", ("rowid", "_rowid_", "oid")) +@pytest.mark.parametrize("method", ("upsert", "insert_replace", "insert_ignore")) +def test_pk_rowid_alias_on_rowid_table(fresh_db, rowid_alias, method): + # rowid and its aliases are valid primary keys for a rowid table even + # though they are not listed among the table's columns - see the Datasette + # upsert API against tables without an explicit primary key + fresh_db["t"].insert({"title": "Hello"}) + assert fresh_db["t"].pks == ["rowid"] + record = {rowid_alias: 1, "title": "Updated"} + if method == "upsert": + table = fresh_db["t"].upsert(record, pk=rowid_alias) + elif method == "insert_replace": + table = fresh_db["t"].insert(record, pk=rowid_alias, replace=True) + else: + table = fresh_db["t"].insert(record, pk=rowid_alias, ignore=True) + assert table.last_pk == 1 + expected_title = "Hello" if method == "insert_ignore" else "Updated" + assert list(fresh_db["t"].rows) == [{"title": expected_title}] + + +def test_insert_ignore_reports_existing_row_compound_pk(fresh_db): + # Compound primary key variant of the ignored-insert lookup + fresh_db["t"].insert_all([{"a": 1, "b": 2, "note": "first"}], pk=("a", "b")) + table = fresh_db["t"].insert( + {"a": 1, "b": 2, "note": "second"}, pk=("a", "b"), ignore=True + ) + assert table.last_pk == (1, 2) + assert list(fresh_db["t"].rows_where("rowid = ?", [table.last_rowid])) == [ + {"a": 1, "b": 2, "note": "first"} + ] + + +def test_insert_ignore_reports_existing_row_list_mode(fresh_db): + # List-based iteration variant of the ignored-insert lookup + fresh_db["t"].insert_all([["id", "title"], [1, "first"]], pk="id") + table = fresh_db["t"].insert_all( + [["id", "title"], [1, "second"]], pk="id", ignore=True + ) + assert table.last_pk == 1 + assert table.last_rowid == 1 + assert list(fresh_db["t"].rows) == [{"id": 1, "title": "first"}] + + +def test_insert_ignore_hash_id_reports_pk(fresh_db): + # With hash_id the pk is the computed hash; the original record has no id + # column to look up so last_rowid is left unset + first = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id") + table = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id", ignore=True) + assert table.last_pk == first.last_pk + assert table.last_rowid is None + assert fresh_db["dogs"].count == 1 + + +def test_insert_ignore_unresolvable_conflict_leaves_pk_unset(fresh_db): + # When the conflict cannot be resolved to a primary key lookup, last_pk and + # last_rowid are left unset rather than reporting a misleading value + + # rowid table with a UNIQUE column and no primary key: no pk to look up + fresh_db["u"].db.execute("create table u (title text unique)") + fresh_db["u"].insert({"title": "x"}) + table = fresh_db["u"].insert({"title": "x"}, ignore=True) + assert table.last_pk is None + assert table.last_rowid is None + assert fresh_db["u"].count == 1 + + # Conflict on a UNIQUE column other than the primary key: the pk value from + # the record does not match the existing row, so the lookup finds nothing + fresh_db["docs"].db.execute( + "create table docs (id integer primary key, email text unique)" + ) + fresh_db["docs"].insert({"id": 1, "email": "a"}, pk="id") + table = fresh_db["docs"].insert({"id": 2, "email": "a"}, ignore=True) + assert table.last_pk is None + assert table.last_rowid is None + assert fresh_db["docs"].count == 1 + + def test_insert_ignore_with_pk_after_other_table_insert(fresh_db): # https://github.com/simonw/sqlite-utils/issues/554 user = {"id": "abc", "name": "david"} From 8bc9213a8e9c52a2d46ffc14da72fcd5af276840 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 08:40:31 -0700 Subject: [PATCH 059/110] Release 4.0 Refs #781, #783, #769 --- docs/changelog.rst | 31 +++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0943dca..9623ec2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,37 @@ Changelog =========== +.. _v4_0: + +4.0 (2026-07-07) +---------------- + +The 4.0 release includes some minor backwards-incompatible fixes (hence the major version number bump) and introduces three major new features: + +- :ref:`Database migrations `, providing a structured mechanism for evolving a project's schema over time. (:issue:`752`) +- :ref:`Nested transaction support ` via ``db.atomic()``, plus numerous improvements to how transactions work across the library. (:issue:`755`) +- Support for :ref:`compound foreign keys `, including creation, transformation and introspection through :ref:`table.foreign_keys `. (:issue:`594`) + +Other notable changes include: + +- Upserts now use SQLite's ``INSERT ... ON CONFLICT ... DO UPDATE SET`` syntax, detect existing table primary keys automatically and reject records that are missing required primary key values. (:issue:`652`) +- ``db.query()`` now executes immediately and rejects statements that do not return rows; use ``db.execute()`` for writes and DDL. +- CSV and TSV imports now detect column types by default, while inserts into existing tables preserve those tables' column types. (:issue:`679`) +- Foreign key handling now preserves ``ON DELETE``/``ON UPDATE`` actions during transforms and resolves referenced primary keys more accurately. (:issue:`530`) +- Column names passed to Python API methods are now matched case-insensitively, mirroring SQLite's own identifier behavior. (:issue:`760`) +- The command-line tool now emits UTF-8 JSON output by default, with ``--ascii`` available to restore escaped output. (:issue:`625`) +- ``table.extract()`` and ``extracts=`` no longer create lookup table records for all-``null`` values. (:issue:`186`) + +See :ref:`upgrading_3_to_4` for details on backwards-incompatible changes. + +The detailed release notes for the features and fixes shipped during the 4.0 pre-release cycle are available in :ref:`4.0a0 `, :ref:`4.0a1 `, :ref:`4.0rc1 `, :ref:`4.0rc2 `, :ref:`4.0rc3 ` and :ref:`4.0rc4 `. + +Bug fixes since 4.0rc4 +~~~~~~~~~~~~~~~~~~~~~~ + +- Fixed 4.0 regressions in ``insert``/``upsert`` against tables that use SQLite's implicit ``rowid`` primary key. Passing ``pk="rowid"``, ``pk="_rowid_"`` or ``pk="oid"`` now works again for rowid tables, and ``last_pk`` is set correctly. (:issue:`781`) +- Fixed ``insert(..., ignore=True)`` and ``insert_all(..., ignore=True)`` so an ignored insert that conflicts with an existing primary key row now reports that existing row in ``last_rowid`` and ``last_pk`` where possible. This also works for compound primary keys and list-mode inserts. (:issue:`783`) + .. _v4_0rc4: 4.0rc4 (2026-07-06) diff --git a/pyproject.toml b/pyproject.toml index c1024c2..a3a42a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.0rc4" +version = "4.0" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 353baf280d3765b9680a49276945a0e96bc9e238 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 11:51:53 -0700 Subject: [PATCH 060/110] Corrected imports in migrations docs --- docs/migrations.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/migrations.rst b/docs/migrations.rst index cfdbf13..23aa9d8 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -27,7 +27,7 @@ Here is a simple example of a ``migrations.py`` file which creates a table, then .. code-block:: python - from sqlite_utils import Database, Migrations + from sqlite_utils import Migrations migrations = Migrations("creatures") @@ -51,6 +51,8 @@ Once you have a ``Migrations(name)`` collection with one or more migrations regi .. code-block:: python + from sqlite_utils import Database + db = Database("creatures.db") migrations.apply(db) From 619770bf427e47657099f1a4fdaa8777ec4c68d8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 17:52:26 -0700 Subject: [PATCH 061/110] sqlite-utils query - to read SQL from stdin, closes #765 --- docs/changelog.rst | 7 +++++++ docs/cli-reference.rst | 4 ++++ docs/cli.rst | 10 ++++++++++ sqlite_utils/cli.py | 8 ++++++++ tests/test_cli.py | 19 +++++++++++++++++++ 5 files changed, 48 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9623ec2..cf9486a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _v_unreleased: + +Unreleased +---------- + +- ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) + .. _v4_0: 4.0 (2026-07-07) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 49eba52..000f88f 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -109,6 +109,10 @@ See :ref:`cli_query`. "select * from chickens where age > :age" \ -p age 1 + Pass "-" as the SQL to read the query from standard input: + + echo "select * from chickens" | sqlite-utils data.db - + Options: --attach ... Additional databases to attach - specify alias and filepath diff --git a/docs/cli.rst b/docs/cli.rst index 063e80f..bbbbf03 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -29,6 +29,16 @@ The ``sqlite-utils query`` command lets you run queries directly against a SQLit .. note:: In Python: :ref:`db.query() ` CLI reference: :ref:`sqlite-utils query ` +Pass ``-`` as the SQL query to read the query from standard input. This is useful for longer queries that would otherwise require careful shell escaping, or for piping in SQL generated by another tool: + +.. code-block:: bash + + echo "select * from dogs" | sqlite-utils query dogs.db - + +.. code-block:: bash + + sqlite-utils query dogs.db - < query.sql + .. _cli_query_json: Returning JSON diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index fe7dbe4..c6496d4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1893,7 +1893,15 @@ def query( sqlite-utils data.db \\ "select * from chickens where age > :age" \\ -p age 1 + + Pass "-" as the SQL to read the query from standard input: + + \b + echo "select * from chickens" | sqlite-utils data.db - """ + if sql == "-": + # Read SQL from standard input + sql = sys.stdin.read() db = sqlite_utils.Database(path) _register_db_for_cleanup(db) for alias, attach_path in attach: diff --git a/tests/test_cli.py b/tests/test_cli.py index 06e14ea..8196a08 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -782,6 +782,25 @@ def test_query_json(db_path, sql, args, expected): assert expected == result.output.strip() +def test_query_sql_from_stdin(db_path): + # https://github.com/simonw/sqlite-utils/issues/765 + db = Database(db_path) + with db.conn: + db["dogs"].insert_all( + [ + {"id": 1, "age": 4, "name": "Cleo"}, + {"id": 2, "age": 2, "name": "Pancakes"}, + ] + ) + result = CliRunner().invoke( + cli.cli, + ["query", db_path, "-"], + input="select name from dogs order by name", + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output) == [{"name": "Cleo"}, {"name": "Pancakes"}] + + def test_query_json_empty(db_path): result = CliRunner().invoke( cli.cli, From fa5d66bf5377b00e976ebb3bf92d806ed8f54270 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 17:56:57 -0700 Subject: [PATCH 062/110] Update create-table --help to mention real --- docs/cli-reference.rst | 4 ++-- sqlite_utils/cli.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 000f88f..fd1199f 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -930,10 +930,10 @@ See :ref:`cli_create_table`. sqlite-utils create-table my.db people \ id integer \ name text \ - height float \ + height real \ photo blob --pk id - Valid column types are text, integer, float and blob. + Valid column types are text, integer, real, float and blob. Options: --pk TEXT Column to use as primary key diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c6496d4..905963f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1634,10 +1634,10 @@ def create_table( sqlite-utils create-table my.db people \\ id integer \\ name text \\ - height float \\ + height real \\ photo blob --pk id - Valid column types are text, integer, float and blob. + Valid column types are text, integer, real, float and blob. """ db = sqlite_utils.Database(path) _register_db_for_cleanup(db) From 8ee0b7c65cfa34f550672872bffa06183bea9963 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 17:57:29 -0700 Subject: [PATCH 063/110] Fix for rogue quote in enable-fts --help --- docs/cli-reference.rst | 2 +- sqlite_utils/cli.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index fd1199f..e57f4b3 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -1028,7 +1028,7 @@ See :ref:`cli_fts`. Usage: sqlite-utils enable-fts [OPTIONS] PATH TABLE COLUMN... - Enable full-text search for specific table and columns" + Enable full-text search for specific table and columns Example: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 905963f..99a8c64 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -706,7 +706,7 @@ def create_index( def enable_fts( path, table, column, fts4, fts5, tokenize, create_triggers, replace, load_extension ): - """Enable full-text search for specific table and columns" + """Enable full-text search for specific table and columns Example: From cf3373e7b7fb9342d705d7b5b8d52fe09ac46a34 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 17:59:58 -0700 Subject: [PATCH 064/110] Refactor --functions option, improve help --- docs/cli-reference.rst | 11 ++++++----- sqlite_utils/cli.py | 29 ++++++++++++++--------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index e57f4b3..c232881 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -138,8 +138,8 @@ See :ref:`cli_query`. -r, --raw Raw output, first column of first row --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query - --functions TEXT Python code or file path defining custom SQL - functions + --functions TEXT Python code or a file path defining custom SQL + functions; can be used multiple times --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -181,8 +181,8 @@ See :ref:`cli_memory`. sqlite-utils memory animals.csv --schema Options: - --functions TEXT Python code or file path defining custom SQL - functions + --functions TEXT Python code or a file path defining custom SQL + functions; can be used multiple times --attach ... Additional databases to attach - specify alias and filepath --flatten Flatten nested JSON objects, so {"foo": {"bar": @@ -383,7 +383,8 @@ See :ref:`cli_bulk`. Options: --batch-size INTEGER Commit every X records - --functions TEXT Python code or file path defining custom SQL functions + --functions TEXT Python code or a file path defining custom SQL + functions; can be used multiple times --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 99a8c64..edf1634 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -153,6 +153,17 @@ def load_extension_option(fn): )(fn) +def functions_option(fn): + return click.option( + "--functions", + help=( + "Python code or a file path defining custom SQL functions; " + "can be used multiple times" + ), + multiple=True, + )(fn) + + @click.group( cls=DefaultGroup, default="query", @@ -1450,11 +1461,7 @@ def upsert( @click.argument("sql") @click.argument("file", type=click.File("rb"), required=True) @click.option("--batch-size", type=int, default=100, help="Commit every X records") -@click.option( - "--functions", - help="Python code or file path defining custom SQL functions", - multiple=True, -) +@functions_option @import_options @load_extension_option def bulk( @@ -1860,11 +1867,7 @@ def drop_view(path, view, ignore, load_extension): type=(str, str), help="Named :parameters for SQL query", ) -@click.option( - "--functions", - help="Python code or file path defining custom SQL functions", - multiple=True, -) +@functions_option @load_extension_option def query( path, @@ -1937,11 +1940,7 @@ def query( nargs=-1, ) @click.argument("sql") -@click.option( - "--functions", - help="Python code or file path defining custom SQL functions", - multiple=True, -) +@functions_option @click.option( "--attach", type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), From aa300942bfcba0bd788951db0b9070cf291c193a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 18:03:25 -0700 Subject: [PATCH 065/110] Update sqlite-utils convert --help, refs #686 --- docs/cli-reference.rst | 7 +++++-- sqlite_utils/cli.py | 9 ++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c232881..105b19a 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -621,6 +621,11 @@ See :ref:`cli_convert`. "value" is a variable with the column value to be converted. + CODE can also be a reference to a callable that takes the value, for example: + + sqlite-utils convert my.db mytable date r.parsedate + sqlite-utils convert my.db mytable data json.loads --import json + Use "-" for CODE to read Python code from standard input. The following common operations are available as recipe functions: @@ -634,7 +639,6 @@ See :ref:`cli_convert`. errors: 'Optional[object]' = None) -> 'Optional[str]' Parse a date and convert it to ISO date format: yyyy-mm-dd - - dayfirst=True: treat xx as the day in xx/yy/zz - yearfirst=True: treat xx as the year in xx/yy/zz - errors=r.IGNORE to ignore values that cannot be parsed @@ -644,7 +648,6 @@ See :ref:`cli_convert`. False, errors: 'Optional[object]' = None) -> 'Optional[str]' Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS - - dayfirst=True: treat xx as the day in xx/yy/zz - yearfirst=True: treat xx as the year in xx/yy/zz - errors=r.IGNORE to ignore values that cannot be parsed diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index edf1634..318b208 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3054,6 +3054,12 @@ def _generate_convert_help(): "value" is a variable with the column value to be converted. + CODE can also be a reference to a callable that takes the value, for example: + + \b + sqlite-utils convert my.db mytable date r.parsedate + sqlite-utils convert my.db mytable data json.loads --import json + Use "-" for CODE to read Python code from standard input. The following common operations are available as recipe functions: @@ -3067,8 +3073,9 @@ def _generate_convert_help(): ] for name in recipe_names: fn = getattr(recipes, name) + doc = textwrap.dedent(fn.__doc__.rstrip()).replace("\b\n", "") help += "\n\nr.{}{}\n\n\b{}".format( - name, str(inspect.signature(fn)), textwrap.dedent(fn.__doc__.rstrip()) + name, str(inspect.signature(fn)), doc ) help += "\n\n" help += textwrap.dedent(""" From ebafb84c93bd3666e0e6acd24bfafe070f25cdee Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 18:09:12 -0700 Subject: [PATCH 066/110] Allow sqlite-utils upsert to infer --pk from existing table --- docs/cli-reference.rst | 3 ++- docs/cli.rst | 2 ++ sqlite_utils/cli.py | 7 +++++-- tests/test_cli.py | 24 +++++++++++++++++++++--- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 105b19a..5ea9200 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -321,6 +321,8 @@ See :ref:`cli_upsert`. incoming record has a primary key that matches an existing record the existing record will be updated. + If the table already exists and has a primary key, --pk can be omitted. + Example: echo '[ @@ -330,7 +332,6 @@ See :ref:`cli_upsert`. Options: --pk TEXT Columns to use as the primary key, e.g. id - [required] --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON diff --git a/docs/cli.rst b/docs/cli.rst index bbbbf03..e373704 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1593,6 +1593,8 @@ For example: This will update the dog with an ID of 2 to have an age of 4, creating a new record (with a null name) if one does not exist. If a row DOES exist the name will be left as-is. +If the table already exists and has a primary key, you can omit the ``--pk`` option and ``sqlite-utils`` will use that existing primary key. + The command will fail if you reference columns that do not exist on the table. To automatically create missing columns, use the ``--alter`` option. .. note:: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 318b208..5e21f92 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -16,6 +16,7 @@ from sqlite_utils.db import ( InvalidColumns, NoTable, NoView, + PrimaryKeyRequired, quote_identifier, ) from sqlite_utils.plugins import ensure_plugins_loaded, pm, get_plugins @@ -1184,7 +1185,7 @@ def insert_upsert_implementation( db.table(table).insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs ) - except (NoTable, InvalidColumns) as e: + except (NoTable, InvalidColumns, PrimaryKeyRequired) as e: raise click.ClickException(str(e)) except Exception as e: if ( @@ -1372,7 +1373,7 @@ def insert( @cli.command() -@insert_upsert_options(require_pk=True) +@insert_upsert_options() def upsert( path, table, @@ -1408,6 +1409,8 @@ def upsert( an incoming record has a primary key that matches an existing record the existing record will be updated. + If the table already exists and has a primary key, --pk can be omitted. + Example: \b diff --git a/tests/test_cli.py b/tests/test_cli.py index 8196a08..bc5d492 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1235,20 +1235,38 @@ def test_upsert(db_path, tmpdir): ] -def test_upsert_pk_required(db_path, tmpdir): +def test_upsert_pk_inferred_from_existing_table(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") + db = Database(db_path) insert_dogs = [ {"id": 1, "name": "Cleo", "age": 4}, {"id": 2, "name": "Nixie", "age": 4}, ] write_json(json_path, insert_dogs) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", json_path, "--pk", "id"], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output + + write_json( + json_path, + [ + {"id": 1, "age": 5}, + {"id": 2, "age": 5}, + ], + ) result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path], catch_exceptions=False, ) - assert result.exit_code == 2 - assert "Error: Missing option '--pk'" in result.output + assert result.exit_code == 0, result.output + assert list(db.query("select * from dogs order by id")) == [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Nixie", "age": 5}, + ] def test_upsert_analyze(db_path, tmpdir): From 23a21c1d6bc8aa1c17a49b37181aadffd7abb8be Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 18:11:34 -0700 Subject: [PATCH 067/110] Ran Black --- sqlite_utils/cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 5e21f92..b99dcf1 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3077,9 +3077,7 @@ def _generate_convert_help(): for name in recipe_names: fn = getattr(recipes, name) doc = textwrap.dedent(fn.__doc__.rstrip()).replace("\b\n", "") - help += "\n\nr.{}{}\n\n\b{}".format( - name, str(inspect.signature(fn)), doc - ) + help += "\n\nr.{}{}\n\n\b{}".format(name, str(inspect.signature(fn)), doc) help += "\n\n" help += textwrap.dedent(""" You can use these recipes like so: From 569608e40f28893262d727adadddbf3d077112da Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 18:48:57 -0700 Subject: [PATCH 068/110] sqlite-utils insert --code option, closes #684 --- docs/changelog.rst | 1 + docs/cli-reference.rst | 18 ++- docs/cli.rst | 21 ++++ sqlite_utils/cli.py | 232 +++++++++++++++++++++++++++------------ tests/test_cli_insert.py | 150 +++++++++++++++++++++++++ 5 files changed, 347 insertions(+), 75 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cf9486a..a9dd4a3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) +- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for providing a block of Python code (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) .. _v4_0: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5ea9200..5798809 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -230,7 +230,7 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr :: - Usage: sqlite-utils insert [OPTIONS] PATH TABLE FILE + Usage: sqlite-utils insert [OPTIONS] PATH TABLE [FILE] Insert records from FILE into a table, creating the table if it does not already exist. @@ -272,8 +272,20 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr echo 'A bunch of words' | sqlite-utils insert words.db words - \ --text --convert '({"word": w} for w in text.split())' + Instead of a FILE you can use --code to provide a block of Python code that + defines the rows to insert, as either a rows() function that yields + dictionaries or a "rows" iterable. --code can also be a path to a .py file: + + sqlite-utils insert data.db creatures --code ' + def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} + ' --pk id + Options: --pk TEXT Columns to use as the primary key, e.g. id + --code TEXT Python code defining a rows() function or iterable + of rows to insert --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON @@ -315,7 +327,7 @@ See :ref:`cli_upsert`. :: - Usage: sqlite-utils upsert [OPTIONS] PATH TABLE FILE + Usage: sqlite-utils upsert [OPTIONS] PATH TABLE [FILE] Upsert records based on their primary key. Works like 'insert' but if an incoming record has a primary key that matches an existing record the existing @@ -332,6 +344,8 @@ See :ref:`cli_upsert`. Options: --pk TEXT Columns to use as the primary key, e.g. id + --code TEXT Python code defining a rows() function or iterable + of rows to insert --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON diff --git a/docs/cli.rst b/docs/cli.rst index e373704..4ed92ed 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1561,6 +1561,27 @@ The result looks like this: COMMIT; +.. _cli_insert_code: + +Inserting rows generated by Python code +======================================= + +Instead of providing a ``FILE`` to import, you can use the ``--code`` option to pass a block of Python code that generates the rows to insert. This is the command-line equivalent of calling ``db["creatures"].insert_all(rows())`` from the :ref:`Python API `. + +Your code should define either a ``rows()`` function that returns or yields dictionaries, or a ``rows`` iterable such as a list of dictionaries: + +.. code-block:: bash + + sqlite-utils insert data.db creatures --code ' + def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} + ' --pk id + +``--code`` can also be given a path to a Python ``.py`` file. + +The ``--code`` option works with both ``sqlite-utils insert`` and ``sqlite-utils upsert``, and composes with table options such as ``--pk``, ``--replace``, ``--alter``, ``--not-null`` and ``--default``. It cannot be combined with a ``FILE`` argument or with input format options such as ``--csv`` or ``--convert``. + .. _cli_insert_replace: Insert-replacing data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index b99dcf1..4eefc70 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -944,13 +944,19 @@ def insert_upsert_options(*, require_pk=False): required=True, ), click.argument("table"), - click.argument("file", type=click.File("rb", lazy=True), required=True), + click.argument( + "file", type=click.File("rb", lazy=True), required=False + ), click.option( "--pk", help="Columns to use as the primary key, e.g. id", multiple=True, required=require_pk, ), + click.option( + "--code", + help="Python code defining a rows() function or iterable of rows to insert", + ), ) + _import_options + ( @@ -1035,11 +1041,118 @@ def insert_upsert_implementation( bulk_sql=None, functions=None, strict=False, + code=None, ): db = sqlite_utils.Database(path) _register_db_for_cleanup(db) _load_extensions(db, load_extension) _maybe_register_functions(db, functions) + + def _insert_docs(docs, tracker=None): + extra_kwargs = { + "ignore": ignore, + "replace": replace, + "truncate": truncate, + "analyze": analyze, + "strict": strict, + } + if not_null: + extra_kwargs["not_null"] = set(not_null) + if default: + extra_kwargs["defaults"] = dict(default) + if upsert: + extra_kwargs["upsert"] = upsert + + # docs should all be dictionaries + docs = (verify_is_dict(doc) for doc in docs) + + # Apply {"$base64": true, ...} decoding, if needed + docs = (decode_base64_values(doc) for doc in docs) + + # For bulk_sql= we use cursor.executemany() instead + if bulk_sql: + if batch_size: + doc_chunks = chunks(docs, batch_size) + else: + doc_chunks = [docs] + for doc_chunk in doc_chunks: + with db.atomic(): + db.conn.cursor().executemany(bulk_sql, doc_chunk) + return + + # table_names() rather than db.table(), which raises NoTable for + # views before the error handling below can deal with them + table_existed_before_insert = table in db.table_names() + try: + db.table(table).insert_all( + docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs + ) + except (NoTable, InvalidColumns, PrimaryKeyRequired) as e: + raise click.ClickException(str(e)) + except Exception as e: + if ( + isinstance(e, OperationalError) + and e.args + and ( + "has no column named" in e.args[0] or "no such column" in e.args[0] + ) + ): + raise click.ClickException( + "{}\n\nTry using --alter to add additional columns".format( + e.args[0] + ) + ) + # If we can find sql= and parameters= arguments, show those + variables = _find_variables(e.__traceback__, ["sql", "parameters"]) + if "sql" in variables and "parameters" in variables: + raise click.ClickException( + "{}\n\nsql = {}\nparameters = {}".format( + str(e), variables["sql"], variables["parameters"] + ) + ) + else: + raise + # Apply detected types only to a table this command created - + # transforming a pre-existing table would rewrite its column types + # and corrupt values such as TEXT zip codes with leading zeros + if ( + tracker is not None + and not table_existed_before_insert + and db.table(table).exists() + ): + db.table(table).transform(types=tracker.types) + + if code is not None: + if file is not None: + raise click.ClickException("--code cannot be used with a FILE argument") + if any( + [ + flatten, + nl, + csv, + tsv, + empty_null, + lines, + text, + convert, + sniff, + no_headers, + delimiter, + quotechar, + encoding, + ] + ): + raise click.ClickException( + "--code cannot be used with input format options" + ) + _insert_docs(_rows_from_code(code)) + return + + if file is None: + raise click.ClickException( + "Provide either a FILE argument or --code to specify rows to insert" + ) + if (delimiter or quotechar or sniff or no_headers) and not tsv: csv = True if (nl + csv + tsv) >= 2: @@ -1147,78 +1260,7 @@ def insert_upsert_implementation( else: docs = (fn(doc) or doc for doc in docs) - extra_kwargs = { - "ignore": ignore, - "replace": replace, - "truncate": truncate, - "analyze": analyze, - "strict": strict, - } - if not_null: - extra_kwargs["not_null"] = set(not_null) - if default: - extra_kwargs["defaults"] = dict(default) - if upsert: - extra_kwargs["upsert"] = upsert - - # docs should all be dictionaries - docs = (verify_is_dict(doc) for doc in docs) - - # Apply {"$base64": true, ...} decoding, if needed - docs = (decode_base64_values(doc) for doc in docs) - - # For bulk_sql= we use cursor.executemany() instead - if bulk_sql: - if batch_size: - doc_chunks = chunks(docs, batch_size) - else: - doc_chunks = [docs] - for doc_chunk in doc_chunks: - with db.atomic(): - db.conn.cursor().executemany(bulk_sql, doc_chunk) - return - - # table_names() rather than db.table(), which raises NoTable for - # views before the error handling below can deal with them - table_existed_before_insert = table in db.table_names() - try: - db.table(table).insert_all( - docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs - ) - except (NoTable, InvalidColumns, PrimaryKeyRequired) as e: - raise click.ClickException(str(e)) - except Exception as e: - if ( - isinstance(e, OperationalError) - and e.args - and ( - "has no column named" in e.args[0] or "no such column" in e.args[0] - ) - ): - raise click.ClickException( - "{}\n\nTry using --alter to add additional columns".format( - e.args[0] - ) - ) - # If we can find sql= and parameters= arguments, show those - variables = _find_variables(e.__traceback__, ["sql", "parameters"]) - if "sql" in variables and "parameters" in variables: - raise click.ClickException( - "{}\n\nsql = {}\nparameters = {}".format( - str(e), variables["sql"], variables["parameters"] - ) - ) - else: - raise - # Apply detected types only to a table this command created - - # transforming a pre-existing table would rewrite its column types - # and corrupt values such as TEXT zip codes with leading zeros - if ( - tracker is not None - and not table_existed_before_insert - and db.table(table).exists() - ): - db.table(table).transform(types=tracker.types) + _insert_docs(docs, tracker=tracker) # Clean up open file-like objects if sniff_buffer: @@ -1261,6 +1303,7 @@ def insert( table, file, pk, + code, flatten, nl, csv, @@ -1332,6 +1375,17 @@ def insert( \b echo 'A bunch of words' | sqlite-utils insert words.db words - \\ --text --convert '({"word": w} for w in text.split())' + + Instead of a FILE you can use --code to provide a block of Python code + that defines the rows to insert, as either a rows() function that yields + dictionaries or a "rows" iterable. --code can also be a path to a .py file: + + \b + sqlite-utils insert data.db creatures --code ' + def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} + ' --pk id """ try: insert_upsert_implementation( @@ -1367,6 +1421,7 @@ def insert( not_null=not_null, default=default, strict=strict, + code=code, ) except UnicodeDecodeError as ex: raise click.ClickException(UNICODE_ERROR.format(ex)) @@ -1379,6 +1434,7 @@ def upsert( table, file, pk, + code, flatten, nl, csv, @@ -1450,6 +1506,7 @@ def upsert( load_extension=load_extension, silent=silent, strict=strict, + code=code, ) except UnicodeDecodeError as ex: raise click.ClickException(UNICODE_ERROR.format(ex)) @@ -3669,3 +3726,32 @@ def _maybe_register_functions(db, functions_list): for functions in functions_list: if isinstance(functions, str) and functions.strip(): _register_functions(db, functions) + + +def _rows_from_code(code): + # code may be a path to a .py file + if "\n" not in code and code.endswith(".py"): + try: + code = pathlib.Path(code).read_text() + except FileNotFoundError: + raise click.ClickException("File not found: {}".format(code)) + namespace = {} + try: + exec(code, namespace) + except SyntaxError as ex: + raise click.ClickException("Error in --code: {}".format(ex)) + rows = namespace.get("rows") + if callable(rows): + rows = rows() + if isinstance(rows, dict): + rows = [rows] + error = click.ClickException( + "--code must define a 'rows' function or iterable of rows to insert" + ) + if rows is None or isinstance(rows, (str, bytes)): + raise error + try: + iter(rows) + except TypeError: + raise error + return rows diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 5af8a2f..e290f97 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -691,3 +691,153 @@ def test_insert_invalid_pk_clean_error(db_path): assert result.exit_code == 1 assert result.exception is None or isinstance(result.exception, SystemExit) assert result.output.startswith("Error: Invalid primary key column") + + +# --code tests, see https://github.com/simonw/sqlite-utils/issues/684 +CODE_ROWS_FUNCTION = """ +def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} +""" + +CODE_ROWS_ITERABLE = """ +rows = [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, +] +""" + + +@pytest.mark.parametrize("code", (CODE_ROWS_FUNCTION, CODE_ROWS_ITERABLE)) +def test_insert_code(tmpdir, code): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", code, "--pk", "id"], + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert db["creatures"].pks == ["id"] + assert list(db["creatures"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, + ] + + +def test_insert_code_from_file(tmpdir): + db_path = str(tmpdir / "dogs.db") + code_path = str(tmpdir / "gen.py") + with open(code_path, "w") as fp: + fp.write(CODE_ROWS_FUNCTION) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", code_path], + ) + assert result.exit_code == 0, result.output + assert list(Database(db_path)["creatures"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, + ] + + +def test_upsert_code(tmpdir): + db_path = str(tmpdir / "dogs.db") + db = Database(db_path) + db["creatures"].insert_all( + [{"id": 1, "name": "old"}, {"id": 2, "name": "Suna"}], pk="id" + ) + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--pk", "id"], + ) + assert result.exit_code == 0, result.output + assert list(db["creatures"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, + ] + + +def test_insert_code_requires_file_or_code(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke(cli.cli, ["insert", db_path, "creatures"]) + assert result.exit_code == 1 + assert "Provide either a FILE argument or --code" in result.output + + +def test_insert_code_mutually_exclusive_with_file(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "-", "--code", CODE_ROWS_FUNCTION], + input="{}", + ) + assert result.exit_code == 1 + assert "--code cannot be used with a FILE argument" in result.output + + +def test_insert_code_rejects_input_format_options(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--csv"], + ) + assert result.exit_code == 1 + assert "--code cannot be used with input format options" in result.output + + +def test_insert_code_missing_rows(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "x = 1"], + ) + assert result.exit_code == 1 + assert "must define a 'rows' function or iterable" in result.output + + +def test_insert_code_single_dict(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "creatures", + "--code", + 'rows = {"id": 1, "name": "Cleo"}', + "--pk", + "id", + ], + ) + assert result.exit_code == 0, result.output + assert list(Database(db_path)["creatures"].rows) == [{"id": 1, "name": "Cleo"}] + + +def test_insert_code_not_iterable(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "rows = 5"], + ) + assert result.exit_code == 1 + assert "must define a 'rows' function or iterable" in result.output + + +def test_insert_code_syntax_error(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "def rows(:"], + ) + assert result.exit_code == 1 + assert "Error in --code" in result.output + + +def test_insert_code_file_not_found(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "missing.py"], + ) + assert result.exit_code == 1 + assert "File not found: missing.py" in result.output From 092f0919c3c254de0801dda60e7bbf8b73818c6c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 18:57:27 -0700 Subject: [PATCH 069/110] Changelog tweak refs #684 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a9dd4a3..5d1816a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ Unreleased ---------- - ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) -- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for providing a block of Python code (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) +- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) .. _v4_0: From d2ac3765ed9f0516bb0cbc2508a5c3907fb6a71a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 19:17:29 -0700 Subject: [PATCH 070/110] sqlite-utils insert/upsert --type colunm-name type option, closes #131 --- docs/changelog.rst | 1 + docs/cli-reference.rst | 8 +++++++ docs/cli.rst | 19 ++++++++++++++++ sqlite_utils/cli.py | 28 +++++++++++++++++++++++- tests/test_cli_insert.py | 47 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d1816a..9e03f39 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased - ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) +- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept ``--type column-name type`` to :ref:`override the type automatically chosen when the table is created `. This is useful for CSV or TSV columns such as ZIP codes that look like integers but should be stored as ``TEXT`` to preserve leading zeros. (:issue:`131`) .. _v4_0: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5798809..39226ac 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -246,6 +246,9 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr - Use --lines to write each incoming line to a column called "line" - Use --text to write the entire input to a column called "text" + Use --type column-name type to override the type automatically chosen when the + table is created. + You can also use --convert to pass a fragment of Python code that will be used to convert each input. @@ -306,6 +309,7 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr --alter Alter existing table to add any missing columns --not-null TEXT Columns that should be created as NOT NULL --default ... Default value that should be set for a column + --type ... Column types to use when creating the table --no-detect-types Treat all CSV/TSV columns as TEXT --analyze Run ANALYZE at the end of this operation --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -335,6 +339,9 @@ See :ref:`cli_upsert`. If the table already exists and has a primary key, --pk can be omitted. + Use --type column-name type to override the type automatically chosen when the + table is created. + Example: echo '[ @@ -366,6 +373,7 @@ See :ref:`cli_upsert`. --alter Alter existing table to add any missing columns --not-null TEXT Columns that should be created as NOT NULL --default ... Default value that should be set for a column + --type ... Column types to use when creating the table --no-detect-types Treat all CSV/TSV columns as TEXT --analyze Run ANALYZE at the end of this operation --load-extension TEXT Path to SQLite extension, with optional :entrypoint diff --git a/docs/cli.rst b/docs/cli.rst index 4ed92ed..731e93b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1366,6 +1366,25 @@ Will produce this schema with automatically detected types: "weight" REAL ); +.. _cli_insert_csv_tsv_column_types: + +Overriding column types +----------------------- + +Use ``--type column-name type`` to override the type automatically chosen when the table is created. This option can be used more than once, and works with both ``insert`` and ``upsert``: + +.. code-block:: bash + + sqlite-utils insert places.db places places.csv --csv \ + --type zipcode text \ + --type score real + +This is useful for values such as ZIP codes, which may look like integers but should be stored as ``TEXT`` to preserve leading zeros. + +The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL`` or ``BLOB``. Column types are matched case-insensitively. + +As with detected column types, ``--type`` only affects tables created by the command. If the table already exists, its existing column types are left unchanged. + To disable type detection and treat all columns as TEXT, use ``--no-detect-types``: .. code-block:: bash diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4eefc70..cf39ff8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -980,6 +980,16 @@ def insert_upsert_options(*, require_pk=False): type=(str, str), help="Default value that should be set for a column", ), + click.option( + "--type", + "types", + type=( + str, + click.Choice(list(VALID_COLUMN_TYPES), case_sensitive=False), + ), + multiple=True, + help="Column types to use when creating the table", + ), click.option( "--no-detect-types", is_flag=True, @@ -1034,6 +1044,7 @@ def insert_upsert_implementation( truncate=False, not_null=None, default=None, + types=None, no_detect_types=False, analyze=False, load_extension=None, @@ -1047,6 +1058,7 @@ def insert_upsert_implementation( _register_db_for_cleanup(db) _load_extensions(db, load_extension) _maybe_register_functions(db, functions) + column_type_overrides = {column: ctype.upper() for column, ctype in (types or [])} def _insert_docs(docs, tracker=None): extra_kwargs = { @@ -1060,6 +1072,8 @@ def insert_upsert_implementation( extra_kwargs["not_null"] = set(not_null) if default: extra_kwargs["defaults"] = dict(default) + if column_type_overrides: + extra_kwargs["columns"] = column_type_overrides if upsert: extra_kwargs["upsert"] = upsert @@ -1120,7 +1134,9 @@ def insert_upsert_implementation( and not table_existed_before_insert and db.table(table).exists() ): - db.table(table).transform(types=tracker.types) + detected_types = tracker.types + detected_types.update(column_type_overrides) + db.table(table).transform(types=detected_types) if code is not None: if file is not None: @@ -1330,6 +1346,7 @@ def insert( truncate, not_null, default, + types, strict, ): """ @@ -1348,6 +1365,9 @@ def insert( - Use --lines to write each incoming line to a column called "line" - Use --text to write the entire input to a column called "text" + Use --type column-name type to override the type automatically chosen + when the table is created. + You can also use --convert to pass a fragment of Python code that will be used to convert each input. @@ -1420,6 +1440,7 @@ def insert( silent=silent, not_null=not_null, default=default, + types=types, strict=strict, code=code, ) @@ -1454,6 +1475,7 @@ def upsert( alter, not_null, default, + types, no_detect_types, analyze, load_extension, @@ -1467,6 +1489,9 @@ def upsert( If the table already exists and has a primary key, --pk can be omitted. + Use --type column-name type to override the type automatically chosen + when the table is created. + Example: \b @@ -1501,6 +1526,7 @@ def upsert( upsert=True, not_null=not_null, default=default, + types=types, no_detect_types=no_detect_types, analyze=analyze, load_extension=load_extension, diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index e290f97..df6f80c 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -664,6 +664,53 @@ def test_insert_csv_detect_types_new_table(db_path): assert db["data"].columns_dict == {"name": str, "age": int, "weight": float} +@pytest.mark.parametrize( + "command,extra_args,input_text,expected_row", + ( + ( + "insert", + [], + "zipcode,score\n01234,9.5\n", + {"zipcode": "01234", "score": 9.5}, + ), + ( + "upsert", + ["--pk", "id"], + "id,zipcode,score\n1,01234,9.5\n", + {"id": 1, "zipcode": "01234", "score": 9.5}, + ), + ), +) +def test_insert_upsert_csv_type_overrides_detected_types( + db_path, command, extra_args, input_text, expected_row +): + result = CliRunner().invoke( + cli.cli, + [ + command, + db_path, + "places", + "-", + "--csv", + ] + + extra_args + + [ + "--type", + "zipcode", + "text", + ], + catch_exceptions=False, + input=input_text, + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + expected_columns = {"zipcode": str, "score": float} + if command == "upsert": + expected_columns = {"id": int, **expected_columns} + assert db["places"].columns_dict == expected_columns + assert list(db["places"].rows) == [expected_row] + + def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): db = Database(db_path) db["places"].insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id") From d302835d57bcf53c36c0dc67356ec292f55a5931 Mon Sep 17 00:00:00 2001 From: Sai Asish Y Date: Wed, 13 May 2026 11:55:13 -0700 Subject: [PATCH 071/110] feat(db): document and test Database.memory and Database.memory_name Refs #590, closes #734 --- docs/python-api.rst | 8 ++++++++ tests/test_constructor.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 11af1f4..f8e0787 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -109,6 +109,14 @@ You can also create a named in-memory database. Unlike regular memory databases db = Database(memory_name="my_shared_database") +After creating a ``Database`` you can use ``db.memory`` and ``db.memory_name`` to tell whether it is backed by an in-memory database and to read the shared cache name. ``db.memory`` is ``True`` for any in-memory database and ``db.memory_name`` holds the name passed to ``memory_name=``, or ``None`` otherwise. + +.. code-block:: python + + db = Database(memory_name="shared") + db.memory # True + db.memory_name # "shared" + Connections use ``PRAGMA recursive_triggers=on`` by default. If you don't want to use `recursive triggers `__ you can turn them off using: .. code-block:: python diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 7714f26..7428dcf 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -87,3 +87,34 @@ def test_legacy_transaction_control_connection_is_accepted(tmpdir): db["t"].insert({"id": 1}, pk="id") assert [r["id"] for r in db["t"].rows] == [1] db.close() + + +def test_memory_attribute_for_memory_true(): + db = Database(memory=True) + assert db.memory is True + assert db.memory_name is None + + +def test_memory_attribute_for_memory_name(): + db = Database(memory_name="shared_attr") + assert db.memory is True + assert db.memory_name == "shared_attr" + + +def test_memory_attribute_for_memory_string_path(): + db = Database(":memory:") + assert db.memory is True + assert db.memory_name is None + + +def test_memory_attribute_for_file_path(tmpdir): + db = Database(str(tmpdir / "file.db")) + assert db.memory is False + assert db.memory_name is None + + +def test_memory_attribute_for_existing_connection(): + conn = sqlite3.connect(":memory:") + db = Database(conn) + assert db.memory is False + assert db.memory_name is None From 7a52214624ae0e2c3fdf07215c1bcfc1393dbd93 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 21:00:43 -0700 Subject: [PATCH 072/110] drop-index command and table.drop_index(index_name) Closes #626 --- docs/changelog.rst | 1 + docs/cli-reference.rst | 26 +++++++++++++++++++++++++- docs/cli.rst | 13 +++++++++++++ docs/python-api.rst | 8 ++++++++ sqlite_utils/cli.py | 28 ++++++++++++++++++++++++++++ sqlite_utils/db.py | 16 ++++++++++++++++ tests/test_cli.py | 18 ++++++++++++++++++ tests/test_create.py | 28 ++++++++++++++++++++++++++++ 8 files changed, 137 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9e03f39..d2c07da 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ Unreleased - ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept ``--type column-name type`` to :ref:`override the type automatically chosen when the table is created `. This is useful for CSV or TSV columns such as ZIP codes that look like integers but should be stored as ``TEXT`` to preserve leading zeros. (:issue:`131`) +- New ``table.drop_index(name)`` method and ``sqlite-utils drop-index`` command for dropping an index by name. Both accept ``ignore=True``/``--ignore`` to ignore a missing index. (:issue:`626`) .. _v4_0: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 39226ac..71e8377 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -19,7 +19,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. go_first = [ "query", "memory", "insert", "upsert", "bulk", "search", "transform", "extract", "schema", "insert-files", "analyze-tables", "convert", "tables", "views", "rows", - "triggers", "indexes", "create-database", "create-table", "create-index", + "triggers", "indexes", "create-database", "create-table", "create-index", "drop-index", "migrate", "enable-fts", "populate-fts", "rebuild-fts", "disable-fts" ] refs = { @@ -46,6 +46,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. "add-foreign-keys": "cli_add_foreign_keys", "index-foreign-keys": "cli_index_foreign_keys", "create-index": "cli_create_index", + "drop-index": "cli_drop_index", "enable-wal": "cli_wal", "enable-counts": "cli_enable_counts", "bulk": "cli_bulk", @@ -1006,6 +1007,29 @@ See :ref:`cli_create_index`. -h, --help Show this message and exit. +.. _cli_ref_drop_index: + +drop-index +========== + +See :ref:`cli_drop_index`. + +:: + + Usage: sqlite-utils drop-index [OPTIONS] PATH TABLE INDEX + + Drop an index by index name from the specified table + + Example: + + sqlite-utils drop-index chickens.db chickens idx_chickens_name + + Options: + --ignore Ignore if index does not exist + --load-extension TEXT Path to SQLite extension, with optional :entrypoint + -h, --help Show this message and exit. + + .. _cli_ref_migrate: migrate diff --git a/docs/cli.rst b/docs/cli.rst index 731e93b..c446a72 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2615,6 +2615,19 @@ If your column names are already prefixed with a hyphen you'll need to manually Add the ``--analyze`` option to run ``ANALYZE`` against the index after it has been created. +.. _cli_drop_index: + +Dropping indexes +================ + +You can drop an index from an existing table using the ``drop-index`` command: + +.. code-block:: bash + + sqlite-utils drop-index mydb.db mytable idx_mytable_col1 + +Use ``--ignore`` to ignore the error if the index does not exist on that table. + .. _cli_fts: Configuring full-text search diff --git a/docs/python-api.rst b/docs/python-api.rst index f8e0787..fed617e 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2877,6 +2877,14 @@ Use ``if_not_exists=True`` to do nothing if an index with that name already exis Pass ``analyze=True`` to run ``ANALYZE`` against the new index after creating it. +You can drop an index from a table using ``.drop_index(index_name)``: + +.. code-block:: python + + db.table("dogs").drop_index("idx_dogs_name") + +Use ``ignore=True`` to ignore the error if the index does not exist. + .. _python_api_analyze: Optimizing index usage with ANALYZE diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cf39ff8..7fab72b 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -692,6 +692,34 @@ def create_index( ) +@cli.command(name="drop-index") +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +@click.argument("index") +@click.option("--ignore", help="Ignore if index does not exist", is_flag=True) +@load_extension_option +def drop_index(path, table, index, ignore, load_extension): + """ + Drop an index by index name from the specified table + + Example: + + \b + sqlite-utils drop-index chickens.db chickens idx_chickens_name + """ + db = sqlite_utils.Database(path) + _register_db_for_cleanup(db) + _load_extensions(db, load_extension) + try: + db.table(table).drop_index(index, ignore=ignore) + except OperationalError as ex: + raise click.ClickException(str(ex)) + + @cli.command(name="enable-fts") @click.argument( "path", diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3033a36..62e3656 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3080,6 +3080,22 @@ class Table(Queryable): self.db.analyze(created_index_name) return self + def drop_index(self, index_name: str, ignore: bool = False): + """ + Drop an index on this table. + + :param index_name: Name of the index to drop + :param ignore: Set to ``True`` to ignore the error if the index does not exist + """ + if index_name not in {index.name for index in self.indexes}: + if ignore: + return self + raise OperationalError( + "No index named {} on table {}".format(index_name, self.name) + ) + self.db.execute("DROP INDEX {}".format(quote_identifier(index_name))) + return self + def add_column( self, col_name: str, diff --git a/tests/test_cli.py b/tests/test_cli.py index bc5d492..a828c70 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -291,6 +291,24 @@ def test_create_index(db_path): ) +def test_drop_index(db_path): + db = Database(db_path) + db["Gosh"].create_index(["c1"]) + assert [index.name for index in db["Gosh"].indexes] == ["idx_Gosh_c1"] + result = CliRunner().invoke(cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1"]) + assert result.exit_code == 0 + assert db["Gosh"].indexes == [] + + result = CliRunner().invoke(cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1"]) + assert result.exit_code == 1 + assert "No index named idx_Gosh_c1" in result.output + + result = CliRunner().invoke( + cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1", "--ignore"] + ) + assert result.exit_code == 0 + + def test_create_index_analyze(db_path): db = Database(db_path) assert "sqlite_stat1" not in db.table_names() diff --git a/tests/test_create.py b/tests/test_create.py index 7fab5e6..d281eb4 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -809,6 +809,34 @@ def test_create_index_if_not_exists(fresh_db): dogs.create_index(["name"], if_not_exists=True) +def test_drop_index(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is_good_dog": True}) + dogs.create_index(["name"]) + assert [index.name for index in dogs.indexes] == ["idx_dogs_name"] + dogs.drop_index("idx_dogs_name") + assert dogs.indexes == [] + + +def test_drop_index_ignore(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"name": "Cleo"}) + with pytest.raises(OperationalError, match="No index named idx_dogs_name"): + dogs.drop_index("idx_dogs_name") + dogs.drop_index("idx_dogs_name", ignore=True) + + +def test_drop_index_wrong_table(fresh_db): + dogs = fresh_db["dogs"] + cats = fresh_db["cats"] + dogs.insert({"name": "Cleo"}) + cats.insert({"name": "Misty"}) + dogs.create_index(["name"]) + with pytest.raises(OperationalError, match="No index named idx_dogs_name"): + cats.drop_index("idx_dogs_name") + assert [index.name for index in dogs.indexes] == ["idx_dogs_name"] + + def test_create_index_desc(fresh_db): dogs = fresh_db["dogs"] dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True}) From 0f2d525d0686c7cafcd819ab20bf0f89c6bf9f69 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 11 Jul 2026 16:20:37 -0700 Subject: [PATCH 073/110] Clarify transaction documentation Based on extensive digging into how this stuff all works. --- docs/python-api.rst | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index fed617e..d444806 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -320,11 +320,15 @@ Every method in this library that writes to the database - ``insert()``, ``upser The same applies to raw SQL executed with :ref:`db.execute() ` - a write statement is committed as soon as it has run. +Another way to think about this is that each sqlite-utils method call is its own unit of work. If several method calls must either all succeed or all fail, use ``db.atomic()`` to turn them into a single unit of work. + You never need to call ``commit()``, and you do not need to close the database to persist your changes. There are exactly two situations where you need to think about transactions: 1. You want to group several write operations together, so they either all succeed or all fail - use :ref:`db.atomic() `. 2. You are :ref:`managing a transaction yourself ` with ``db.begin()``, in which case nothing is committed until you commit - the library will never commit a transaction you opened. +``with Database(...) as db:`` is not a transaction block. It manages the lifetime of the database connection and closes it on exit. Use ``with db.atomic():`` for a transaction. + .. _python_api_atomic: Grouping changes with db.atomic() @@ -340,6 +344,27 @@ Use ``db.atomic()`` to group multiple operations in a single transaction: The transaction commits when the block exits. If an exception is raised, changes made inside the block will be rolled back. +This matters when several operations represent a single logical change. Without ``db.atomic()``, an earlier method call remains committed if a later one fails: + +.. code-block:: python + + # These are two separate transactions + db.table("accounts").update(1, {"balance": 90}) + db.table("accounts").update(2, {"balance": 110}) + + # These updates either both succeed or both fail + with db.atomic(): + db.table("accounts").update(1, {"balance": 90}) + db.table("accounts").update(2, {"balance": 110}) + +Transactions can also improve performance. Calling ``insert()`` repeatedly outside ``db.atomic()`` creates and commits a separate transaction for every call. For bulk inserts, prefer :ref:`insert_all() `. If you need to call several different methods in a loop, wrap the loop in ``db.atomic()``: + +.. code-block:: python + + with db.atomic(): + for row in rows: + db.table("events").insert(row) + ``db.atomic()`` can be nested. Nested blocks use SQLite savepoints, so an exception in an inner block can roll back to that savepoint without rolling back the entire outer transaction: .. code-block:: python @@ -368,6 +393,8 @@ Write statements executed with :ref:`db.execute() ` follow t db.execute("insert into news (headline) values (?)", ["Dog wins award"]) # Already committed +``db.execute()`` participates in sqlite-utils transaction handling. Calling ``db.conn.execute()`` directly bypasses that policy and leaves transaction handling to Python's underlying ``sqlite3.Connection``. Prefer ``db.execute()`` unless you deliberately need the lower-level API. + If a transaction is open - because the call happens inside a ``db.atomic()`` block, or after ``db.begin()`` - the statement becomes part of that transaction instead, and commits when the transaction commits: .. code-block:: python @@ -399,6 +426,8 @@ You can take full manual control using the ``db.begin()``, ``db.commit()`` and ` The library will never commit a transaction you opened. If you call write methods such as ``insert()`` - or use ``db.atomic()`` - while your transaction is open, they participate in it using SQLite savepoints instead of committing: exiting an ``atomic()`` block releases its savepoint, but nothing is saved to disk until you commit the outer transaction yourself. If you roll back, their changes are rolled back too. +Prefer ``db.atomic()`` or ``db.begin()``, ``db.commit()`` and ``db.rollback()`` over mixing sqlite-utils transaction methods with calls to ``db.conn.commit()``, ``db.conn.rollback()`` or raw transaction-control SQL. Mixing the two layers makes it much harder to tell which layer owns the current transaction. + Two related safeguards to be aware of: - ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, because changing the journal mode would commit it as a side effect. @@ -409,9 +438,11 @@ Two related safeguards to be aware of: Supported connection modes -------------------------- -``db.atomic()`` and the automatic per-method transactions require a connection in Python's default transaction handling mode. Passing a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options to ``Database()`` raises a ``sqlite_utils.db.TransactionError``. +``db.atomic()`` and the automatic per-method transactions currently require a connection using Python's legacy transaction control mode (``sqlite3.LEGACY_TRANSACTION_CONTROL`` on Python 3.12 and later). Passing a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options to ``Database()`` raises a ``sqlite_utils.db.TransactionError``. -This is because ``commit()`` and ``rollback()`` behave differently on those connections - under ``autocommit=True`` they are documented no-ops - which would cause every write made by this library to be silently discarded when the connection closed, rather than failing loudly. +Connections using ``autocommit=False`` are not supported because Python keeps a transaction open continuously. sqlite-utils uses ``Connection.in_transaction`` to distinguish its own transactions from transactions opened by its caller, and that distinction is not available in this mode. + +Connections using ``autocommit=True`` are also currently rejected because sqlite-utils has not formally exposed that as a supported configuration. .. _python_api_table: From 6531a57863ce23d502e504fd8fcd375fbe5cbb7f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 11 Jul 2026 16:28:36 -0700 Subject: [PATCH 074/110] Delete obsolete test_memory_attribute_for_existing_connection test Refs https://github.com/simonw/sqlite-utils/issues/789#issuecomment-4949146177 Closes #789 --- tests/test_constructor.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 7428dcf..a619fba 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -111,10 +111,3 @@ def test_memory_attribute_for_file_path(tmpdir): db = Database(str(tmpdir / "file.db")) assert db.memory is False assert db.memory_name is None - - -def test_memory_attribute_for_existing_connection(): - conn = sqlite3.connect(":memory:") - db = Database(conn) - assert db.memory is False - assert db.memory_name is None From dc61f75a0ba84bca00f88da374df29610e8164cb Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 11 Jul 2026 16:42:59 -0700 Subject: [PATCH 075/110] sqlite-utils upsert optional --pk in changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d2c07da..7333b26 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Unreleased - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept ``--type column-name type`` to :ref:`override the type automatically chosen when the table is created `. This is useful for CSV or TSV columns such as ZIP codes that look like integers but should be stored as ``TEXT`` to preserve leading zeros. (:issue:`131`) - New ``table.drop_index(name)`` method and ``sqlite-utils drop-index`` command for dropping an index by name. Both accept ``ignore=True``/``--ignore`` to ignore a missing index. (:issue:`626`) +- ``sqlite-utils upsert`` can now infer the primary key of an existing table, so ``--pk`` can be omitted when upserting into a table that already has a primary key. .. _v4_0: From b74b72703588863464880232e596001d958df180 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 11 Jul 2026 16:43:37 -0700 Subject: [PATCH 076/110] .transform(strict=) and sqlite-utils transform --strict/--no-strict (#788) * .transform(strict=) and sqlite-utils transform --strict/--no-strict Closes #787 --- docs/changelog.rst | 2 ++ docs/cli-reference.rst | 2 ++ docs/cli.rst | 8 +++++- docs/python-api.rst | 23 +++++++++++++++ sqlite_utils/cli.py | 8 ++++++ sqlite_utils/db.py | 13 ++++++++- tests/test_cli.py | 59 +++++++++++++++++++++++++++++++++++++ tests/test_transform.py | 64 +++++++++++++++++++++++++++++++++++++---- 8 files changed, 171 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7333b26..61f8381 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,8 @@ Unreleased ---------- +- ``table.transform()`` and ``table.transform_sql()`` now accept ``strict=True`` or ``strict=False`` to change a table's SQLite strict mode. Omitting the option, or passing ``strict=None``, preserves the existing mode. (:issue:`787`) +- The ``sqlite-utils transform`` command now accepts ``--strict`` and ``--no-strict`` to change a table's SQLite strict mode. Omitting both options preserves the existing mode. (:issue:`787`) - ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept ``--type column-name type`` to :ref:`override the type automatically chosen when the table is created `. This is useful for CSV or TSV columns such as ZIP codes that look like integers but should be stored as ``TEXT`` to preserve leading zeros. (:issue:`131`) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 71e8377..9fafe28 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -508,6 +508,8 @@ See :ref:`cli_transform_table`. Add a foreign key constraint from a column to another table with another column --drop-foreign-key TEXT Drop foreign key constraint for this column + --strict / --no-strict Enable or disable STRICT mode (default: + preserve current mode) --sql Output SQL without executing it --load-extension TEXT Path to SQLite extension, with optional :entrypoint diff --git a/docs/cli.rst b/docs/cli.rst index c446a72..dce36d9 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2182,7 +2182,7 @@ Use ``--ignore`` to ignore the error if the table does not exist. Transforming tables =================== -The ``transform`` command allows you to apply complex transformations to a table that cannot be implemented using a regular SQLite ``ALTER TABLE`` command. See :ref:`python_api_transform` for details of how this works. The ``transform`` command preserves a table's ``STRICT`` mode. +The ``transform`` command allows you to apply complex transformations to a table that cannot be implemented using a regular SQLite ``ALTER TABLE`` command. See :ref:`python_api_transform` for details of how this works. By default, the ``transform`` command preserves a table's ``STRICT`` mode. .. code-block:: bash @@ -2228,6 +2228,12 @@ Every option for this table (with the exception of ``--pk-none``) can be specifi ``--add-foreign-key column other_table other_column`` Add a foreign key constraint to ``column`` pointing to ``other_table.other_column``. +``--strict`` + Convert the table to a `SQLite STRICT table `__. The command fails if the available SQLite version does not support strict tables. If existing rows contain values that are incompatible with their declared column types the transformation fails and the original table is left unchanged. + +``--no-strict`` + Convert a strict table back to a regular non-strict table. + If you want to see the SQL that will be executed to make the change without actually executing it, add the ``--sql`` flag. For example: .. code-block:: bash diff --git a/docs/python-api.rst b/docs/python-api.rst index d444806..8b07677 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1753,6 +1753,29 @@ To alter the type of a column, use the ``types=`` argument: See :ref:`python_api_add_column` for a list of available types. +.. _python_api_transform_strict: + +Changing strict mode +-------------------- + +The optional ``strict=`` parameter can change whether a table uses `SQLite STRICT mode `__. Pass ``strict=True`` to convert a regular table to a strict table: + +.. code-block:: python + + table.transform(strict=True) + +Pass ``strict=False`` to convert a strict table back to a regular non-strict table: + +.. code-block:: python + + table.transform(strict=False) + +The default is ``strict=None``, which preserves the table's existing strict mode. + +Passing ``strict=True`` raises ``sqlite_utils.db.TransformError`` if the available SQLite version does not support strict tables. + +Converting to a strict table validates all existing rows as they are copied into the replacement table. If a value is incompatible with its declared column type, SQLite raises ``sqlite3.IntegrityError`` and the transformation is rolled back, leaving the original table and its data unchanged. + .. _python_api_transform_rename_columns: Renaming columns diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 7fab72b..e0b8969 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2718,6 +2718,11 @@ def schema( multiple=True, help="Drop foreign key constraint for this column", ) +@click.option( + "--strict/--no-strict", + default=None, + help="Enable or disable STRICT mode (default: preserve current mode)", +) @click.option("--sql", is_flag=True, help="Output SQL without executing it") @load_extension_option def transform( @@ -2735,6 +2740,7 @@ def transform( default_none, add_foreign_keys, drop_foreign_keys, + strict, sql, load_extension, ): @@ -2796,6 +2802,7 @@ def transform( defaults=default_dict, drop_foreign_keys=drop_foreign_keys_value, add_foreign_keys=add_foreign_keys_value, + strict=strict, ): click.echo(line) else: @@ -2809,6 +2816,7 @@ def transform( defaults=default_dict, drop_foreign_keys=drop_foreign_keys_value, add_foreign_keys=add_foreign_keys_value, + strict=strict, ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 62e3656..d709fb9 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2514,6 +2514,7 @@ class Table(Queryable): foreign_keys: Optional[ForeignKeysType] = None, column_order: Optional[List[str]] = None, keep_table: Optional[str] = None, + strict: Optional[bool] = None, ) -> "Table": """ Apply an advanced alter table, including operations that are not supported by @@ -2536,6 +2537,8 @@ class Table(Queryable): to use when creating the table :param keep_table: If specified, the existing table will be renamed to this and will not be dropped + :param strict: Set to ``True`` to make the table strict or ``False`` to make it + non-strict. Defaults to ``None``, which preserves the existing strict mode. """ if not self.exists(): raise ValueError("Cannot transform a table that doesn't exist yet") @@ -2551,6 +2554,7 @@ class Table(Queryable): foreign_keys=foreign_keys, column_order=column_order, keep_table=keep_table, + strict=strict, ) pragma_foreign_keys_was_on = bool( self.db.execute("PRAGMA foreign_keys").fetchone()[0] @@ -2587,6 +2591,8 @@ class Table(Queryable): self.db.execute("PRAGMA defer_foreign_keys=OFF;") if should_disable_foreign_keys: self.db.execute("PRAGMA foreign_keys=1;") + if strict is not None: + self._defaults["strict"] = strict return self def transform_sql( @@ -2604,6 +2610,7 @@ class Table(Queryable): column_order: Optional[List[str]] = None, tmp_suffix: Optional[str] = None, keep_table: Optional[str] = None, + strict: Optional[bool] = None, ) -> List[str]: """ Return a list of SQL statements that should be executed in order to apply this transformation. @@ -2624,7 +2631,11 @@ class Table(Queryable): :param tmp_suffix: Suffix to use for the temporary table name :param keep_table: If specified, the existing table will be renamed to this and will not be dropped + :param strict: Set to ``True`` to make the table strict or ``False`` to make it + non-strict. Defaults to ``None``, which preserves the existing strict mode. """ + if strict is True and not self.db.supports_strict: + raise TransformError("SQLite does not support STRICT tables") types = types or {} rename = rename or {} drop = drop or set() @@ -2806,7 +2817,7 @@ class Table(Queryable): defaults=create_table_defaults, foreign_keys=create_table_foreign_keys, column_order=column_order, - strict=self.strict, + strict=self.strict if strict is None else strict, ).strip() ) diff --git a/tests/test_cli.py b/tests/test_cli.py index a828c70..a2135b0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,7 @@ from sqlite_utils.db import Index, ForeignKey from click.testing import CliRunner from pathlib import Path import subprocess +import sqlite3 import sys import json import os @@ -1939,6 +1940,64 @@ def test_transform_sql(db_path): assert db["dogs"].schema == original_schema +@pytest.mark.parametrize( + "initial_strict,args,expected_strict", + ( + (False, [], False), + (True, [], True), + (False, ["--strict"], True), + (True, ["--no-strict"], False), + ), +) +def test_transform_strict_option(db_path, initial_strict, args, expected_strict): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db["dogs"].create({"id": int}, strict=initial_strict) + + result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs"] + args) + + assert result.exit_code == 0, result.output + assert db["dogs"].strict is expected_strict + + +@pytest.mark.parametrize( + "initial_strict,flag,sql_is_strict", + ( + (False, "--strict", True), + (True, "--no-strict", False), + ), +) +def test_transform_strict_option_sql(db_path, initial_strict, flag, sql_is_strict): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db["dogs"].create({"id": int}, strict=initial_strict) + + result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs", flag, "--sql"]) + + assert result.exit_code == 0, result.output + assert (") STRICT;" in result.output) is sql_is_strict + assert db["dogs"].strict is initial_strict + + +def test_transform_strict_option_with_invalid_data(db_path): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + dogs = db["dogs"] + dogs.create({"id": int}) + dogs.insert({"id": "not-an-integer"}) + + result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs", "--strict"]) + + assert result.exit_code == 1 + assert isinstance(result.exception, sqlite3.IntegrityError) + assert dogs.strict is False + assert list(dogs.rows) == [{"id": "not-an-integer"}] + assert not any(name.startswith("dogs_new_") for name in db.table_names()) + + @pytest.mark.parametrize( "extra_args,expected_schema", ( diff --git a/tests/test_transform.py b/tests/test_transform.py index 71518be..f0f5019 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,3 +1,5 @@ +import sqlite3 + from sqlite_utils.db import ForeignKey, TransformError from sqlite_utils.utils import OperationalError import pytest @@ -566,13 +568,63 @@ def test_transform_preserves_rowids(fresh_db, table_type): assert previous_rows == next_rows -@pytest.mark.parametrize("strict", (False, True)) -def test_transform_strict(fresh_db, strict): - dogs = fresh_db.table("dogs", strict=strict) +@pytest.mark.parametrize( + "initial_strict,transform_strict,expected_strict", + ( + (False, None, False), + (True, None, True), + (False, True, True), + (True, False, False), + ), +) +def test_transform_strict(fresh_db, initial_strict, transform_strict, expected_strict): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + dogs = fresh_db.table("dogs", strict=initial_strict) dogs.insert({"id": 1, "name": "Cleo"}) - assert dogs.strict == strict or not fresh_db.supports_strict - dogs.transform(not_null={"name"}) - assert dogs.strict == strict or not fresh_db.supports_strict + assert dogs.strict is initial_strict + dogs.transform(strict=transform_strict) + assert dogs.strict is expected_strict + + +def test_transform_to_strict_with_invalid_data(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + dogs = fresh_db["dogs"] + dogs.create({"id": int}) + dogs.insert({"id": "not-an-integer"}) + + with pytest.raises(sqlite3.IntegrityError): + dogs.transform(strict=True) + + assert dogs.strict is False + assert list(dogs.rows) == [{"id": "not-an-integer"}] + assert fresh_db.table_names() == ["dogs"] + + +def test_transform_strict_updates_default(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + table = fresh_db.table("items", strict=True) + table.create({"id": int}) + + table.transform(strict=False) + assert table.strict is False + + table.create({"id": int}, replace=True) + assert table.strict is False + + +@pytest.mark.parametrize("method_name", ("transform", "transform_sql")) +def test_transform_to_strict_not_supported(fresh_db, method_name): + table = fresh_db["items"] + table.create({"id": int}) + fresh_db._supports_strict = False + + with pytest.raises(TransformError, match="SQLite does not support STRICT tables"): + getattr(table, method_name)(strict=True) + + assert table.strict is False @pytest.mark.parametrize( From 57c16173912e00c639b9620cadc4dfe475ddb273 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 11 Jul 2026 16:48:41 -0700 Subject: [PATCH 077/110] Release 4.1 Refs #131, #626, #684, #765, #787 --- docs/changelog.rst | 12 ++++++------ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 61f8381..5b9355f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,18 +4,18 @@ Changelog =========== -.. _v_unreleased: +.. _v4_1: -Unreleased ----------- +4.1 (2026-07-11) +---------------- -- ``table.transform()`` and ``table.transform_sql()`` now accept ``strict=True`` or ``strict=False`` to change a table's SQLite strict mode. Omitting the option, or passing ``strict=None``, preserves the existing mode. (:issue:`787`) -- The ``sqlite-utils transform`` command now accepts ``--strict`` and ``--no-strict`` to change a table's SQLite strict mode. Omitting both options preserves the existing mode. (:issue:`787`) -- ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept ``--type column-name type`` to :ref:`override the type automatically chosen when the table is created `. This is useful for CSV or TSV columns such as ZIP codes that look like integers but should be stored as ``TEXT`` to preserve leading zeros. (:issue:`131`) - New ``table.drop_index(name)`` method and ``sqlite-utils drop-index`` command for dropping an index by name. Both accept ``ignore=True``/``--ignore`` to ignore a missing index. (:issue:`626`) +- ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) - ``sqlite-utils upsert`` can now infer the primary key of an existing table, so ``--pk`` can be omitted when upserting into a table that already has a primary key. +- ``table.transform()`` and ``table.transform_sql()`` now accept ``strict=True`` or ``strict=False`` to change a table's `SQLite strict mode `__. Omitting the option preserves the existing mode. (:issue:`787`) +- The ``sqlite-utils transform`` command now accepts ``--strict`` and ``--no-strict`` to change a table's strict mode. (:issue:`787`) .. _v4_0: diff --git a/pyproject.toml b/pyproject.toml index a3a42a9..971f5a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.0" +version = "4.1" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 5e8822efd27da9bc8495990f1decadea5f2b3167 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 11 Jul 2026 18:52:09 -0700 Subject: [PATCH 078/110] Clarify usage of named parameters in CLI documentation --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index dce36d9..8f65f45 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -361,7 +361,7 @@ To return the first column of each result as raw data, separated by newlines, us Using named parameters ---------------------- -You can pass named parameters to the query using ``-p``: +You can pass named parameters to the query using ``-p name value``: .. code-block:: bash From 3f0471701b5f8c7de888a467020e5dd34310ce9a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 11 Jul 2026 21:45:21 -0700 Subject: [PATCH 079/110] Add cross-reference notes between CLI and Python API documentation (#791) --- docs/cli.rst | 96 +++++++++++++++++++++++++++++++++++++ docs/python-api.rst | 114 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 8f65f45..2e506dd 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -424,6 +424,9 @@ The ``--functions`` option can be used multiple times to load functions from mul from urllib.parse import urlparse return urlparse(url).path' +.. note:: + In Python: :ref:`db.register_function() ` + .. _cli_query_extensions: SQLite extensions @@ -1024,6 +1027,9 @@ To show more than 10 common values, use ``--common-limit 20``. To skip the most sqlite-utils analyze-tables github.db tags --common-limit 20 --no-least +.. note:: + In Python: :ref:`table.analyze_column() ` CLI reference: :ref:`sqlite-utils analyze-tables ` + .. _cli_analyze_tables_save: Saving the analyzed table details @@ -1191,6 +1197,9 @@ You can delete all the existing rows in the table before inserting the new recor You can add the ``--analyze`` option to run ``ANALYZE`` against the table after the rows have been inserted. +.. note:: + In Python: :ref:`table.insert_all() ` CLI reference: :ref:`sqlite-utils insert ` + .. _cli_inserting_data_binary: Inserting binary data @@ -1615,6 +1624,9 @@ To replace a dog with in ID of 2 with a new record, run the following: echo '{"id": 2, "name": "Pancakes", "age": 3}' | \ sqlite-utils insert dogs.db dogs - --pk=id --replace +.. note:: + In Python: :ref:`table.insert(..., replace=True) ` CLI reference: :ref:`sqlite-utils insert ` + .. _cli_upsert: Upserting data @@ -1641,6 +1653,9 @@ The command will fail if you reference columns that do not exist on the table. T ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 `__ for details of this change. +.. note:: + In Python: :ref:`table.upsert() ` CLI reference: :ref:`sqlite-utils upsert ` + .. _cli_bulk: Executing SQL in bulk @@ -1842,6 +1857,9 @@ You can include named parameters in your where clause and populate them using on The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. +.. note:: + In Python: :ref:`table.convert() ` CLI reference: :ref:`sqlite-utils convert ` + .. _cli_convert_import: Importing additional modules @@ -2140,6 +2158,9 @@ If a table with the same name already exists, you will get an error. You can cho You can also pass ``--transform`` to transform the existing table to match the new schema. See :ref:`python_api_explicit_create` in the Python library documentation for details of how this option works. +.. note:: + In Python: :ref:`table.create() ` CLI reference: :ref:`sqlite-utils create-table ` + .. _cli_renaming_tables: Renaming a table @@ -2153,6 +2174,9 @@ Yo ucan rename a table using the ``rename-table`` command: Pass ``--ignore`` to ignore any errors caused by the table not existing, or the new name already being in use. +.. note:: + In Python: :ref:`db.rename_table() ` CLI reference: :ref:`sqlite-utils rename-table ` + .. _cli_duplicate_table: Duplicating tables @@ -2164,6 +2188,9 @@ The ``duplicate`` command duplicates a table - creating a new table with the sam sqlite-utils duplicate books.db authors authors_copy +.. note:: + In Python: :ref:`table.duplicate() ` CLI reference: :ref:`sqlite-utils duplicate ` + .. _cli_drop_table: Dropping tables @@ -2177,6 +2204,9 @@ You can drop a table using the ``drop-table`` command: Use ``--ignore`` to ignore the error if the table does not exist. +.. note:: + In Python: :ref:`table.drop() ` CLI reference: :ref:`sqlite-utils drop-table ` + .. _cli_transform_table: Transforming tables @@ -2260,6 +2290,9 @@ If you want to see the SQL that will be executed to make the change without actu DROP TABLE "roadside_attractions"; ALTER TABLE "roadside_attractions_new_4033a60276b9" RENAME TO "roadside_attractions"; +.. note:: + In Python: :ref:`table.transform() ` CLI reference: :ref:`sqlite-utils transform ` + .. _cli_transform_table_add_primary_key_to_rowid: Adding a primary key to a rowid table @@ -2437,6 +2470,9 @@ After running the above, the command ``sqlite-utils schema global.db`` reveals t CREATE UNIQUE INDEX "idx_countries_country_name" ON "countries" ("country", "name"); +.. note:: + In Python: :ref:`table.extract() ` CLI reference: :ref:`sqlite-utils extract ` + .. _cli_create_view: Creating views @@ -2458,6 +2494,9 @@ You can create a view using the ``create-view`` command: Use ``--replace`` to replace an existing view of the same name, and ``--ignore`` to do nothing if a view already exists. +.. note:: + In Python: :ref:`db.create_view() ` CLI reference: :ref:`sqlite-utils create-view ` + .. _cli_drop_view: Dropping views @@ -2471,6 +2510,9 @@ You can drop a view using the ``drop-view`` command: Use ``--ignore`` to ignore the error if the view does not exist. +.. note:: + In Python: :ref:`view.drop() ` CLI reference: :ref:`sqlite-utils drop-view ` + .. _cli_add_column: Adding columns @@ -2511,6 +2553,9 @@ You can set a ``NOT NULL DEFAULT 'x'`` constraint on the new column using ``--no sqlite-utils add-column mydb.db dogs friends_count integer --not-null-default 0 +.. note:: + In Python: :ref:`table.add_column() ` CLI reference: :ref:`sqlite-utils add-column ` + .. _cli_add_column_alter: Adding columns automatically on insert/update @@ -2522,6 +2567,9 @@ You can use the ``--alter`` option to automatically add new columns if the data sqlite-utils insert dogs.db dogs new-dogs.json --pk=id --alter +.. note:: + In Python: :ref:`table.insert(..., alter=True) ` + .. _cli_add_foreign_key: Adding foreign key constraints @@ -2549,6 +2597,9 @@ Add ``--ignore`` to ignore an existing foreign key (as opposed to returning an e See :ref:`python_api_add_foreign_key` in the Python API documentation for further details, including how the automatic table guessing mechanism works. +.. note:: + In Python: :ref:`table.add_foreign_key() ` CLI reference: :ref:`sqlite-utils add-foreign-key ` + .. _cli_add_foreign_keys: Adding multiple foreign keys at once @@ -2564,6 +2615,9 @@ Adding a foreign key requires a ``VACUUM``. On large databases this can be an ex When you are using this command each foreign key needs to be defined in full, as four arguments - the table, column, other table and other column. +.. note:: + In Python: :ref:`db.add_foreign_keys() ` CLI reference: :ref:`sqlite-utils add-foreign-keys ` + .. _cli_index_foreign_keys: Adding indexes for all foreign keys @@ -2575,6 +2629,9 @@ If you want to ensure that every foreign key column in your database has a corre sqlite-utils index-foreign-keys books.db +.. note:: + In Python: :ref:`db.index_foreign_keys() ` CLI reference: :ref:`sqlite-utils index-foreign-keys ` + .. _cli_defaults_not_null: Setting defaults and not null constraints @@ -2590,6 +2647,9 @@ You can use the ``--not-null`` and ``--default`` options (to both ``insert`` and --default age 2 \ --default score 5 +.. note:: + In Python: :ref:`not_null= and defaults= arguments ` + .. _cli_create_index: Creating indexes @@ -2621,6 +2681,9 @@ If your column names are already prefixed with a hyphen you'll need to manually Add the ``--analyze`` option to run ``ANALYZE`` against the index after it has been created. +.. note:: + In Python: :ref:`table.create_index() ` CLI reference: :ref:`sqlite-utils create-index ` + .. _cli_drop_index: Dropping indexes @@ -2634,6 +2697,9 @@ You can drop an index from an existing table using the ``drop-index`` command: Use ``--ignore`` to ignore the error if the index does not exist on that table. +.. note:: + In Python: :ref:`table.drop_index() ` CLI reference: :ref:`sqlite-utils drop-index ` + .. _cli_fts: Configuring full-text search @@ -2687,6 +2753,9 @@ You can rebuild every FTS table by running ``rebuild-fts`` without passing any t sqlite-utils rebuild-fts mydb.db +.. note:: + In Python: :ref:`table.enable_fts() ` CLI reference: :ref:`sqlite-utils enable-fts ` + .. _cli_search: Executing searches @@ -2743,6 +2812,9 @@ Use the ``--sql`` option to output the SQL that would be executed, rather than r order by "documents_fts".rank +.. note:: + In Python: :ref:`table.search() ` CLI reference: :ref:`sqlite-utils search ` + .. _cli_enable_counts: Enabling cached counts @@ -2766,6 +2838,9 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y sqlite-utils reset-counts mydb.db +.. note:: + In Python: :ref:`table.enable_counts() ` CLI reference: :ref:`sqlite-utils enable-counts ` + .. _cli_analyze: Optimizing index usage with ANALYZE @@ -2789,6 +2864,9 @@ You can run it against specific tables, or against specific named indexes, by pa You can also run ``ANALYZE`` as part of another command using the ``--analyze`` option. This is supported by the ``create-index``, ``insert`` and ``upsert`` commands. +.. note:: + In Python: :ref:`db.analyze() ` CLI reference: :ref:`sqlite-utils analyze ` + .. _cli_vacuum: Vacuum @@ -2800,6 +2878,9 @@ You can run VACUUM to optimize your database like so: sqlite-utils vacuum mydb.db +.. note:: + In Python: :ref:`db.vacuum() ` CLI reference: :ref:`sqlite-utils vacuum ` + .. _cli_optimize: Optimize @@ -2823,6 +2904,9 @@ To optimize specific tables rather than every FTS table, pass those tables as ex sqlite-utils optimize mydb.db table_1 table_2 +.. note:: + In Python: :ref:`table.optimize() ` CLI reference: :ref:`sqlite-utils optimize ` + .. _cli_wal: WAL mode @@ -2842,6 +2926,9 @@ You can disable WAL mode using ``disable-wal``: Both of these commands accept one or more database files as arguments. +.. note:: + In Python: :ref:`db.enable_wal() and db.disable_wal() ` CLI reference: :ref:`sqlite-utils enable-wal ` + .. _cli_dump: Dumping the database to SQL @@ -2857,6 +2944,9 @@ The ``dump`` command outputs a SQL dump of the schema and full contents of the s ... COMMIT; +.. note:: + In Python: :ref:`db.iterdump() ` CLI reference: :ref:`sqlite-utils dump ` + .. _cli_load_extension: Loading SQLite extensions @@ -2904,6 +2994,9 @@ Eight (case-insensitive) types are allowed: * GEOMETRYCOLLECTION * GEOMETRY +.. note:: + In Python: :ref:`table.add_geometry_column() ` CLI reference: :ref:`sqlite-utils add-geometry-column ` + .. _cli_spatialite_indexes: Adding spatial indexes @@ -2917,6 +3010,9 @@ Once you have a geometry column, you can speed up bounding box queries by adding See this `SpatiaLite Cookbook recipe `__ for examples of how to use a spatial index. +.. note:: + In Python: :ref:`table.create_spatial_index() ` CLI reference: :ref:`sqlite-utils create-spatial-index ` + .. _cli_install: Installing packages diff --git a/docs/python-api.rst b/docs/python-api.rst index 8b07677..1ed238e 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -184,6 +184,9 @@ You can attach an additional database using the ``.attach()`` method, providing You can reference tables in the attached database using the alias value you passed to ``db.attach(alias, filepath)`` as a prefix, for example the ``second.table_in_second`` reference in the SQL query above. +.. note:: + In the CLI: :ref:`sqlite-utils --attach ` + .. _python_api_tracing: Tracing queries @@ -254,6 +257,9 @@ If a query returns more than one column with the same name - a join between two A suffix that would collide with another column in the query is skipped - ``select 1 as id, 2 as id, 3 as id_2`` returns ``{'id': 1, 'id_3': 2, 'id_2': 3}``. The same renaming is applied by ``table.rows_where()`` and ``table.search()``. +.. note:: + In the CLI: :ref:`sqlite-utils query ` + .. _python_api_execute: db.execute(sql, params) @@ -497,6 +503,9 @@ You can also iterate through the table objects themselves using the ``.tables`` >>> db.tables [] +.. note:: + In the CLI: :ref:`sqlite-utils tables ` + .. _python_api_views: Listing views @@ -522,6 +531,9 @@ View objects are similar to Table objects, except that any attempts to insert or * ``rows_where(where, where_args, order_by, select)`` * ``drop()`` +.. note:: + In the CLI: :ref:`sqlite-utils views ` + .. _python_api_rows: Listing rows @@ -577,6 +589,9 @@ This method also accepts ``offset=`` and ``limit=`` arguments, for specifying an ... print(row) {'id': 1, 'age': 4, 'name': 'Cleo'} +.. note:: + In the CLI: :ref:`sqlite-utils rows ` + .. _python_api_rows_count_where: Counting rows @@ -661,6 +676,9 @@ The ``db.schema`` property returns the full SQL schema for the database as a str "name" TEXT ); +.. note:: + In the CLI: :ref:`sqlite-utils schema ` + .. _python_api_creating_tables: Creating tables @@ -809,6 +827,9 @@ You can pass ``strict=True`` to create a table in ``STRICT`` mode: "name": str, }, strict=True) +.. note:: + In the CLI: :ref:`sqlite-utils create-table ` + .. _python_api_compound_primary_keys: Compound primary keys @@ -989,6 +1010,9 @@ Here's an example that uses these features: # ) +.. note:: + In the CLI: :ref:`sqlite-utils insert --not-null and --default ` + .. _python_api_rename_table: Renaming a table @@ -1006,6 +1030,9 @@ This executes the following SQL: ALTER TABLE [my_table] RENAME TO [new_name_for_my_table] +.. note:: + In the CLI: :ref:`sqlite-utils rename-table ` + .. _python_api_duplicate: Duplicating tables @@ -1021,6 +1048,9 @@ The new ``authors_copy`` table will now contain a duplicate copy of the data fro This method raises ``sqlite_utils.db.NoTable`` if the table does not exist. +.. note:: + In the CLI: :ref:`sqlite-utils duplicate ` + .. _python_api_bulk_inserts: Bulk inserts @@ -1063,6 +1093,9 @@ You can delete all the existing rows in the table before inserting the new recor Pass ``analyze=True`` to run ``ANALYZE`` against the table after inserting the new records. +.. note:: + In the CLI: :ref:`sqlite-utils insert ` + .. _python_api_insert_lists: Inserting data from a list or tuple iterator @@ -1140,6 +1173,9 @@ To replace any existing records that have a matching primary key, use the ``repl .. note:: Prior to sqlite-utils 2.0 the ``.upsert()`` and ``.upsert_all()`` methods worked the same way as ``.insert(replace=True)`` does today. See :ref:`python_api_upsert` for the new behaviour of those methods introduced in 2.0. +.. note:: + In the CLI: :ref:`sqlite-utils insert --replace ` + .. _python_api_update: Updating a specific record @@ -1225,6 +1261,9 @@ Every record passed to ``upsert()`` or ``upsert_all()`` must include a value for .. note:: ``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 `__ for details of this change. +.. note:: + In the CLI: :ref:`sqlite-utils upsert ` + .. _python_api_old_upsert: Alternative upserts using INSERT OR IGNORE @@ -1576,6 +1615,9 @@ You can set a ``NOT NULL DEFAULT 'x'`` constraint on the new column using ``not_ db.table("dogs").add_column("friends_count", int, not_null_default=0) +.. note:: + In the CLI: :ref:`sqlite-utils add-column ` + .. _python_api_add_column_alter: Adding columns automatically on insert/update @@ -1599,6 +1641,9 @@ You can insert or update data that includes new columns and have the table autom new_table = db.table("new_table", alter=True) new_table.insert({"name": "Gareth", "age": 32, "shoe_size": 11}) +.. note:: + In the CLI: :ref:`sqlite-utils insert --alter ` + .. _python_api_add_foreign_key: Adding foreign key constraints @@ -1657,6 +1702,9 @@ Use ``on_delete=`` and ``on_update=`` to specify ``ON DELETE`` and ``ON UPDATE`` This creates a foreign key with an ``ON DELETE CASCADE`` clause, so deleting an author will also delete their books (provided foreign key enforcement is enabled with ``PRAGMA foreign_keys = ON``). Valid actions are ``"SET NULL"``, ``"SET DEFAULT"``, ``"CASCADE"``, ``"RESTRICT"`` and the default ``"NO ACTION"``. +.. note:: + In the CLI: :ref:`sqlite-utils add-foreign-key ` + .. _python_api_add_foreign_keys: Adding multiple foreign key constraints at once @@ -1677,6 +1725,9 @@ This method runs the same checks as ``.add_foreign_keys()`` and will raise ``sql Foreign keys that already exist are silently skipped, so repeated calls are idempotent - but only if they match exactly. Requesting a foreign key that exists with different ``ON DELETE``/``ON UPDATE`` actions raises ``AlterError``: use ``table.transform()`` to change the actions of an existing foreign key. +.. note:: + In the CLI: :ref:`sqlite-utils add-foreign-keys ` + .. _python_api_index_foreign_keys: Adding indexes for all foreign keys @@ -1690,6 +1741,9 @@ If you want to ensure that every foreign key column in your database has a corre Compound foreign keys get a single composite index across their columns. +.. note:: + In the CLI: :ref:`sqlite-utils index-foreign-keys ` + .. _python_api_drop: Dropping a table or view @@ -1711,6 +1765,9 @@ Pass ``ignore=True`` if you want to ignore the error caused by the table or view db.table("my_table").drop(ignore=True) +.. note:: + In the CLI: :ref:`sqlite-utils drop-table ` and :ref:`sqlite-utils drop-view ` + .. _python_api_transform: Transforming a table @@ -1739,6 +1796,9 @@ To keep the original table around instead of dropping it, pass the ``keep_table= This method raises a ``sqlite_utils.db.TransformError`` exception if the table cannot be transformed, usually because there are existing constraints or indexes that are incompatible with modifications to the columns. +.. note:: + In the CLI: :ref:`sqlite-utils transform ` + .. _python_api_transform_alter_column_types: Altering column types @@ -2094,6 +2154,9 @@ This produces a lookup table like so: Rows where every extracted column is ``null`` are not extracted: no record is created for them in the lookup table and their foreign key column is left as ``null``. When extracting multiple columns, rows where at least one of the extracted columns has a value will be extracted as usual. +.. note:: + In the CLI: :ref:`sqlite-utils extract ` + .. _python_api_hash: Setting an ID based on the hash of the row contents @@ -2155,6 +2218,9 @@ You can pass ``ignore=True`` to silently ignore an existing view and do nothing, select * from dogs where is_good_dog = 1 """, replace=True) +.. note:: + In the CLI: :ref:`sqlite-utils create-view ` + Storing JSON ============ @@ -2273,6 +2339,9 @@ If you are using ``pysqlite3`` the underlying method may be missing. If you inst pip install sqlite-dump +.. note:: + In the CLI: :ref:`sqlite-utils dump ` + .. _python_api_introspection: Introspecting tables and views @@ -2472,6 +2541,9 @@ The ``.indexes`` property returns all indexes created for a table, as a list of Index(seq=4, name='"Street_Tree_List_qCaretaker"', unique=0, origin='c', partial=0, columns=['qCaretaker']), Index(seq=5, name='"Street_Tree_List_PlantType"', unique=0, origin='c', partial=0, columns=['PlantType'])] +.. note:: + In the CLI: :ref:`sqlite-utils indexes ` + .. _python_api_introspection_xindexes: .xindexes @@ -2515,6 +2587,9 @@ The ``.triggers`` property lists database triggers. It can be used on both datab >>> db.triggers ... similar output to db.table("authors").triggers +.. note:: + In the CLI: :ref:`sqlite-utils triggers ` + .. _python_api_introspection_triggers_dict: .triggers_dict @@ -2663,6 +2738,9 @@ To remove the FTS tables and triggers you created, use the ``disable_fts()`` tab db.table("dogs").disable_fts() +.. note:: + In the CLI: :ref:`sqlite-utils enable-fts ` + .. _python_api_quote_fts: Quoting characters for use in search @@ -2727,6 +2805,9 @@ To return just the title and published columns for three matches for ``"dog"`` w ): print(article) +.. note:: + In the CLI: :ref:`sqlite-utils search ` + .. _python_api_fts_search_sql: Building SQL queries with table.search_sql() @@ -2811,6 +2892,9 @@ This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("rebuild"); +.. note:: + In the CLI: :ref:`sqlite-utils rebuild-fts ` + .. _python_api_fts_optimize: Optimizing a full-text search table @@ -2826,6 +2910,9 @@ This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("optimize"); +.. note:: + In the CLI: :ref:`sqlite-utils optimize ` + .. _python_api_cached_table_counts: Cached table counts using triggers @@ -2888,6 +2975,9 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y db.reset_counts() +.. note:: + In the CLI: :ref:`sqlite-utils enable-counts ` + .. _python_api_create_index: Creating indexes @@ -2939,6 +3029,9 @@ You can drop an index from a table using ``.drop_index(index_name)``: Use ``ignore=True`` to ignore the error if the index does not exist. +.. note:: + In the CLI: :ref:`sqlite-utils create-index ` and :ref:`sqlite-utils drop-index ` + .. _python_api_analyze: Optimizing index usage with ANALYZE @@ -2966,6 +3059,9 @@ To run against all indexes attached to a specific table, you can either pass the db.table("dogs").analyze() +.. note:: + In the CLI: :ref:`sqlite-utils analyze ` + .. _python_api_vacuum: Vacuum @@ -2977,6 +3073,9 @@ You can optimize your database by running VACUUM against it like so: Database("my_database.db").vacuum() +.. note:: + In the CLI: :ref:`sqlite-utils vacuum ` + .. _python_api_wal: WAL mode @@ -3004,6 +3103,9 @@ You can check the current journal mode for a database using the ``journal_mode`` This will usually be ``wal`` or ``delete`` (meaning WAL is disabled), but can have other values - see the `PRAGMA journal_mode `__ documentation. +.. note:: + In the CLI: :ref:`sqlite-utils enable-wal and disable-wal ` + .. _python_api_suggest_column_types: Suggesting column types @@ -3144,6 +3246,9 @@ You can cause ``sqlite3`` to return more useful errors, including the traceback sqlite3.enable_callback_tracebacks(True) +.. note:: + In the CLI: :ref:`sqlite-utils query --functions ` + .. _python_api_quote: Quoting strings for use in SQL @@ -3276,6 +3381,9 @@ Initialize SpatiaLite .. automethod:: sqlite_utils.db.Database.init_spatialite :noindex: +.. note:: + In the CLI: :ref:`sqlite-utils create-database --init-spatialite ` + .. _python_api_gis_find_spatialite: Finding SpatiaLite @@ -3291,6 +3399,9 @@ Adding geometry columns .. automethod:: sqlite_utils.db.Table.add_geometry_column :noindex: +.. note:: + In the CLI: :ref:`sqlite-utils add-geometry-column ` + .. _python_api_gis_create_spatial_index: Creating a spatial index @@ -3298,3 +3409,6 @@ Creating a spatial index .. automethod:: sqlite_utils.db.Table.create_spatial_index :noindex: + +.. note:: + In the CLI: :ref:`sqlite-utils create-spatial-index ` From d71420065903ff54247b5062b8c6af6165b7e638 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Jul 2026 05:00:32 -0700 Subject: [PATCH 080/110] Add test: transform does not cascade-delete referencing records (#792) > Add a test that covers what happens if you run transform against a table with ON CASCADE DELETE for one of its foreign keys - those records should not be deleted during the transform even though the table is dropped as part of that procedure --- tests/test_transform.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_transform.py b/tests/test_transform.py index f0f5019..3cc5ba6 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -432,6 +432,43 @@ def test_transform_verify_foreign_keys(fresh_db): assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] +@pytest.mark.parametrize("use_pragma_foreign_keys", [False, True]) +def test_transform_on_delete_cascade_does_not_delete_records( + fresh_db, use_pragma_foreign_keys +): + # Transforming a table drops and recreates it - if another table references + # it with ON DELETE CASCADE and PRAGMA foreign_keys is on, that drop must + # not cascade and delete the referencing records + if use_pragma_foreign_keys: + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + # Transform the table on the other end of the cascading foreign key + fresh_db["authors"].transform(rename={"name": "author_name"}) + assert list(fresh_db["authors"].rows) == [ + {"id": 1, "author_name": "Ursula K. Le Guin"} + ] + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + # Transforming the table with the cascading foreign key should not + # delete its records either + fresh_db["books"].transform(rename={"title": "book_title"}) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "book_title": "The Dispossessed", "author_id": 1} + ] + if use_pragma_foreign_keys: + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + def test_transform_add_foreign_keys_from_scratch(fresh_db): _add_country_city_continent(fresh_db) fresh_db["places"].insert(_CAVEAU) From f66ddcb215e76dcbc1fa1dff4359f2ac3dc702e5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Jul 2026 08:43:51 -0700 Subject: [PATCH 081/110] Transform now refuses to run inside a transaction if destructive foreign keys exist (#795) * Transform now refuses to run inside a transaction if destructive foreign keys exist Closes #794 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014StVTWQJpFhfZJK2CYVBwv --- docs/python-api.rst | 33 ++++++++++- sqlite_utils/db.py | 35 ++++++++++++ tests/test_transform.py | 124 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 190 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 1ed238e..43b734d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -434,9 +434,10 @@ The library will never commit a transaction you opened. If you call write method Prefer ``db.atomic()`` or ``db.begin()``, ``db.commit()`` and ``db.rollback()`` over mixing sqlite-utils transaction methods with calls to ``db.conn.commit()``, ``db.conn.rollback()`` or raw transaction-control SQL. Mixing the two layers makes it much harder to tell which layer owns the current transaction. -Two related safeguards to be aware of: +Some related safeguards to be aware of: - ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, because changing the journal mode would commit it as a side effect. +- ``table.transform()`` raises a ``sqlite_utils.db.TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions, because the pragma cannot be turned off mid-transaction to protect those referencing rows - see :ref:`python_api_transform_foreign_keys_transactions`. - Closing the database - explicitly with ``db.close()``, or by exiting a ``with Database(...) as db:`` block - rolls back any transaction that is still open, see :ref:`python_api_close`. .. _python_api_transactions_modes: @@ -1996,6 +1997,36 @@ If you want to do something more advanced, you can call the ``table.transform_sq This method will return a list of SQL statements that should be executed to implement the change. You can then make modifications to that SQL - or add additional SQL statements - before executing it yourself. +.. _python_api_transform_foreign_keys_transactions: + +Foreign keys and transactions +----------------------------- + +Because ``.transform()`` drops the old table, running it with ``PRAGMA foreign_keys`` enabled could fire ``ON DELETE`` actions on any tables that reference it - an inbound ``ON DELETE CASCADE`` foreign key would silently delete those referencing rows. To prevent this, ``.transform()`` turns ``PRAGMA foreign_keys`` off for the duration of the operation and restores it afterwards, running ``PRAGMA foreign_key_check`` before committing. + +``PRAGMA foreign_keys`` cannot be changed inside a transaction, so this protection is impossible if you call ``.transform()`` while a transaction is already open - for example inside a ``with db.atomic():`` block or after ``db.begin()``. If ``PRAGMA foreign_keys`` is on and another table references the table being transformed with a destructive ``ON DELETE`` action - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT`` - the method will refuse to run and raise a ``sqlite_utils.db.TransactionError``: + +.. code-block:: python + + from sqlite_utils.db import TransactionError + + try: + with db.atomic(): + db["authors"].transform(types={"id": str}) + except TransactionError as ex: + print("Could not transform in transaction:", ex) + +To transform such a table either call ``.transform()`` outside of the transaction, or execute ``PRAGMA foreign_keys = off`` before opening it: + +.. code-block:: python + + db.execute("PRAGMA foreign_keys = off") + with db.atomic(): + db["authors"].transform(types={"id": str}) + db.execute("PRAGMA foreign_keys = on") + +Tables referenced by foreign keys without a destructive action (the default ``NO ACTION``, or ``RESTRICT``) can still be transformed inside a transaction - sqlite-utils uses ``PRAGMA defer_foreign_keys`` to postpone the foreign key checks until the transaction commits. + .. _python_api_extract: Extracting columns into a separate table diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d709fb9..e97b7d9 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2522,6 +2522,11 @@ class Table(Queryable): See :ref:`python_api_transform` for full details. + Raises :py:class:`sqlite_utils.db.TransactionError` if called while a + transaction is open with ``PRAGMA foreign_keys`` enabled and the table + is referenced by foreign keys with destructive ``ON DELETE`` actions - + see :ref:`python_api_transform_foreign_keys_transactions`. + :param types: Columns that should have their type changed, for example ``{"weight": float}`` :param rename: Columns to rename, for example ``{"headline": "title"}`` :param drop: Columns to drop @@ -2566,6 +2571,36 @@ class Table(Queryable): should_defer_foreign_keys = ( pragma_foreign_keys_was_on and already_in_transaction ) + if should_defer_foreign_keys: + # PRAGMA foreign_keys is a no-op inside a transaction, and + # defer_foreign_keys only defers violation checks, not ON DELETE + # actions - so dropping the old table would still fire destructive + # actions on any tables that reference it. Refuse rather than + # silently modify or delete those rows. + destructive_fks = [ + (table.name, fk) + for table in self.db.tables + for fk in table.foreign_keys + if fk.other_table == self.name + and fk.on_delete in ("CASCADE", "SET NULL", "SET DEFAULT") + ] + if destructive_fks: + raise TransactionError( + "Cannot transform table {table} while a transaction is open: " + "PRAGMA foreign_keys cannot be changed inside a transaction, " + "and the table is referenced by foreign keys with ON DELETE " + "actions that would fire when the old table is dropped: " + "{fks}. Call transform() outside of the transaction, or " + 'execute "PRAGMA foreign_keys = off" before opening it.'.format( + table=self.name, + fks=", ".join( + "{}.{} (ON DELETE {})".format( + table_name, ", ".join(fk.columns), fk.on_delete + ) + for table_name, fk in destructive_fks + ), + ) + ) defer_foreign_keys_was_on = False try: if should_disable_foreign_keys: diff --git a/tests/test_transform.py b/tests/test_transform.py index 3cc5ba6..362f1ca 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,6 +1,6 @@ import sqlite3 -from sqlite_utils.db import ForeignKey, TransformError +from sqlite_utils.db import ForeignKey, TransactionError, TransformError from sqlite_utils.utils import OperationalError import pytest @@ -469,6 +469,128 @@ def test_transform_on_delete_cascade_does_not_delete_records( assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] +@pytest.mark.parametrize("on_delete", ["CASCADE", "SET NULL", "SET DEFAULT", "cascade"]) +def test_transform_in_transaction_refuses_destructive_on_delete(fresh_db, on_delete): + # PRAGMA foreign_keys is a no-op inside a transaction, so transforming a + # table referenced by ON DELETE CASCADE / SET NULL / SET DEFAULT foreign + # keys inside an open transaction would fire those actions when the old + # table is dropped - transform() should refuse instead + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE {} + ); + """.format(on_delete)) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + previous_schema = fresh_db["authors"].schema + with fresh_db.atomic(): + with pytest.raises(TransactionError) as excinfo: + fresh_db["authors"].transform(rename={"name": "author_name"}) + message = str(excinfo.value) + assert "books" in message + assert "ON DELETE {}".format(on_delete.upper()) in message + # Nothing should have changed + assert fresh_db["authors"].schema == previous_schema + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + +def test_transform_in_transaction_refuses_self_referential_cascade(fresh_db): + # The copied table carries a foreign key referencing the original table + # name, so a self-referential cascade would wipe the copy too + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE categories ( + id INTEGER PRIMARY KEY, + name TEXT, + parent_id INTEGER REFERENCES categories(id) ON DELETE CASCADE + ); + """) + fresh_db["categories"].insert_all( + [ + {"id": 1, "name": "Fiction", "parent_id": None}, + {"id": 2, "name": "Science Fiction", "parent_id": 1}, + ] + ) + with fresh_db.atomic(): + with pytest.raises(TransactionError) as excinfo: + fresh_db["categories"].transform(rename={"name": "title"}) + assert "categories" in str(excinfo.value) + assert fresh_db["categories"].count == 2 + + +def test_transform_in_transaction_allowed_with_no_action_foreign_key(fresh_db): + # An inbound foreign key without a destructive ON DELETE action is safe + # inside a transaction thanks to PRAGMA defer_foreign_keys + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + with fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "author_name"}) + assert list(fresh_db["authors"].rows) == [ + {"id": 1, "author_name": "Ursula K. Le Guin"} + ] + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + +def test_transform_in_transaction_allowed_for_child_table(fresh_db): + # The table being transformed only has an outbound foreign key - dropping + # it fires no ON DELETE actions, so this is allowed inside a transaction + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + with fresh_db.atomic(): + fresh_db["books"].transform(rename={"title": "book_title"}) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "book_title": "The Dispossessed", "author_id": 1} + ] + + +def test_transform_in_transaction_allowed_with_foreign_keys_off(fresh_db): + # With PRAGMA foreign_keys off (the default) no cascades can fire, so + # transform inside a transaction is safe even with a CASCADE schema + fresh_db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + with fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "author_name"}) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ] + + def test_transform_add_foreign_keys_from_scratch(fresh_db): _add_country_city_continent(fresh_db) fresh_db["places"].insert(_CAVEAU) From 458b3ab5b169eff1f8319c44a7c320c68f54d28b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Jul 2026 13:52:14 -0700 Subject: [PATCH 082/110] Release 4.1.1 Refs #791, #792, #794, #795 --- docs/changelog.rst | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5b9355f..a8006c3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _v4_1_1: + +4.1.1 (2026-07-12) +------------------ + +- ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`) +- The CLI and Python API documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) .. _v4_1: 4.1 (2026-07-11) diff --git a/pyproject.toml b/pyproject.toml index 971f5a0..003322c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.1" +version = "4.1.1" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From a947dc673923ff6e95b41d3dfabe1cbd95e6de86 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 12 Jul 2026 17:14:01 -0700 Subject: [PATCH 083/110] Changelog now links to CLI and Python API in most recent entry --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a8006c3..4c868f4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ ------------------ - ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`) -- The CLI and Python API documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) +- The :ref:`CLI ` and :ref:`Python API ` documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) .. _v4_1: 4.1 (2026-07-11) From 69a1c0d960abb20ac03a085142bd59f7fbe002f7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 14:53:12 -0700 Subject: [PATCH 084/110] Fixes for Ruff>=0.16.0 (#814) * Automated upgrades by Ruff uvx --with 'ruff>=0.16.0' ruff check . --fix --unsafe-fixes * Fix remaining Ruff errors with GPT-5.6 Sol high https://gist.github.com/simonw/6da7906a9fea6e90da131c21a9055199 * Fix flake E501 long lines * New Protocol for migrations to make ty happy --- .gitignore | 1 + docs/cli-reference.rst | 4 +- docs/conf.py | 11 +- pyproject.toml | 9 +- sqlite_utils/__init__.py | 7 +- sqlite_utils/cli.py | 199 +++--- sqlite_utils/db.py | 988 +++++++++++++---------------- sqlite_utils/hookspecs.py | 3 +- sqlite_utils/migrations.py | 27 +- sqlite_utils/plugins.py | 10 +- sqlite_utils/recipes.py | 12 +- sqlite_utils/utils.py | 111 ++-- tests/conftest.py | 5 +- tests/test_analyze_tables.py | 10 +- tests/test_atomic.py | 81 ++- tests/test_cli.py | 88 +-- tests/test_cli_bulk.py | 8 +- tests/test_cli_convert.py | 16 +- tests/test_cli_insert.py | 17 +- tests/test_cli_memory.py | 13 +- tests/test_cli_migrate.py | 3 +- tests/test_column_affinity.py | 3 +- tests/test_constructor.py | 6 +- tests/test_convert.py | 3 +- tests/test_create.py | 68 +- tests/test_create_view.py | 1 + tests/test_default_value.py | 2 +- tests/test_delete.py | 2 +- tests/test_docs.py | 12 +- tests/test_duplicate.py | 6 +- tests/test_enable_counts.py | 10 +- tests/test_extract.py | 15 +- tests/test_extracts.py | 19 +- tests/test_foreign_keys.py | 3 +- tests/test_fts.py | 26 +- tests/test_get.py | 1 + tests/test_gis.py | 5 +- tests/test_hypothesis.py | 3 +- tests/test_insert_files.py | 14 +- tests/test_introspect.py | 7 +- tests/test_list_mode.py | 1 + tests/test_lookup.py | 3 +- tests/test_m2m.py | 6 +- tests/test_migrations.py | 8 +- tests/test_plugins.py | 13 +- tests/test_query.py | 3 +- tests/test_recipes.py | 6 +- tests/test_recreate.py | 6 +- tests/test_rows_from_file.py | 6 +- tests/test_sniff.py | 6 +- tests/test_suggest_column_types.py | 4 +- tests/test_tracer.py | 54 +- tests/test_transform.py | 33 +- tests/test_update.py | 2 +- tests/test_upsert.py | 5 +- tests/test_utils.py | 6 +- tests/test_wal.py | 32 +- 57 files changed, 974 insertions(+), 1049 deletions(-) diff --git a/.gitignore b/.gitignore index 6743708..5b5d2c6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ venv .schema .vscode .hypothesis +.claude/ Pipfile Pipfile.lock uv.lock diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 9fafe28..a4ec402 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -662,7 +662,7 @@ See :ref:`cli_convert`. Convert a string like a,b,c into a JSON array ["a", "b", "c"] r.parsedate(value: 'str', dayfirst: 'bool' = False, yearfirst: 'bool' = False, - errors: 'Optional[object]' = None) -> 'Optional[str]' + errors: 'object | None' = None) -> 'str | None' Parse a date and convert it to ISO date format: yyyy-mm-dd - dayfirst=True: treat xx as the day in xx/yy/zz @@ -671,7 +671,7 @@ See :ref:`cli_convert`. - errors=r.SET_NULL to set values that cannot be parsed to null r.parsedatetime(value: 'str', dayfirst: 'bool' = False, yearfirst: 'bool' = - False, errors: 'Optional[object]' = None) -> 'Optional[str]' + False, errors: 'object | None' = None) -> 'str | None' Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS - dayfirst=True: treat xx as the day in xx/yy/zz diff --git a/docs/conf.py b/docs/conf.py index 4f29b39..62d4642 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,10 +1,7 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - import inspect -from pathlib import Path -from subprocess import Popen, PIPE, check_output import sys +from pathlib import Path +from subprocess import PIPE, CalledProcessError, Popen, check_output # This file is execfile()d with the current directory set to its # containing dir. @@ -50,7 +47,7 @@ extlinks = { def _linkcode_git_ref(): try: return check_output(["git", "rev-parse", "HEAD"]).decode("utf8").strip() - except Exception: + except (CalledProcessError, OSError): return "main" @@ -79,7 +76,7 @@ def linkcode_resolve(domain, info): obj = inspect.unwrap(obj) source_file = inspect.getsourcefile(obj) _, line_number = inspect.getsourcelines(obj) - except Exception: + except (OSError, TypeError, ValueError): return None if source_file is None: diff --git a/pyproject.toml b/pyproject.toml index 003322c..6bc0a64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,14 @@ build-backend = "setuptools.build_meta" max-line-length = 160 # Black compatibility, E203 whitespace before ':': extend-ignore = ["E203"] -extend-exclude = [".venv", "build", "dist", "docs", "sqlite_utils.egg-info"] +extend-exclude = [ + ".venv", + ".claude", + "build", + "dist", + "docs", + "sqlite_utils.egg-info", +] [tool.setuptools.package-data] sqlite_utils = ["py.typed"] diff --git a/sqlite_utils/__init__.py b/sqlite_utils/__init__.py index 58ee7ab..0d25716 100644 --- a/sqlite_utils/__init__.py +++ b/sqlite_utils/__init__.py @@ -1,7 +1,6 @@ -from .utils import suggest_column_types -from .hookspecs import hookimpl -from .hookspecs import hookspec from .db import Database +from .hookspecs import hookimpl, hookspec from .migrations import Migrations +from .utils import suggest_column_types -__all__ = ["Database", "Migrations", "suggest_column_types", "hookimpl", "hookspec"] +__all__ = ["Database", "Migrations", "hookimpl", "hookspec", "suggest_column_types"] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index e0b8969..dab4b67 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,17 +1,30 @@ import base64 +import csv as csv_std import difflib -from typing import Any -import click -from click_default_group import DefaultGroup -from datetime import datetime, timezone import hashlib +import inspect +import io +import itertools +import json +import os import pathlib +import pdb # noqa: T100 +import sys +import textwrap +from datetime import datetime, timezone from runpy import run_module +from typing import Any + +import click +import tabulate +from click_default_group import DefaultGroup + import sqlite_utils +from sqlite_utils import recipes from sqlite_utils.db import ( + DEFAULT, AlterError, BadMultiValues, - DEFAULT, DescIndex, InvalidColumns, NoTable, @@ -19,36 +32,28 @@ from sqlite_utils.db import ( PrimaryKeyRequired, quote_identifier, ) -from sqlite_utils.plugins import ensure_plugins_loaded, pm, get_plugins +from sqlite_utils.plugins import ensure_plugins_loaded, get_plugins, pm from sqlite_utils.utils import maximize_csv_field_size_limit -from sqlite_utils import recipes -import textwrap -import inspect -import io -import itertools -import json -import os -import pdb -import sys -import csv as csv_std -import tabulate + from .utils import ( + Format, OperationalError, + TypeTracker, _compile_code, chunks, + decode_base64_values, dedupe_keys, file_progress, find_spatialite, - flatten as _flatten, - sqlite3, - decode_base64_values, progressbar, rows_from_file, - Format, - TypeTracker, + sqlite3, +) +from .utils import ( + flatten as _flatten, ) -CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) +CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]} def _register_db_for_cleanup(db): @@ -67,7 +72,7 @@ def _close_databases(ctx): for db in ctx.meta.get("_databases_to_close", []): try: db.close() - except Exception: + except sqlite3.Error: pass @@ -174,7 +179,6 @@ def functions_option(fn): @click.version_option() def cli(): "Commands for interacting with a SQLite database" - pass @cli.command() @@ -891,7 +895,7 @@ def enable_counts(path, tables, load_extension): # Check all tables exist bad_tables = [table for table in tables if not db[table].exists()] if bad_tables: - raise click.ClickException("Invalid tables: {}".format(bad_tables)) + raise click.ClickException(f"Invalid tables: {bad_tables}") for table in tables: db.table(table).enable_counts() @@ -1140,9 +1144,7 @@ def insert_upsert_implementation( ) ): raise click.ClickException( - "{}\n\nTry using --alter to add additional columns".format( - e.args[0] - ) + f"{e.args[0]}\n\nTry using --alter to add additional columns" ) # If we can find sql= and parameters= arguments, show those variables = _find_variables(e.__traceback__, ["sql", "parameters"]) @@ -1240,7 +1242,7 @@ def insert_upsert_implementation( reader = csv_std.reader(decoded, **csv_reader_args) # type: ignore first_row = next(reader) if no_headers: - headers = ["untitled_{}".format(i + 1) for i in range(len(first_row))] + headers = [f"untitled_{i + 1}" for i in range(len(first_row))] reader = itertools.chain([first_row], reader) else: headers = first_row @@ -1269,9 +1271,7 @@ def insert_upsert_implementation( docs = [docs] except json.decoder.JSONDecodeError as ex: raise click.ClickException( - "Invalid JSON - use --csv for CSV or --tsv for TSV files\n\nJSON error: {}".format( - ex - ) + f"Invalid JSON - use --csv for CSV or --tsv for TSV files\n\nJSON error: {ex}" ) if flatten: docs = (_flatten(doc) for doc in docs) @@ -1290,7 +1290,7 @@ def insert_upsert_implementation( docs = (fn(doc["line"]) for doc in docs) elif text: # Special case: this is allowed to be an iterable - text_value = list(docs)[0]["text"] + text_value = next(iter(docs))["text"] fn_return = fn(text_value) if isinstance(fn_return, dict): docs = [fn_return] @@ -1774,17 +1774,14 @@ def create_table( ctype = columns.pop(0) if ctype.upper() not in VALID_COLUMN_TYPES: raise click.ClickException( - "column types must be one of {}".format(VALID_COLUMN_TYPES) + f"column types must be one of {VALID_COLUMN_TYPES}" ) coltypes[name] = ctype.upper() # Does table already exist? - if table in db.table_names(): - if not ignore and not replace and not transform: - raise click.ClickException( - 'Table "{}" already exists. Use --replace to delete and replace it.'.format( - table - ) - ) + if table in db.table_names() and not ignore and not replace and not transform: + raise click.ClickException( + f'Table "{table}" already exists. Use --replace to delete and replace it.' + ) db.table(table).create( coltypes, pk=pks[0] if len(pks) == 1 else pks, @@ -1819,7 +1816,7 @@ def duplicate(path, table, new_table, ignore, load_extension): db.table(table).duplicate(new_table) except NoTable: if not ignore: - raise click.ClickException('Table "{}" does not exist'.format(table)) + raise click.ClickException(f'Table "{table}" does not exist') @cli.command(name="rename-table") @@ -1843,9 +1840,7 @@ def rename_table(path, table, new_name, ignore, load_extension): db.rename_table(table, new_name) except sqlite3.OperationalError as ex: if not ignore: - raise click.ClickException( - 'Table "{}" could not be renamed. {}'.format(table, str(ex)) - ) + raise click.ClickException(f'Table "{table}" could not be renamed. {ex!s}') @cli.command(name="drop-table") @@ -1874,10 +1869,10 @@ def drop_table(path, table, ignore, load_extension): # A view exists with this name if not ignore: raise click.ClickException( - '"{}" is a view, not a table - use drop-view to drop it'.format(table) + f'"{table}" is a view, not a table - use drop-view to drop it' ) except OperationalError: - raise click.ClickException('Table "{}" does not exist'.format(table)) + raise click.ClickException(f'Table "{table}" does not exist') @cli.command(name="create-view") @@ -1919,9 +1914,7 @@ def create_view(path, view, select, ignore, replace, load_extension): db.view(view).drop() else: raise click.ClickException( - 'View "{}" already exists. Use --replace to delete and replace it.'.format( - view - ) + f'View "{view}" already exists. Use --replace to delete and replace it.' ) db.create_view(view, select) @@ -1953,9 +1946,9 @@ def drop_view(path, view, ignore, load_extension): return if view in db.table_names(): raise click.ClickException( - '"{}" is a table, not a view - use drop-table to drop it'.format(view) + f'"{view}" is a table, not a view - use drop-table to drop it' ) - raise click.ClickException('View "{}" does not exist'.format(view)) + raise click.ClickException(f'View "{view}" does not exist') @cli.command() @@ -2177,7 +2170,7 @@ def memory( file_path = pathlib.Path(path) stem = file_path.stem if stem_counts.get(stem): - file_table = "{}_{}".format(stem, stem_counts[stem]) + file_table = f"{stem}_{stem_counts[stem]}" else: file_table = stem stem_counts[stem] = stem_counts.get(stem, 1) + 1 @@ -2196,14 +2189,14 @@ def memory( if tracker is not None and db.table(file_table).exists(): db.table(file_table).transform(types=tracker.types) # Add convenient t / t1 / t2 views - view_names = ["t{}".format(i + 1)] + view_names = [f"t{i + 1}"] if i == 0: view_names.append("t") for view_name in view_names: if not db[view_name].exists(): db.create_view( view_name, - "select * from {}".format(quote_identifier(file_table)), + f"select * from {quote_identifier(file_table)}", ) finally: if should_close_fp and fp: @@ -2373,19 +2366,17 @@ def search( # Check table exists table_obj = db.table(dbtable) if not table_obj.exists(): - raise click.ClickException("Table '{}' does not exist".format(dbtable)) + raise click.ClickException(f"Table '{dbtable}' does not exist") if not table_obj.detect_fts(): raise click.ClickException( - "Table '{}' is not configured for full-text search".format(dbtable) + f"Table '{dbtable}' is not configured for full-text search" ) if column: # Check they all exist table_columns = table_obj.columns_dict for c in column: if c not in table_columns: - raise click.ClickException( - "Table '{}' has no column '{}".format(dbtable, c) - ) + raise click.ClickException(f"Table '{dbtable}' has no column '{c}") sql = table_obj.search_sql(columns=column, order_by=order, limit=limit) if show_sql: click.echo(sql) @@ -2412,7 +2403,7 @@ def search( except click.ClickException as e: if "malformed MATCH expression" in str(e) or "unterminated string" in str(e): raise click.ClickException( - "{}\n\nTry running this again with the --quote option".format(str(e)) + f"{e!s}\n\nTry running this again with the --quote option" ) else: raise @@ -2479,15 +2470,15 @@ def rows( columns = "*" if column: columns = ", ".join(quote_identifier(c) for c in column) - sql = "select {} from {}".format(columns, quote_identifier(dbtable)) + sql = f"select {columns} from {quote_identifier(dbtable)}" if where: sql += " where " + where if order: sql += " order by " + order if limit: - sql += " limit {}".format(limit) + sql += f" limit {limit}" if offset: - sql += " offset {}".format(offset) + sql += f" offset {offset}" ctx.invoke( query, path=path, @@ -2760,7 +2751,7 @@ def transform( for column, ctype in type: if ctype.upper() not in VALID_COLUMN_TYPES: raise click.ClickException( - "column types must be one of {}".format(VALID_COLUMN_TYPES) + f"column types must be one of {VALID_COLUMN_TYPES}" ) types[column] = ctype.upper() @@ -2858,12 +2849,12 @@ def extract( db = sqlite_utils.Database(path) _register_db_for_cleanup(db) _load_extensions(db, load_extension) - kwargs: dict[str, Any] = dict( - columns=columns, - table=other_table, - fk_column=fk_column, - rename=dict(rename), - ) + kwargs: dict[str, Any] = { + "columns": columns, + "table": other_table, + "fk_column": fk_column, + "rename": dict(rename), + } try: db.table(table).extract(**kwargs) except (NoTable, InvalidColumns) as e: @@ -2958,7 +2949,7 @@ def insert_files( with progressbar(paths_and_relative_paths, silent=silent) as bar: def to_insert(): - for path, relative_path in bar: + for file_path, relative_path in bar: row = {} # content_text is special case as it considers 'encoding' @@ -2970,19 +2961,21 @@ def insert_files( raise UnicodeDecodeErrorForPath(e, resolved) lookups = dict(FILE_COLUMNS, content_text=_content_text) - if path == "-": + if file_path == "-": stdin_data = sys.stdin.buffer.read() # We only support a subset of columns for this case lookups = { "name": lambda p: name or "-", "path": lambda p: name or "-", - "content": lambda p: stdin_data, - "content_text": lambda p: stdin_data.decode( + "content": lambda p, data=stdin_data: data, + "content_text": lambda p, data=stdin_data: data.decode( encoding or "utf-8" ), - "sha256": lambda p: hashlib.sha256(stdin_data).hexdigest(), - "md5": lambda p: hashlib.md5(stdin_data).hexdigest(), - "size": lambda p: len(stdin_data), + "sha256": lambda p, data=stdin_data: hashlib.sha256( + data + ).hexdigest(), + "md5": lambda p, data=stdin_data: hashlib.md5(data).hexdigest(), + "size": lambda p, data=stdin_data: len(data), } for coldef in column: if ":" in coldef: @@ -2990,7 +2983,7 @@ def insert_files( else: colname, coltype = coldef, coldef try: - value = lookups[coltype](path) + value = lookups[coltype](file_path) row[colname] = value except KeyError: raise click.ClickException( @@ -3018,7 +3011,7 @@ def insert_files( except UnicodeDecodeErrorForPath as e: raise click.ClickException( UNICODE_ERROR.format( - "Could not read file '{}' as text\n\n{}".format(e.path, e.exception) + f"Could not read file '{e.path}' as text\n\n{e.exception}" ) ) @@ -3196,7 +3189,7 @@ def _generate_convert_help(): for name in recipe_names: fn = getattr(recipes, name) doc = textwrap.dedent(fn.__doc__.rstrip()).replace("\b\n", "") - help += "\n\nr.{}{}\n\n\b{}".format(name, str(inspect.signature(fn)), doc) + help += f"\n\nr.{name}{inspect.signature(fn)!s}\n\n\b{doc}" help += "\n\n" help += textwrap.dedent(""" You can use these recipes like so: @@ -3299,7 +3292,7 @@ def convert( """.format( column=columns[0], table=table, - where=" where {}".format(where) if where is not None else "", + where=f" where {where}" if where is not None else "", ) for row in db.conn.execute(sql, where_args).fetchall(): click.echo(str(row[0])) @@ -3319,7 +3312,7 @@ def convert( def wrapped_fn(value): try: return fn_(value) - except Exception as ex: + except Exception as ex: # noqa: BLE001 print("\nException raised, dropping into pdb...:", ex) pdb.post_mortem(ex.__traceback__) sys.exit(1) @@ -3339,9 +3332,7 @@ def convert( ) except BadMultiValues as e: raise click.ClickException( - "When using --multi code must return a Python dictionary - returned: {}".format( - repr(e.values) - ) + f"When using --multi code must return a Python dictionary - returned: {e.values!r}" ) @@ -3459,7 +3450,7 @@ def create_spatial_index(db_path, table, column_name, load_extension): def _find_migration_files(migrations): if not migrations: - migrations = [pathlib.Path(".").resolve()] + migrations = [pathlib.Path.cwd()] files = set() for path_str in migrations: path = pathlib.Path(path_str) @@ -3484,7 +3475,7 @@ def _load_migration_sets(files): "__file__": str(filepath), "__name__": "__sqlite_utils_migration__", } - exec(code, namespace) + exec(code, namespace) # noqa: S102 migration_sets.extend( obj for obj in namespace.values() if _compatible_migration_set(obj) ) @@ -3493,17 +3484,17 @@ def _load_migration_sets(files): def _display_migration_list(db, migration_sets): for migration_set in migration_sets: - click.echo("Migrations for: {}".format(migration_set.name)) + click.echo(f"Migrations for: {migration_set.name}") click.echo() click.echo(" Applied:") for migration in migration_set.applied(db): - click.echo(" {} - {}".format(migration.name, migration.applied_at)) + click.echo(f" {migration.name} - {migration.applied_at}") click.echo() click.echo(" Pending:") output = False for migration in migration_set.pending(db): output = True - click.echo(" {}".format(migration.name)) + click.echo(f" {migration.name}") if not output: click.echo(" (none)") click.echo() @@ -3583,7 +3574,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose): prev_schema = db.schema if verbose: - click.echo("Migrating {}".format(db_path)) + click.echo(f"Migrating {db_path}") click.echo("\nSchema before:\n") click.echo(textwrap.indent(prev_schema, " ") or " (empty)") click.echo() @@ -3594,9 +3585,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose): names = {m.name for m in migration_set.pending(db)} names.update(m.name for m in migration_set.applied(db)) known_names.update(names) - known_names.update( - "{}:{}".format(migration_set.name, name) for name in names - ) + known_names.update(f"{migration_set.name}:{name}" for name in names) unknown = [value for value in stop_before if value not in known_names] if unknown: raise click.ClickException( @@ -3652,7 +3641,7 @@ def _render_common(title, values): return "" lines = [title] for value, count in values: - lines.append(" {}: {}".format(count, value)) + lines.append(f" {count}: {value}") return "\n".join(lines) @@ -3722,7 +3711,7 @@ def maybe_json(value): if not isinstance(value, str): return value stripped = value.strip() - if not (stripped.startswith("{") or stripped.startswith("[")): + if not (stripped.startswith(("{", "["))): return value try: return json.loads(stripped) @@ -3740,7 +3729,7 @@ def json_binary(value): def verify_is_dict(doc): if not isinstance(doc, dict): raise click.ClickException( - "Rows must all be dictionaries, got: {}".format(repr(doc)[:1000]) + f"Rows must all be dictionaries, got: {repr(doc)[:1000]}" ) return doc @@ -3768,14 +3757,14 @@ def _register_functions(db, functions): try: functions = pathlib.Path(functions).read_text() except FileNotFoundError: - raise click.ClickException("File not found: {}".format(functions)) + raise click.ClickException(f"File not found: {functions}") sqlite3.enable_callback_tracebacks(True) globals = {} try: - exec(functions, globals) + exec(functions, globals) # noqa: S102 except SyntaxError as ex: - raise click.ClickException("Error in functions definition: {}".format(ex)) + raise click.ClickException(f"Error in functions definition: {ex}") # Register all callables in the locals dict: for name, value in globals.items(): if callable(value) and not name.startswith("_"): @@ -3796,12 +3785,12 @@ def _rows_from_code(code): try: code = pathlib.Path(code).read_text() except FileNotFoundError: - raise click.ClickException("File not found: {}".format(code)) + raise click.ClickException(f"File not found: {code}") namespace = {} try: - exec(code, namespace) + exec(code, namespace) # noqa: S102 except SyntaxError as ex: - raise click.ClickException("Error in --code: {}".format(ex)) + raise click.ClickException(f"Error in --code: {ex}") rows = namespace.get("rows") if callable(rows): rows = rows() diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e97b7d9..9a00123 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,19 +1,4 @@ -from .utils import ( - chunks, - dedupe_keys, - hash_record, - sqlite3, - OperationalError, - suggest_column_types, - types_for_column_types, - column_affinity, - progressbar, - find_spatialite, -) import binascii -from collections import namedtuple -from dataclasses import dataclass, field -from collections.abc import Mapping import contextlib import datetime import decimal @@ -25,26 +10,36 @@ import os import pathlib import re import secrets -from sqlite_fts4 import rank_bm25 import textwrap -from typing import ( - cast, - Any, - Callable, - Dict, - Generator, - Iterable, - Sequence, - Set, - Type, - Union, - Optional, - List, - Tuple, -) import uuid +from collections import namedtuple +from collections.abc import Callable, Generator, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from types import TracebackType +from typing import ( + Any, + Union, + cast, +) + +from sqlite_fts4 import rank_bm25 +from typing_extensions import Self + from sqlite_utils.plugins import ensure_plugins_loaded, pm +from .utils import ( + OperationalError, + chunks, + column_affinity, + dedupe_keys, + find_spatialite, + hash_record, + progressbar, + sqlite3, + suggest_column_types, + types_for_column_types, +) + try: iterdump = importlib.import_module("sqlite_dump").iterdump except ImportError: @@ -226,11 +221,11 @@ class ForeignKey: table: str # column/other_column are None for compound keys, which would break # ordering against str values - comparison uses columns/other_columns - column: Optional[str] = field(compare=False) + column: str | None = field(compare=False) other_table: str - other_column: Optional[str] = field(compare=False) - columns: Tuple[str, ...] = () - other_columns: Tuple[str, ...] = () + other_column: str | None = field(compare=False) + columns: tuple[str, ...] = () + other_columns: tuple[str, ...] = () is_compound: bool = False on_delete: str = "NO ACTION" on_update: str = "NO ACTION" @@ -259,9 +254,9 @@ def _fk_actions_sql(fk: ForeignKey) -> str: "ON UPDATE/ON DELETE clauses for a foreign key, or an empty string." actions = "" if fk.on_update and fk.on_update != "NO ACTION": - actions += " ON UPDATE {}".format(fk.on_update) + actions += f" ON UPDATE {fk.on_update}" if fk.on_delete and fk.on_delete != "NO ACTION": - actions += " ON DELETE {}".format(fk.on_delete) + actions += f" ON DELETE {fk.on_delete}" return actions @@ -278,20 +273,20 @@ class TransformError(Exception): # A single column name, or a tuple of columns for a compound foreign key -ForeignKeyColumns = Union[str, Tuple[str, ...], List[str]] +ForeignKeyColumns = str | tuple[str, ...] | list[str] # (table, column(s), other_table, other_column(s)) -ForeignKeyTuple = Tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] +ForeignKeyTuple = tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] -ForeignKeyIndicator = Union[ - str, - ForeignKey, - Tuple[ForeignKeyColumns, str], - Tuple[ForeignKeyColumns, str, ForeignKeyColumns], - ForeignKeyTuple, -] +ForeignKeyIndicator = ( + str + | ForeignKey + | tuple[ForeignKeyColumns, str] + | tuple[ForeignKeyColumns, str, ForeignKeyColumns] + | ForeignKeyTuple +) -ForeignKeysType = Union[Iterable[ForeignKeyIndicator], List[ForeignKeyIndicator]] +ForeignKeysType = Iterable[ForeignKeyIndicator] | list[ForeignKeyIndicator] class Default: @@ -300,7 +295,7 @@ class Default: DEFAULT = Default() -Tracer = Callable[[str, Optional[Union[Sequence[Any], Dict[str, Any]]]], None] +Tracer = Callable[[str, Sequence[Any] | dict[str, Any] | None], None] def _iter_complete_sql_statements(sql: str) -> Generator[str, None, None]: @@ -316,7 +311,7 @@ def _iter_complete_sql_statements(sql: str) -> Generator[str, None, None]: yield statement_sql -COLUMN_TYPE_MAPPING: Dict[Any, str] = { +COLUMN_TYPE_MAPPING: dict[Any, str] = { float: "REAL", int: "INTEGER", bool: "INTEGER", @@ -512,12 +507,12 @@ class Database: def __init__( self, - filename_or_conn: Optional[Union[str, pathlib.Path, sqlite3.Connection]] = None, + filename_or_conn: str | pathlib.Path | sqlite3.Connection | None = None, memory: bool = False, - memory_name: Optional[str] = None, + memory_name: str | None = None, recreate: bool = False, recursive_triggers: bool = True, - tracer: Optional[Tracer] = None, + tracer: Tracer | None = None, use_counts_table: bool = False, execute_plugins: bool = True, use_old_upsert: bool = False, @@ -532,7 +527,7 @@ class Database: ): raise ValueError("Either specify a filename_or_conn or pass memory=True") if memory_name: - uri = "file:{}?mode=memory&cache=shared".format(memory_name) + uri = f"file:{memory_name}?mode=memory&cache=shared" self.conn = sqlite3.connect( uri, uri=True, @@ -569,7 +564,7 @@ class Database: "transaction handling - connections created with " "autocommit=True or autocommit=False are not supported" ) - self._tracer: Optional[Tracer] = tracer + self._tracer: Tracer | None = tracer if recursive_triggers: self.execute("PRAGMA recursive_triggers=on;") self._registered_functions: set = set() @@ -579,14 +574,14 @@ class Database: pm.hook.prepare_connection(conn=self.conn) self.strict = strict - def __enter__(self) -> "Database": + def __enter__(self) -> Self: return self def __exit__( self, - exc_type: Optional[Type[BaseException]], - exc_val: Optional[BaseException], - exc_tb: Optional[object], + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, ) -> None: self.close() @@ -602,8 +597,8 @@ class Database: Nested blocks use SQLite savepoints. """ if self.conn.in_transaction: - savepoint = "sqlite_utils_{}".format(secrets.token_hex(16)) - self.conn.execute("SAVEPOINT {};".format(savepoint)) + savepoint = f"sqlite_utils_{secrets.token_hex(16)}" + self.conn.execute(f"SAVEPOINT {savepoint};") try: yield self except BaseException: @@ -612,11 +607,11 @@ class Database: # anyway would mask the original exception with # "no such savepoint" if self.conn.in_transaction: - self.conn.execute("ROLLBACK TO SAVEPOINT {};".format(savepoint)) - self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) + self.conn.execute(f"ROLLBACK TO SAVEPOINT {savepoint};") + self.conn.execute(f"RELEASE SAVEPOINT {savepoint};") raise else: - self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) + self.conn.execute(f"RELEASE SAVEPOINT {savepoint};") else: self.conn.execute("BEGIN") try: @@ -695,9 +690,7 @@ class Database: self.conn.isolation_level = old_isolation_level @contextlib.contextmanager - def tracer( - self, tracer: Optional[Tracer] = None - ) -> Generator["Database", None, None]: + def tracer(self, tracer: Tracer | None = None) -> Generator["Database", None, None]: """ Context manager to temporarily set a tracer function - all executed SQL queries will be passed to this. @@ -734,15 +727,15 @@ class Database: return self.table(table_name) def __repr__(self) -> str: - return "".format(self.conn) + return f"" def register_function( self, - fn: Optional[Callable] = None, + fn: Callable | None = None, deterministic: bool = False, replace: bool = False, - name: Optional[str] = None, - ) -> Optional[Callable[[Callable], Callable]]: + name: str | None = None, + ) -> Callable[[Callable], Callable] | None: """ ``fn`` will be made available as a function within SQL, with the same name and number of arguments. Can be used as a decorator:: @@ -770,7 +763,7 @@ class Database: arity = len(inspect.signature(fn).parameters) if not replace and (fn_name, arity) in self._registered_functions: return fn - kwargs: Dict[str, bool] = {} + kwargs: dict[str, bool] = {} registered = False if deterministic: # Try this, but fall back if sqlite3.NotSupportedError @@ -796,7 +789,7 @@ class Database: "Register the ``rank_bm25(match_info)`` function used for calculating relevance with SQLite FTS4." self.register_function(rank_bm25, deterministic=True, replace=True) - def attach(self, alias: str, filepath: Union[str, pathlib.Path]) -> None: + def attach(self, alias: str, filepath: str | pathlib.Path) -> None: """ Attach another SQLite database file to this connection with the specified alias, equivalent to:: @@ -805,15 +798,13 @@ class Database: :param alias: Alias name to use :param filepath: Path to SQLite database file on disk """ - attach_sql = """ - ATTACH DATABASE '{}' AS {}; - """.format( - str(pathlib.Path(filepath).resolve()), quote_identifier(alias) - ).strip() + attach_sql = f""" + ATTACH DATABASE '{pathlib.Path(filepath).resolve()!s}' AS {quote_identifier(alias)}; + """.strip() self.execute(attach_sql) def query( - self, sql: str, params: Optional[Union[Sequence, Dict[str, Any]]] = None + self, sql: str, params: Sequence | dict[str, Any] | None = None ) -> Generator[dict, None, None]: """ Execute ``sql`` and return an iterable of dictionaries representing each row. @@ -891,7 +882,7 @@ class Database: self.conn.execute('RELEASE "sqlite_utils_query"') def execute( - self, sql: str, parameters: Optional[Union[Sequence, Dict[str, Any]]] = None + self, sql: str, parameters: Sequence | dict[str, Any] | None = None ) -> sqlite3.Cursor: """ Execute SQL query and return a ``sqlite3.Cursor``. @@ -960,7 +951,7 @@ class Database: :param table_name: Name of the table """ if table_name in self.view_names(): - raise NoTable("Table {} is actually a view".format(table_name)) + raise NoTable(f"Table {table_name} is actually a view") kwargs.setdefault("strict", self.strict) return Table(self, table_name, **kwargs) @@ -973,11 +964,9 @@ class Database: if view_name not in self.view_names(): if view_name in self.table_names(): raise NoView( - "View {name} does not exist - {name} is a table".format( - name=view_name - ) + f"View {view_name} does not exist - {view_name} is a table" ) - raise NoView("View {} does not exist".format(view_name)) + raise NoView(f"View {view_name} does not exist") return View(self, view_name) def quote(self, value: str) -> str: @@ -1013,9 +1002,7 @@ class Database: query += '"' bits = _quote_fts_re.split(query) bits = [b for b in bits if b and b != '""'] - return " ".join( - '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits - ) + return " ".join(f'"{bit}"' if not bit.startswith('"') else bit for bit in bits) def quote_default_value(self, value: str) -> str: if any( @@ -1036,11 +1023,11 @@ class Database: if str(value).endswith(")"): # Expr - return "({})".format(value) + return f"({value})" return self.quote(value) - def table_names(self, fts4: bool = False, fts5: bool = False) -> List[str]: + def table_names(self, fts4: bool = False, fts5: bool = False) -> list[str]: """ List of string table names in this database. @@ -1055,7 +1042,7 @@ class Database: sql = "select name from sqlite_master where {}".format(" AND ".join(where)) return [r[0] for r in self.execute(sql).fetchall()] - def view_names(self) -> List[str]: + def view_names(self) -> list[str]: "List of string view names in this database." return [ r[0] @@ -1065,17 +1052,17 @@ class Database: ] @property - def tables(self) -> List["Table"]: + def tables(self) -> list["Table"]: "List of Table objects in this database." return [self.table(name) for name in self.table_names()] @property - def views(self) -> List["View"]: + def views(self) -> list["View"]: "List of View objects in this database." return [self.view(name) for name in self.view_names()] @property - def triggers(self) -> List[Trigger]: + def triggers(self) -> list[Trigger]: "List of ``(name, table_name, sql)`` tuples representing triggers in this database." return [ Trigger(*r) @@ -1085,7 +1072,7 @@ class Database: ] @property - def triggers_dict(self) -> Dict[str, str]: + def triggers_dict(self) -> dict[str, str]: "A ``{trigger_name: sql}`` dictionary of triggers in this database." return {trigger.name: trigger.sql for trigger in self.triggers} @@ -1107,14 +1094,12 @@ class Database: "Does this database support STRICT mode?" if not hasattr(self, "_supports_strict"): try: - table_name = "t{}".format(secrets.token_hex(16)) + table_name = f"t{secrets.token_hex(16)}" with self.atomic(): - self.conn.execute( - "create table {} (name text) strict".format(table_name) - ) - self.conn.execute("drop table {}".format(table_name)) + self.conn.execute(f"create table {table_name} (name text) strict") + self.conn.execute(f"drop table {table_name}") self._supports_strict = True - except Exception: + except sqlite3.OperationalError: self._supports_strict = False return self._supports_strict @@ -1122,32 +1107,28 @@ class Database: def supports_on_conflict(self) -> bool: # SQLite's upsert is implemented as INSERT INTO ... ON CONFLICT DO ... if not hasattr(self, "_supports_on_conflict"): - table_name = "t{}".format(secrets.token_hex(16)) + table_name = f"t{secrets.token_hex(16)}" try: with self.atomic(): self.conn.execute( - "create table {} (id integer primary key, name text)".format( - table_name - ) + f"create table {table_name} (id integer primary key, name text)" ) self.conn.execute( - "insert into {} (id, name) values (1, 'one')".format(table_name) + f"insert into {table_name} (id, name) values (1, 'one')" ) self.conn.execute( - ( - "insert into {} (id, name) values (1, 'two') " - "on conflict do update set name = 'two'" - ).format(table_name) + f"insert into {table_name} (id, name) values (1, 'two') " + "on conflict do update set name = 'two'" ) self._supports_on_conflict = True - except Exception: + except sqlite3.OperationalError: self._supports_on_conflict = False finally: - self.conn.execute("drop table if exists {}".format(table_name)) + self.conn.execute(f"drop table if exists {table_name}") return self._supports_on_conflict @property - def sqlite_version(self) -> Tuple[int, ...]: + def sqlite_version(self) -> tuple[int, ...]: "Version of SQLite, as a tuple of integers for example ``(3, 36, 0)``." row = self.execute("select sqlite_version()").fetchall()[0] return tuple(map(int, row[0].split("."))) @@ -1191,7 +1172,7 @@ class Database: # guarantee of atomic() and of user-managed transactions if self.conn.in_transaction: raise TransactionError( - "{} cannot be used while a transaction is open".format(operation) + f"{operation} cannot be used while a transaction is open" ) def _ensure_counts_table(self) -> None: @@ -1212,14 +1193,14 @@ class Database: table.enable_counts() self.use_counts_table = True - def cached_counts(self, tables: Optional[Iterable[str]] = None) -> Dict[str, int]: + def cached_counts(self, tables: Iterable[str] | None = None) -> dict[str, int]: """ Return ``{table_name: count}`` dictionary of cached counts for specified tables, or all tables if ``tables`` not provided. :param tables: Subset list of tables to return counts for. """ - sql = 'select "table", count from {}'.format(self._counts_table_name) + sql = f'select "table", count from {self._counts_table_name}' tables_list = list(tables) if tables else None if tables_list: sql += ' where "table" in ({})'.format(", ".join("?" for _ in tables_list)) @@ -1241,13 +1222,13 @@ class Database: ) def execute_returning_dicts( - self, sql: str, params: Optional[Union[Sequence, Dict[str, Any]]] = None - ) -> List[dict]: + self, sql: str, params: Sequence | dict[str, Any] | None = None + ) -> list[dict]: return list(self.query(sql, params)) def resolve_foreign_keys( self, name: str, foreign_keys: ForeignKeysType - ) -> List[ForeignKey]: + ) -> list[ForeignKey]: """ Given a list of differing foreign_keys definitions, return a list of fully resolved ForeignKey() named tuples. @@ -1274,7 +1255,7 @@ class Database: fks.append(ForeignKey(name, fk, other_table, other_column)) continue if not isinstance(fk, (tuple, list)): - raise ValueError( + raise ValueError( # noqa: TRY004 "foreign_keys= should be a list of tuples, " "ForeignKey objects or column name strings" ) @@ -1282,9 +1263,7 @@ class Database: if len(tuple_or_list) == 4: if tuple_or_list[0] != name: raise ValueError( - "First item in {} should have been {}".format( - tuple_or_list, name - ) + f"First item in {tuple_or_list} should have been {name}" ) tuple_or_list = tuple_or_list[1:] if len(tuple_or_list) not in (2, 3): @@ -1299,8 +1278,8 @@ class Database: if len(tuple_or_list) == 3: if not isinstance(tuple_or_list[2], (list, tuple)): raise ValueError( - "Compound foreign key {} should reference a tuple " - "of other columns".format(tuple(tuple_or_list)) + f"Compound foreign key {tuple(tuple_or_list)} should reference a tuple " + "of other columns" ) other_columns = tuple(tuple_or_list[2]) else: @@ -1308,8 +1287,8 @@ class Database: other_columns = tuple(self.table(other_table).pks) if len(columns) != len(other_columns): raise ValueError( - "Compound foreign key {} should have the same number " - "of columns on both sides".format(tuple(tuple_or_list)) + f"Compound foreign key {tuple(tuple_or_list)} should have the same number " + "of columns on both sides" ) if len(columns) == 1: # Single-column key passed as a one-item list @@ -1389,15 +1368,15 @@ class Database: def create_table_sql( self, name: str, - columns: Dict[str, Any], - pk: Optional[Any] = None, - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, - hash_id: Optional[str] = None, - hash_id_columns: Optional[Iterable[str]] = None, - extracts: Optional[Union[Dict[str, str], List[str]]] = None, + columns: dict[str, Any], + pk: Any | None = None, + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, + hash_id: str | None = None, + hash_id_columns: Iterable[str] | None = None, + extracts: dict[str, str] | list[str] | None = None, if_not_exists: bool = False, strict: bool = False, ) -> str: @@ -1419,7 +1398,7 @@ class Database: """ if hash_id_columns and (hash_id is None): hash_id = "id" - resolved_fks: List[ForeignKey] = [ + resolved_fks: list[ForeignKey] = [ self._resolve_foreign_key_casing(fk, columns) for fk in self.resolve_foreign_keys(name, foreign_keys or []) ] @@ -1449,15 +1428,11 @@ class Database: raise ValueError("Tables must have at least one column") if not all(n in columns for n in not_null): raise ValueError( - "not_null set {} includes items not in columns {}".format( - repr(not_null), repr(set(columns.keys())) - ) + f"not_null set {not_null!r} includes items not in columns {set(columns.keys())!r}" ) if not all(n in columns for n in defaults): raise ValueError( - "defaults set {} includes items not in columns {}".format( - repr(set(defaults)), repr(set(columns.keys())) - ) + f"defaults set {set(defaults)!r} includes items not in columns {set(columns.keys())!r}" ) column_items = list(columns.items()) if column_order is not None: @@ -1477,9 +1452,7 @@ class Database: if other_column != "rowid" and not any( c for c in self[fk.other_table].columns if c.name == other_column ): - raise AlterError( - "No such column: {}.{}".format(fk.other_table, other_column) - ) + raise AlterError(f"No such column: {fk.other_table}.{other_column}") column_defs = [] # ensure pk is a tuple @@ -1500,16 +1473,12 @@ class Database: column_extras.append("NOT NULL") if column_name in defaults and defaults[column_name] is not None: column_extras.append( - "DEFAULT {}".format(self.quote_default_value(defaults[column_name])) + f"DEFAULT {self.quote_default_value(defaults[column_name])}" ) if column_name in foreign_keys_by_column: fk = foreign_keys_by_column[column_name] column_extras.append( - "REFERENCES {}({}){}".format( - quote_identifier(fk.other_table), - quote_identifier(cast(str, fk.other_column)), - _fk_actions_sql(fk), - ) + f"REFERENCES {quote_identifier(fk.other_table)}({quote_identifier(cast(str, fk.other_column))}){_fk_actions_sql(fk)}" ) column_type_str = COLUMN_TYPE_MAPPING[column_type] # Special case for strict tables to map FLOAT to REAL @@ -1566,15 +1535,15 @@ class Database: def create_table( self, name: str, - columns: Dict[str, Any], - pk: Optional[Any] = None, - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, - hash_id: Optional[str] = None, - hash_id_columns: Optional[Iterable[str]] = None, - extracts: Optional[Union[Dict[str, str], List[str]]] = None, + columns: dict[str, Any], + pk: Any | None = None, + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, + hash_id: str | None = None, + hash_id_columns: Iterable[str] | None = None, + extracts: dict[str, str] | list[str] | None = None, if_not_exists: bool = False, replace: bool = False, ignore: bool = False, @@ -1618,11 +1587,11 @@ class Database: resolve_casing(col_name, existing_columns): col_type for col_name, col_type in columns.items() } - missing_columns = dict( - (col_name, col_type) + missing_columns = { + col_name: col_type for col_name, col_type in columns.items() if col_name not in existing_columns - ) + } columns_to_drop = [ column for column in existing_columns if column not in columns ] @@ -1709,9 +1678,7 @@ class Database: :param new_name: Name to rename it to """ self.execute( - "ALTER TABLE {} RENAME TO {}".format( - quote_identifier(name), quote_identifier(new_name) - ) + f"ALTER TABLE {quote_identifier(name)} RENAME TO {quote_identifier(new_name)}" ) def create_view( @@ -1727,23 +1694,20 @@ class Database: """ if ignore and replace: raise ValueError("Use one or the other of ignore/replace, not both") - create_sql = "CREATE VIEW {name} AS {sql}".format( - name=quote_identifier(name), sql=sql - ) - if ignore or replace: - # Does view exist already? - if name in self.view_names(): - if ignore: + create_sql = f"CREATE VIEW {quote_identifier(name)} AS {sql}" + if (ignore or replace) and name in self.view_names(): + # View exists already + if ignore: + return self + elif replace: + # If SQL is the same, do nothing + if create_sql == self[name].schema: return self - elif replace: - # If SQL is the same, do nothing - if create_sql == self[name].schema: - return self - self[name].drop() + self[name].drop() self.execute(create_sql) return self - def m2m_table_candidates(self, table: str, other_table: str) -> List[str]: + def m2m_table_candidates(self, table: str, other_table: str) -> list[str]: """ Given two table names returns the name of tables that could define a many-to-many relationship between those two tables, based on having @@ -1762,7 +1726,7 @@ class Database: return candidates def add_foreign_keys( - self, foreign_keys: Iterable[Union[ForeignKey, ForeignKeyTuple]] + self, foreign_keys: Iterable[ForeignKey | ForeignKeyTuple] ) -> None: """ See :ref:`python_api_add_foreign_keys`. @@ -1782,7 +1746,7 @@ class Database: "(table, column, other_table, other_column)" ) - foreign_keys_to_create: List[ForeignKey] = [] + foreign_keys_to_create: list[ForeignKey] = [] # Verify that all tables and columns exist for fk in foreign_keys: @@ -1823,7 +1787,7 @@ class Database: table = fk_object.table other_table = fk_object.other_table if not self.table(table).exists(): - raise AlterError("No such table: {}".format(table)) + raise AlterError(f"No such table: {table}") table_obj = self.table(table) fk_object = self._resolve_foreign_key_casing( fk_object, table_obj.columns_dict @@ -1832,18 +1796,16 @@ class Database: other_columns = fk_object.other_columns for column in columns: if column not in table_obj.columns_dict: - raise AlterError("No such column: {} in {}".format(column, table)) + raise AlterError(f"No such column: {column} in {table}") if not self[other_table].exists(): - raise AlterError("No such other_table: {}".format(other_table)) + raise AlterError(f"No such other_table: {other_table}") for other_column in other_columns: if ( other_column != "rowid" and other_column not in self[other_table].columns_dict ): raise AlterError( - "No such other_column: {} in {}".format( - other_column, other_table - ) + f"No such other_column: {other_column} in {other_table}" ) # Silently skip foreign keys that exist already - but only if # they match exactly, including ON DELETE/ON UPDATE actions @@ -1874,7 +1836,7 @@ class Database: ) # Group them by table - by_table: Dict[str, List[ForeignKey]] = {} + by_table: dict[str, list[ForeignKey]] = {} for fk_object in foreign_keys_to_create: by_table.setdefault(fk_object.table, []).append(fk_object) @@ -1899,7 +1861,7 @@ class Database: "Run a SQLite ``VACUUM`` against the database." self.execute("VACUUM;") - def analyze(self, name: Optional[str] = None) -> None: + def analyze(self, name: str | None = None) -> None: """ Run ``ANALYZE`` against the entire database or a named table or index. @@ -1907,7 +1869,7 @@ class Database: """ sql = "ANALYZE" if name is not None: - sql += " {}".format(quote_identifier(name)) + sql += f" {quote_identifier(name)}" self.execute(sql) def iterdump(self) -> Generator[str, None, None]: @@ -1922,7 +1884,7 @@ class Database: "conn.iterdump() not found - try pip install sqlite-dump" ) - def init_spatialite(self, path: Optional[str] = None) -> bool: + def init_spatialite(self, path: str | None = None) -> bool: """ The ``init_spatialite`` method will load and initialize the SpatiaLite extension. The ``path`` argument should be an absolute path to the compiled extension, which @@ -1980,8 +1942,8 @@ class Queryable: def count_where( self, - where: Optional[str] = None, - where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + where: str | None = None, + where_args: Sequence | dict[str, Any] | None = None, ) -> int: """ Executes ``SELECT count(*) FROM table WHERE ...`` and returns a count. @@ -1990,7 +1952,7 @@ class Queryable: :param where_args: Parameters to use with that fragment - an iterable for ``id > ?`` parameters, or a dictionary for ``id > :id`` """ - sql = "select count(*) from {}".format(quote_identifier(self.name)) + sql = f"select count(*) from {quote_identifier(self.name)}" if where is not None: sql += " where " + where return self.db.execute(sql, where_args or []).fetchone()[0] @@ -2005,19 +1967,19 @@ class Queryable: return self.count_where() @property - def rows(self) -> Generator[Dict[str, Any], None, None]: + def rows(self) -> Generator[dict[str, Any], None, None]: "Iterate over every dictionaries for each row in this table or view." return self.rows_where() def rows_where( self, - where: Optional[str] = None, - where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, - order_by: Optional[str] = None, + where: str | None = None, + where_args: Sequence | dict[str, Any] | None = None, + order_by: str | None = None, select: str = "*", - limit: Optional[int] = None, - offset: Optional[int] = None, - ) -> Generator[Dict[str, Any], None, None]: + limit: int | None = None, + offset: int | None = None, + ) -> Generator[dict[str, Any], None, None]: """ Iterate over every row in this table or view that matches the specified where clause. @@ -2033,15 +1995,15 @@ class Queryable: """ if not self.exists(): return - sql = "select {} from {}".format(select, quote_identifier(self.name)) + sql = f"select {select} from {quote_identifier(self.name)}" if where is not None: sql += " where " + where if order_by is not None: sql += " order by " + order_by if limit is not None: - sql += " limit {}".format(limit) + sql += f" limit {limit}" if offset is not None: - sql += " offset {}".format(offset) + sql += f" offset {offset}" cursor = self.db.execute(sql, where_args or []) columns = dedupe_keys(c[0] for c in cursor.description) for row in cursor: @@ -2049,12 +2011,12 @@ class Queryable: def pks_and_rows_where( self, - where: Optional[str] = None, - where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, - order_by: Optional[str] = None, - limit: Optional[int] = None, - offset: Optional[int] = None, - ) -> Generator[Tuple[Any, Dict[str, Any]], None, None]: + where: str | None = None, + where_args: Sequence | dict[str, Any] | None = None, + order_by: str | None = None, + limit: int | None = None, + offset: int | None = None, + ) -> Generator[tuple[Any, dict[str, Any]], None, None]: """ Like ``.rows_where()`` but returns ``(pk, row)`` pairs - ``pk`` can be a single value or tuple. @@ -2096,17 +2058,17 @@ class Queryable: yield row_pk, row @property - def columns(self) -> List["Column"]: + def columns(self) -> list["Column"]: "List of :ref:`Columns ` representing the columns in this table or view." if not self.exists(): return [] rows = self.db.execute( - "PRAGMA table_info({})".format(quote_identifier(self.name)) + f"PRAGMA table_info({quote_identifier(self.name)})" ).fetchall() return [Column(*row) for row in rows] @property - def columns_dict(self) -> Dict[str, Any]: + def columns_dict(self) -> dict[str, Any]: "``{column_name: python-type}`` dictionary representing columns in this table or view." return {column.name: column_affinity(column.type) for column in self.columns} @@ -2146,48 +2108,48 @@ class Table(Queryable): """ #: The ``rowid`` of the last inserted, updated or selected row. - last_rowid: Optional[int] = None + last_rowid: int | None = None #: The primary key of the last inserted, updated or selected row. - last_pk: Optional[Any] = None + last_pk: Any | None = None def __init__( self, db: Database, name: str, - pk: Optional[Any] = None, - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, + pk: Any | None = None, + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, batch_size: int = 100, - hash_id: Optional[str] = None, - hash_id_columns: Optional[Iterable[str]] = None, + hash_id: str | None = None, + hash_id_columns: Iterable[str] | None = None, alter: bool = False, ignore: bool = False, replace: bool = False, - extracts: Optional[Union[Dict[str, str], List[str]]] = None, - conversions: Optional[dict] = None, - columns: Optional[Dict[str, Any]] = None, + extracts: dict[str, str] | list[str] | None = None, + conversions: dict | None = None, + columns: dict[str, Any] | None = None, strict: bool = False, ): super().__init__(db, name) - self._defaults = dict( - pk=pk, - foreign_keys=foreign_keys, - column_order=column_order, - not_null=not_null, - defaults=defaults, - batch_size=batch_size, - hash_id=hash_id, - hash_id_columns=hash_id_columns, - alter=alter, - ignore=ignore, - replace=replace, - extracts=extracts, - conversions=conversions or {}, - columns=columns, - strict=strict, - ) + self._defaults = { + "pk": pk, + "foreign_keys": foreign_keys, + "column_order": column_order, + "not_null": not_null, + "defaults": defaults, + "batch_size": batch_size, + "hash_id": hash_id, + "hash_id_columns": hash_id_columns, + "alter": alter, + "ignore": ignore, + "replace": replace, + "extracts": extracts, + "conversions": conversions or {}, + "columns": columns, + "strict": strict, + } def __repr__(self) -> str: return "
".format( @@ -2212,7 +2174,7 @@ class Table(Queryable): return self.name in self.db.table_names() @property - def pks(self) -> List[str]: + def pks(self) -> list[str]: """ Primary key columns for this table, in PRIMARY KEY declaration order - ``PRAGMA table_info`` sets ``is_pk`` to the 1-based position of each @@ -2234,7 +2196,7 @@ class Table(Queryable): "Does this table use ``rowid`` for its primary key (no other primary keys are specified)?" return not any(column for column in self.columns if column.is_pk) - def get(self, pk_values: Union[list, tuple, str, int]) -> dict: + def get(self, pk_values: list | tuple | str | int) -> dict: """ Return row (as dictionary) for the specified primary key. @@ -2253,17 +2215,17 @@ class Table(Queryable): ) ) - wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in pks] + wheres = [f"{quote_identifier(pk_name)} = ?" for pk_name in pks] rows = self.rows_where(" and ".join(wheres), pk_values) try: - row = list(rows)[0] + row = next(iter(rows)) self.last_pk = last_pk return row - except IndexError: + except StopIteration: raise NotFoundError @property - def foreign_keys(self) -> List["ForeignKey"]: + def foreign_keys(self) -> list["ForeignKey"]: """ List of foreign keys defined on this table. @@ -2273,12 +2235,12 @@ class Table(Queryable): """ # PRAGMA foreign_key_list returns one row per column, grouped by "id" # with "seq" giving the column order within a compound foreign key. - by_id: Dict[int, list] = {} + by_id: dict[int, list] = {} for row in self.db.execute( - "PRAGMA foreign_key_list({})".format(quote_identifier(self.name)) + f"PRAGMA foreign_key_list({quote_identifier(self.name)})" ).fetchall(): if row is not None: - id, seq, table_name, from_, to_, on_update, on_delete, match = row + id, seq, table_name, from_, to_, on_update, on_delete, _match = row by_id.setdefault(id, []).append( (seq, table_name, from_, to_, on_update, on_delete) ) @@ -2311,7 +2273,7 @@ class Table(Queryable): return fks @property - def virtual_table_using(self) -> Optional[str]: + def virtual_table_using(self) -> str | None: "Type of virtual table, or ``None`` if this is not a virtual table." match = _virtual_table_using_re.match(self.schema) if match is None: @@ -2319,18 +2281,16 @@ class Table(Queryable): return match.groupdict()["using"].upper() @property - def indexes(self) -> List[Index]: + def indexes(self) -> list[Index]: "List of indexes defined on this table." - sql = 'PRAGMA index_list("{}")'.format(self.name) + sql = f'PRAGMA index_list("{self.name}")' indexes = [] for row in self.db.execute_returning_dicts(sql): index_name = row["name"] index_name_quoted = ( - '"{}"'.format(index_name) - if not index_name.startswith('"') - else index_name + f'"{index_name}"' if not index_name.startswith('"') else index_name ) - column_sql = "PRAGMA index_info({})".format(index_name_quoted) + column_sql = f"PRAGMA index_info({index_name_quoted})" columns = [] for seqno, cid, name in self.db.execute(column_sql).fetchall(): columns.append(name) @@ -2343,18 +2303,16 @@ class Table(Queryable): return indexes @property - def xindexes(self) -> List[XIndex]: + def xindexes(self) -> list[XIndex]: "List of indexes defined on this table using the more detailed ``XIndex`` format." - sql = 'PRAGMA index_list("{}")'.format(self.name) + sql = f'PRAGMA index_list("{self.name}")' indexes = [] for row in self.db.execute_returning_dicts(sql): index_name = row["name"] index_name_quoted = ( - '"{}"'.format(index_name) - if not index_name.startswith('"') - else index_name + f'"{index_name}"' if not index_name.startswith('"') else index_name ) - column_sql = "PRAGMA index_xinfo({})".format(index_name_quoted) + column_sql = f"PRAGMA index_xinfo({index_name_quoted})" index_columns = [] for info in self.db.execute(column_sql).fetchall(): index_columns.append(XIndexColumn(*info)) @@ -2362,7 +2320,7 @@ class Table(Queryable): return indexes @property - def triggers(self) -> List[Trigger]: + def triggers(self) -> list[Trigger]: "List of triggers defined on this table." return [ Trigger(*r) @@ -2374,12 +2332,12 @@ class Table(Queryable): ] @property - def triggers_dict(self) -> Dict[str, str]: + def triggers_dict(self) -> dict[str, str]: "``{trigger_name: sql}`` dictionary of triggers defined on this table." return {trigger.name: trigger.sql for trigger in self.triggers} @property - def default_values(self) -> Dict[str, Any]: + def default_values(self) -> dict[str, Any]: "``{column_name: default_value}`` dictionary of default values for columns in this table." return { column.name: _decode_default_value(column.default_value) @@ -2396,20 +2354,20 @@ class Table(Queryable): def create( self, - columns: Dict[str, Any], - pk: Optional[Any] = DEFAULT, - foreign_keys: Union[Optional[ForeignKeysType], Default] = DEFAULT, - column_order: Union[Optional[List[str]], Default] = DEFAULT, - not_null: Union[Optional[Iterable[str]], Default] = DEFAULT, - defaults: Union[Optional[Dict[str, Any]], Default] = DEFAULT, - hash_id: Union[Optional[str], Default] = DEFAULT, - hash_id_columns: Union[Optional[Iterable[str]], Default] = DEFAULT, - extracts: Union[Optional[Union[Dict[str, str], List[str]]], Default] = DEFAULT, + columns: dict[str, Any], + pk: Any | None = DEFAULT, + foreign_keys: ForeignKeysType | None | Default = DEFAULT, + column_order: list[str] | None | Default = DEFAULT, + not_null: Iterable[str] | None | Default = DEFAULT, + defaults: dict[str, Any] | None | Default = DEFAULT, + hash_id: str | None | Default = DEFAULT, + hash_id_columns: Iterable[str] | None | Default = DEFAULT, + extracts: dict[str, str] | list[str] | None | Default = DEFAULT, if_not_exists: bool = False, replace: bool = False, ignore: bool = False, transform: bool = False, - strict: Union[bool, Default] = DEFAULT, + strict: bool | Default = DEFAULT, ) -> "Table": """ Create a table with the specified columns. @@ -2493,28 +2451,25 @@ class Table(Queryable): if not self.exists(): raise NoTable(f"Table {self.name} does not exist") with self.db.atomic(): - sql = "CREATE TABLE {} AS SELECT * FROM {};".format( - quote_identifier(new_name), - quote_identifier(self.name), - ) + sql = f"CREATE TABLE {quote_identifier(new_name)} AS SELECT * FROM {quote_identifier(self.name)};" self.db.execute(sql) return self.db.table(new_name) def transform( self, *, - types: Optional[dict] = None, - rename: Optional[dict] = None, - drop: Optional[Iterable] = None, - pk: Optional[Any] = DEFAULT, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, - drop_foreign_keys: Optional[Iterable[str]] = None, - add_foreign_keys: Optional[ForeignKeysType] = None, - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - keep_table: Optional[str] = None, - strict: Optional[bool] = None, + types: dict | None = None, + rename: dict | None = None, + drop: Iterable | None = None, + pk: Any | None = DEFAULT, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, + drop_foreign_keys: Iterable[str] | None = None, + add_foreign_keys: ForeignKeysType | None = None, + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + keep_table: str | None = None, + strict: bool | None = None, ) -> "Table": """ Apply an advanced alter table, including operations that are not supported by @@ -2633,20 +2588,20 @@ class Table(Queryable): def transform_sql( self, *, - types: Optional[dict] = None, - rename: Optional[dict] = None, - drop: Optional[Iterable] = None, - pk: Optional[Any] = DEFAULT, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, - drop_foreign_keys: Optional[Iterable] = None, - add_foreign_keys: Optional[ForeignKeysType] = None, - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - tmp_suffix: Optional[str] = None, - keep_table: Optional[str] = None, - strict: Optional[bool] = None, - ) -> List[str]: + types: dict | None = None, + rename: dict | None = None, + drop: Iterable | None = None, + pk: Any | None = DEFAULT, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, + drop_foreign_keys: Iterable | None = None, + add_foreign_keys: ForeignKeysType | None = None, + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + tmp_suffix: str | None = None, + keep_table: str | None = None, + strict: bool | None = None, + ) -> list[str]: """ Return a list of SQL statements that should be executed in order to apply this transformation. @@ -2689,7 +2644,7 @@ class Table(Queryable): if isinstance(not_null, dict): not_null = { resolve_casing(c, existing_columns): v - for c, v in cast(Dict[str, Any], not_null).items() + for c, v in cast(dict[str, Any], not_null).items() } elif isinstance(not_null, set): not_null = {resolve_casing(c, existing_columns) for c in not_null} @@ -2700,7 +2655,7 @@ class Table(Queryable): if column_order is not None: column_order = [resolve_casing(c, existing_columns) for c in column_order] - create_table_foreign_keys: List[ForeignKeyIndicator] = [] + create_table_foreign_keys: list[ForeignKeyIndicator] = [] if foreign_keys is not None: if add_foreign_keys is not None: @@ -2777,9 +2732,7 @@ class Table(Queryable): for fk in self.db.resolve_foreign_keys(self.name, add_foreign_keys): create_table_foreign_keys.append(fk_with_renamed_columns(fk)) - new_table_name = "{}_new_{}".format( - self.name, tmp_suffix or os.urandom(6).hex() - ) + new_table_name = f"{self.name}_new_{tmp_suffix or os.urandom(6).hex()}" current_column_pairs = list(self.columns_dict.items()) new_column_pairs = [] copy_from_to = {column: column for column, _ in current_column_pairs} @@ -2824,9 +2777,7 @@ class Table(Queryable): pass else: raise ValueError( - "not_null must be a dict or a set or None, it was {}".format( - repr(not_null) - ) + f"not_null must be a dict or a set or None, it was {not_null!r}" ) # defaults= create_table_defaults = { @@ -2876,17 +2827,13 @@ class Table(Queryable): # Drop (or keep) the old table if keep_table: sqls.append( - "ALTER TABLE {} RENAME TO {};".format( - quote_identifier(self.name), quote_identifier(keep_table) - ) + f"ALTER TABLE {quote_identifier(self.name)} RENAME TO {quote_identifier(keep_table)};" ) else: - sqls.append("DROP TABLE {};".format(quote_identifier(self.name))) + sqls.append(f"DROP TABLE {quote_identifier(self.name)};") # Rename the new one sqls.append( - "ALTER TABLE {} RENAME TO {};".format( - quote_identifier(new_table_name), quote_identifier(self.name) - ) + f"ALTER TABLE {quote_identifier(new_table_name)} RENAME TO {quote_identifier(self.name)};" ) # Re-add existing indexes for index in self.indexes: @@ -2904,7 +2851,7 @@ class Table(Queryable): if keep_table: sqls.append(f"DROP INDEX IF EXISTS {quote_identifier(index.name)};") for col in index.columns: - if col in rename.keys() or col in drop: + if col in rename or col in drop: raise TransformError( f"Index '{index.name}' column '{col}' is not in updated table '{self.name}'. " f"You must manually drop this index prior to running this transformation " @@ -2916,10 +2863,10 @@ class Table(Queryable): def extract( self, - columns: Union[str, Iterable[str]], - table: Optional[str] = None, - fk_column: Optional[str] = None, - rename: Optional[Dict[str, str]] = None, + columns: str | Iterable[str], + table: str | None = None, + fk_column: str | None = None, + rename: dict[str, str] | None = None, ) -> "Table": """ Extract specified columns into a separate table. @@ -2938,15 +2885,13 @@ class Table(Queryable): rename = {resolve_casing(k, self.columns_dict): v for k, v in rename.items()} if not set(columns).issubset(self.columns_dict.keys()): raise InvalidColumns( - "Invalid columns {} for table with columns {}".format( - columns, list(self.columns_dict.keys()) - ) + f"Invalid columns {columns} for table with columns {list(self.columns_dict.keys())}" ) with self.db.atomic(): table = table or "_".join(columns) lookup_table = self.db.table(table) - fk_column = fk_column or "{}_id".format(table) - magic_lookup_column = "{}_{}".format(fk_column, os.urandom(6).hex()) + fk_column = fk_column or f"{table}_id" + magic_lookup_column = f"{fk_column}_{os.urandom(6).hex()}" # Populate the lookup table with all of the extracted unique values lookup_columns_definition = { @@ -2959,16 +2904,12 @@ class Table(Queryable): lookup_table.columns_dict.items() ): raise InvalidColumns( - "Lookup table {} already exists but does not have columns {}".format( - table, lookup_columns_definition - ) + f"Lookup table {table} already exists but does not have columns {lookup_columns_definition}" ) else: lookup_table.create( { - **{ - "id": int, - }, + "id": int, **lookup_columns_definition, }, pk="id", @@ -2978,19 +2919,14 @@ class Table(Queryable): # Rows where every extracted column is null are left alone - they # get a null foreign key and no lookup table record, see #186 all_columns_are_null = " AND ".join( - "{} IS NULL".format(quote_identifier(c)) for c in columns + f"{quote_identifier(c)} IS NULL" for c in columns ) # INSERT OR IGNORE dedupes against the unique index, but unique # indexes treat NULLs as distinct - the NOT EXISTS guard uses IS # comparison so NULL-containing rows match existing lookup rows # instead of being inserted again already_in_lookup = " AND ".join( - "{lookup}.{lookup_col} IS {source}.{source_col}".format( - lookup=quote_identifier(table), - lookup_col=quote_identifier(rename.get(column) or column), - source=quote_identifier(self.name), - source_col=quote_identifier(column), - ) + f"{quote_identifier(table)}.{quote_identifier(rename.get(column) or column)} IS {quote_identifier(self.name)}.{quote_identifier(column)}" for column in columns ) self.db.execute( @@ -3018,12 +2954,10 @@ class Table(Queryable): quote_identifier(magic_lookup_column), quote_identifier(table), where=" AND ".join( - "{}.{} IS {}.{}".format( - quote_identifier(self.name), - quote_identifier(column), - quote_identifier(table), - quote_identifier(rename.get(column) or column), - ) + f"{quote_identifier(self.name)}." + f"{quote_identifier(column)} IS " + f"{quote_identifier(table)}." + f"{quote_identifier(rename.get(column) or column)}" for column in columns ), all_null=all_columns_are_null, @@ -3052,8 +2986,8 @@ class Table(Queryable): def create_index( self, - columns: Iterable[Union[str, DescIndex]], - index_name: Optional[str] = None, + columns: Iterable[str | DescIndex], + index_name: str | None = None, unique: bool = False, if_not_exists: bool = False, find_unique_name: bool = False, @@ -3080,16 +3014,14 @@ class Table(Queryable): columns_sql = [] for column in columns: if isinstance(column, DescIndex): - columns_sql.append("{} desc".format(quote_identifier(column))) + columns_sql.append(f"{quote_identifier(column)} desc") else: columns_sql.append(quote_identifier(column)) suffix = None created_index_name = None while True: - created_index_name = ( - "{}_{}".format(index_name, suffix) if suffix else index_name - ) + created_index_name = f"{index_name}_{suffix}" if suffix else index_name sql = ( textwrap.dedent(""" CREATE {unique}INDEX {if_not_exists}{index_name} @@ -3121,7 +3053,7 @@ class Table(Queryable): suffix += 1 continue else: - raise e + raise if analyze: self.db.analyze(created_index_name) return self @@ -3136,19 +3068,17 @@ class Table(Queryable): if index_name not in {index.name for index in self.indexes}: if ignore: return self - raise OperationalError( - "No index named {} on table {}".format(index_name, self.name) - ) - self.db.execute("DROP INDEX {}".format(quote_identifier(index_name))) + raise OperationalError(f"No index named {index_name} on table {self.name}") + self.db.execute(f"DROP INDEX {quote_identifier(index_name)}") return self def add_column( self, col_name: str, - col_type: Optional[Any] = None, - fk: Optional[str] = None, - fk_col: Optional[str] = None, - not_null_default: Optional[Any] = None, + col_type: Any | None = None, + fk: str | None = None, + fk_col: str | None = None, + not_null_default: Any | None = None, ): """ Add a column to this table. See :ref:`python_api_add_column`. @@ -3163,12 +3093,12 @@ class Table(Queryable): if fk is not None: # fk must be a valid table if fk not in self.db.table_names(): - raise AlterError("table '{}' does not exist".format(fk)) + raise AlterError(f"table '{fk}' does not exist") # if fk_col specified, must be a valid column if fk_col is not None: fk_col = resolve_casing(fk_col, self.db[fk].columns_dict) if fk_col not in self.db[fk].columns_dict: - raise AlterError("table '{}' has no column {}".format(fk, fk_col)) + raise AlterError(f"table '{fk}' has no column {fk_col}") else: # automatically set fk_col to first primary_key of fk table pks = sorted( @@ -3185,8 +3115,8 @@ class Table(Queryable): col_type = str not_null_sql = None if not_null_default is not None: - not_null_sql = "NOT NULL DEFAULT {}".format( - self.db.quote_default_value(not_null_default) + not_null_sql = ( + f"NOT NULL DEFAULT {self.db.quote_default_value(not_null_default)}" ) sql = "ALTER TABLE {} ADD COLUMN {} {col_type}{not_null_default};".format( quote_identifier(self.name), @@ -3206,7 +3136,7 @@ class Table(Queryable): :param ignore: Set to ``True`` to ignore the error if the table does not exist """ try: - self.db.execute("DROP TABLE {}".format(quote_identifier(self.name))) + self.db.execute(f"DROP TABLE {quote_identifier(self.name)}") except sqlite3.OperationalError: if not ignore: raise @@ -3238,16 +3168,14 @@ class Table(Queryable): return existing_tables[table] # If we get here there's no obvious candidate - raise an error raise NoObviousTable( - "No obvious foreign key table for column '{}' - tried {}".format( - column, repr(possibilities) - ) + f"No obvious foreign key table for column '{column}' - tried {possibilities!r}" ) def guess_foreign_column(self, other_table: str) -> str: pks = [c for c in self.db[other_table].columns if c.is_pk] if len(pks) != 1: raise BadPrimaryKey( - "Could not detect single primary key for table '{}'".format(other_table) + f"Could not detect single primary key for table '{other_table}'" ) else: return pks[0].name @@ -3255,8 +3183,8 @@ class Table(Queryable): def add_foreign_key( self, column: ForeignKeyColumns, - other_table: Optional[str] = None, - other_column: Optional[ForeignKeyColumns] = None, + other_table: str | None = None, + other_column: ForeignKeyColumns | None = None, ignore: bool = False, on_delete: str = "NO ACTION", on_update: str = "NO ACTION", @@ -3279,7 +3207,7 @@ class Table(Queryable): # Ensure columns exist for col in columns: if col not in self.columns_dict: - raise AlterError("No such column: {}".format(col)) + raise AlterError(f"No such column: {col}") # If other_table is not specified, attempt to guess it from the column if other_table is None: if len(columns) > 1: @@ -3312,7 +3240,7 @@ class Table(Queryable): not [c for c in self.db[other_table].columns if c.name == other_col] and other_col != "rowid" ): - raise AlterError("No such column: {}.{}".format(other_table, other_col)) + raise AlterError(f"No such column: {other_table}.{other_col}") # Check we do not already have an existing foreign key if any( fk @@ -3413,9 +3341,7 @@ class Table(Queryable): def has_counts_triggers(self) -> bool: "Does this table have triggers setup to update cached counts?" trigger_names = { - "{table}{counts_table}_{suffix}".format( - counts_table=self.db._counts_table_name, table=self.name, suffix=suffix - ) + f"{self.name}{self.db._counts_table_name}_{suffix}" for suffix in ["insert", "delete"] } return trigger_names.issubset(self.triggers_dict.keys()) @@ -3425,7 +3351,7 @@ class Table(Queryable): columns: Iterable[str], fts_version: str = "FTS5", create_triggers: bool = False, - tokenize: Optional[str] = None, + tokenize: str | None = None, replace: bool = False, ): """ @@ -3452,13 +3378,13 @@ class Table(Queryable): table_fts=quote_identifier(self.name + "_fts"), columns=", ".join(quote_identifier(c) for c in columns), fts_version=fts_version, - tokenize="\n tokenize='{}',".format(tokenize) if tokenize else "", + tokenize=f"\n tokenize='{tokenize}'," if tokenize else "", ) ) should_recreate = False - if replace and self.db["{}_fts".format(self.name)].exists(): + if replace and self.db[f"{self.name}_fts"].exists(): # Does the table need to be recreated? - fts_schema = self.db["{}_fts".format(self.name)].schema + fts_schema = self.db[f"{self.name}_fts"].schema if fts_schema != create_fts_sql: should_recreate = True expected_triggers = {self.name + suffix for suffix in ("_ai", "_ad", "_au")} @@ -3477,8 +3403,8 @@ class Table(Queryable): self.populate_fts(columns) if create_triggers: - old_cols = ", ".join("old.{}".format(quote_identifier(c)) for c in columns) - new_cols = ", ".join("new.{}".format(quote_identifier(c)) for c in columns) + old_cols = ", ".join(f"old.{quote_identifier(c)}" for c in columns) + new_cols = ", ".join(f"new.{quote_identifier(c)}" for c in columns) columns_quoted = ", ".join(quote_identifier(c) for c in columns) table = quote_identifier(self.name) table_fts = quote_identifier(self.name + "_fts") @@ -3550,7 +3476,7 @@ class Table(Queryable): with self.db.atomic(): for trigger_name in trigger_names: self.db.execute( - "DROP TRIGGER IF EXISTS {}".format(quote_identifier(trigger_name)) + f"DROP TRIGGER IF EXISTS {quote_identifier(trigger_name)}" ) return self @@ -3568,7 +3494,7 @@ class Table(Queryable): ) return self - def detect_fts(self) -> Optional[str]: + def detect_fts(self) -> str | None: "Detect if table has a corresponding FTS virtual table and return it" sql = textwrap.dedent(""" SELECT name FROM sqlite_master @@ -3583,8 +3509,8 @@ class Table(Queryable): ) """).strip() args = { - "like": "%VIRTUAL TABLE%USING FTS%content=[{}]%".format(self.name), - "like2": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), + "like": f"%VIRTUAL TABLE%USING FTS%content=[{self.name}]%", + "like2": f'%VIRTUAL TABLE%USING FTS%content="{self.name}"%', "table": self.name, } rows = self.db.execute(sql, args).fetchall() @@ -3605,11 +3531,11 @@ class Table(Queryable): def search_sql( self, - columns: Optional[Iterable[str]] = None, - order_by: Optional[str] = None, - limit: Optional[int] = None, - offset: Optional[int] = None, - where: Optional[str] = None, + columns: Iterable[str] | None = None, + order_by: str | None = None, + limit: int | None = None, + offset: int | None = None, + where: str | None = None, include_rank: bool = False, ) -> str: """ " @@ -3626,16 +3552,16 @@ class Table(Queryable): original = "original_" if self.name == "original" else "original" original_quoted = quote_identifier(original) columns_sql = "*" - columns_with_prefix_sql = "{}.*".format(original_quoted) + columns_with_prefix_sql = f"{original_quoted}.*" if columns: columns_sql = ",\n ".join(quote_identifier(c) for c in columns) columns_with_prefix_sql = ",\n ".join( - "{}.{}".format(original_quoted, quote_identifier(c)) for c in columns + f"{original_quoted}.{quote_identifier(c)}" for c in columns ) fts_table = self.detect_fts() if not fts_table: raise ValueError( - "Full-text search is not configured for table '{}'".format(self.name) + f"Full-text search is not configured for table '{self.name}'" ) fts_table_quoted = quote_identifier(fts_table) virtual_table_using = self.db.table(fts_table).virtual_table_using @@ -3658,22 +3584,20 @@ class Table(Queryable): {limit_offset} """).strip() if virtual_table_using == "FTS5": - rank_implementation = "{}.rank".format(fts_table_quoted) + rank_implementation = f"{fts_table_quoted}.rank" else: self.db.register_fts4_bm25() - rank_implementation = "rank_bm25(matchinfo({}, 'pcnalx'))".format( - fts_table_quoted - ) + rank_implementation = f"rank_bm25(matchinfo({fts_table_quoted}, 'pcnalx'))" if include_rank: columns_with_prefix_sql += ",\n " + rank_implementation + " rank" limit_offset = "" if limit is not None: - limit_offset += " limit {}".format(limit) + limit_offset += f" limit {limit}" if offset is not None: - limit_offset += " offset {}".format(offset) + limit_offset += f" offset {offset}" return sql.format( dbtable=quote_identifier(self.name), - where_clause="\n where {}".format(where) if where else "", + where_clause=f"\n where {where}" if where else "", original=original_quoted, columns=columns_sql, columns_with_prefix=columns_with_prefix_sql, @@ -3685,12 +3609,12 @@ class Table(Queryable): def search( self, q: str, - order_by: Optional[str] = None, - columns: Optional[Iterable[str]] = None, - limit: Optional[int] = None, - offset: Optional[int] = None, - where: Optional[str] = None, - where_args: Optional[Union[Iterable, dict]] = None, + order_by: str | None = None, + columns: Iterable[str] | None = None, + limit: int | None = None, + offset: int | None = None, + where: str | None = None, + where_args: Iterable | dict | None = None, include_rank: bool = False, quote: bool = False, ) -> Generator[dict, None, None]: @@ -3736,7 +3660,7 @@ class Table(Queryable): def value_or_default(self, key: str, value: Any) -> Any: return self._defaults[key] if value is DEFAULT else value - def delete(self, pk_values: Union[list, tuple, str, int, float]) -> "Table": + def delete(self, pk_values: list | tuple | str | float) -> "Table": """ Delete row matching the specified primary key. @@ -3745,7 +3669,7 @@ class Table(Queryable): if not isinstance(pk_values, (list, tuple)): pk_values = [pk_values] self.get(pk_values) - wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in self.pks] + wheres = [f"{quote_identifier(pk_name)} = ?" for pk_name in self.pks] sql = "delete from {} where {wheres}".format( quote_identifier(self.name), wheres=" and ".join(wheres) ) @@ -3755,8 +3679,8 @@ class Table(Queryable): def delete_where( self, - where: Optional[str] = None, - where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + where: str | None = None, + where_args: Sequence | dict[str, Any] | None = None, analyze: bool = False, ) -> "Table": """ @@ -3771,7 +3695,7 @@ class Table(Queryable): """ if not self.exists(): return self - sql = "delete from {}".format(quote_identifier(self.name)) + sql = f"delete from {quote_identifier(self.name)}" if where is not None: sql += " where " + where with self.db.atomic(): @@ -3782,10 +3706,10 @@ class Table(Queryable): def update( self, - pk_values: Union[list, tuple, str, int, float], - updates: Optional[dict] = None, + pk_values: list | tuple | str | float, + updates: dict | None = None, alter: bool = False, - conversions: Optional[dict] = None, + conversions: dict | None = None, ) -> "Table": """ Execute a SQL ``UPDATE`` against the specified row. @@ -3816,7 +3740,7 @@ class Table(Queryable): "{} = {}".format(quote_identifier(key), conversions.get(key, "?")) ) args.append(jsonify_if_needed(value)) - wheres = ["{} = ?".format(quote_identifier(pk_name)) for pk_name in pks] + wheres = [f"{quote_identifier(pk_name)} = ?" for pk_name in pks] args.extend(pk_values) sql = "update {} set {sets} where {wheres}".format( quote_identifier(self.name), @@ -3841,14 +3765,14 @@ class Table(Queryable): def convert( self, - columns: Union[str, List[str]], + columns: str | list[str], fn: Callable, - output: Optional[str] = None, - output_type: Optional[Any] = None, + output: str | None = None, + output_type: Any | None = None, drop: bool = False, multi: bool = False, - where: Optional[str] = None, - where_args: Optional[Union[Sequence, Dict[str, Any]]] = None, + where: str | None = None, + where_args: Sequence | dict[str, Any] | None = None, show_progress: bool = False, ) -> "Table": """ @@ -3905,15 +3829,11 @@ class Table(Queryable): quote_identifier(self.name), sets=", ".join( [ - "{} = {}({})".format( - quote_identifier(output or column), - fn_name, - quote_identifier(column), - ) + f"{quote_identifier(output or column)} = {fn_name}({quote_identifier(column)})" for column in columns ] ), - where=" where {}".format(where) if where is not None else "", + where=f" where {where}" if where is not None else "", ) with self.db.atomic(): self.db.execute(sql, where_args or []) @@ -3926,7 +3846,7 @@ class Table(Queryable): ): # First we execute the function pk_to_values = {} - new_column_types: Dict[str, Set[type]] = {} + new_column_types: dict[str, set[type]] = {} pks = self.pks with progressbar( @@ -3958,15 +3878,17 @@ class Table(Queryable): self.add_column(column_name, column_type) # Run the updates - with progressbar( - length=self.count, silent=not show_progress, label="2: Updating" - ) as bar: - with self.db.atomic(): - for pk, updates in pk_to_values.items(): - self.update(pk, updates) - bar.update(1) - if drop: - self.transform(drop=(column,)) + with ( + progressbar( + length=self.count, silent=not show_progress, label="2: Updating" + ) as bar, + self.db.atomic(), + ): + for pk, updates in pk_to_values.items(): + self.update(pk, updates) + bar.update(1) + if drop: + self.transform(drop=(column,)) def build_insert_queries_and_params( self, @@ -4166,9 +4088,7 @@ class Table(Queryable): ) for col in set_cols ), - wheres=" AND ".join( - "{} = ?".format(quote_identifier(pk)) for pk in pks - ), + wheres=" AND ".join(f"{quote_identifier(pk)} = ?" for pk in pks), ) queries_and_params.append( ( @@ -4201,7 +4121,7 @@ class Table(Queryable): replace, ignore, list_mode=False, - ) -> Optional[sqlite3.Cursor]: + ) -> sqlite3.Cursor | None: queries_and_params = self.build_insert_queries_and_params( extracts, chunk, @@ -4271,21 +4191,21 @@ class Table(Queryable): def insert( self, - record: Dict[str, Any], + record: dict[str, Any], pk=DEFAULT, foreign_keys=DEFAULT, - column_order: Optional[Union[List[str], Default]] = DEFAULT, - not_null: Optional[Union[Iterable[str], Default]] = DEFAULT, - defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT, - hash_id: Optional[Union[str, Default]] = DEFAULT, - hash_id_columns: Optional[Union[Iterable[str], Default]] = DEFAULT, - alter: Optional[Union[bool, Default]] = DEFAULT, - ignore: Optional[Union[bool, Default]] = DEFAULT, - replace: Optional[Union[bool, Default]] = DEFAULT, - extracts: Optional[Union[Dict[str, str], List[str], Default]] = DEFAULT, - conversions: Optional[Union[Dict[str, str], Default]] = DEFAULT, - columns: Optional[Union[Dict[str, Any], Default]] = DEFAULT, - strict: Optional[Union[bool, Default]] = DEFAULT, + column_order: list[str] | Default | None = DEFAULT, + not_null: Iterable[str] | Default | None = DEFAULT, + defaults: dict[str, Any] | Default | None = DEFAULT, + hash_id: str | Default | None = DEFAULT, + hash_id_columns: Iterable[str] | Default | None = DEFAULT, + alter: bool | Default | None = DEFAULT, + ignore: bool | Default | None = DEFAULT, + replace: bool | Default | None = DEFAULT, + extracts: dict[str, str] | list[str] | Default | None = DEFAULT, + conversions: dict[str, str] | Default | None = DEFAULT, + columns: dict[str, Any] | Default | None = DEFAULT, + strict: bool | Default | None = DEFAULT, ) -> "Table": """ Insert a single record into the table. The table will be created with a schema that matches @@ -4340,10 +4260,7 @@ class Table(Queryable): def insert_all( self, - records: Union[ - Iterable[Dict[str, Any]], - Iterable[Sequence[Any]], - ], + records: Iterable[dict[str, Any]] | Iterable[Sequence[Any]], pk=DEFAULT, foreign_keys=DEFAULT, column_order=DEFAULT, @@ -4440,7 +4357,7 @@ class Table(Queryable): # Detect if we're using list-based iteration or dict-based iteration list_mode = False - column_names: List[str] = [] + column_names: list[str] = [] # Fix up any records with square braces in the column names (only for dict mode) # We'll handle this differently for list mode @@ -4460,7 +4377,7 @@ class Table(Queryable): raise ValueError( "When using list-based iteration, the first yielded value must be a list of column name strings" ) - column_names = cast(List[str], list(first_record)) + column_names = cast(list[str], list(first_record)) all_columns = column_names num_columns = len(column_names) # Get the actual first data record @@ -4469,7 +4386,7 @@ class Table(Queryable): except StopIteration: return self # Only headers, no data if not isinstance(first_record, (list, tuple)): - raise ValueError( + raise ValueError( # noqa: TRY004 "After column names list, all subsequent records must also be lists" ) else: @@ -4479,13 +4396,11 @@ class Table(Queryable): first_record = next(records_iter) except StopIteration: return self - first_record = cast(Dict[str, Any], first_record) + first_record = cast(dict[str, Any], first_record) num_columns = len(first_record.keys()) if num_columns > SQLITE_MAX_VARS: - raise ValueError( - "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) - ) + raise ValueError(f"Rows can have a maximum of {SQLITE_MAX_VARS} columns") batch_size = ( 1 if num_columns == 0 @@ -4495,7 +4410,7 @@ class Table(Queryable): self.last_pk = None if truncate and self.exists(): with self.db.atomic(): - self.db.execute("DELETE FROM {};".format(quote_identifier(self.name))) + self.db.execute(f"DELETE FROM {quote_identifier(self.name)};") result = None for chunk in chunks(itertools.chain([first_record], records_iter), batch_size): chunk = list(chunk) @@ -4508,7 +4423,7 @@ class Table(Queryable): chunk_as_dicts = [dict(zip(column_names, row)) for row in chunk] column_types = suggest_column_types(chunk_as_dicts) else: - dict_chunk = cast(List[Dict[str, Any]], chunk) + dict_chunk = cast(list[dict[str, Any]], chunk) column_types = suggest_column_types(dict_chunk) if extracts: for col in extracts: @@ -4535,10 +4450,10 @@ class Table(Queryable): if hash_id: all_columns.insert(0, hash_id) else: - all_columns_set: Set[str] = set() - for record in cast(List[Dict[str, Any]], chunk): + all_columns_set: set[str] = set() + for record in cast(list[dict[str, Any]], chunk): all_columns_set.update(record.keys()) - all_columns = list(sorted(all_columns_set)) + all_columns = sorted(all_columns_set) if hash_id: all_columns.insert(0, hash_id) if deferred_invalid_pk_check is not None: @@ -4553,7 +4468,7 @@ class Table(Queryable): raise invalid_pk_error else: if not list_mode: - for record in cast(List[Dict[str, Any]], chunk): + for record in cast(list[dict[str, Any]], chunk): all_columns += [ column for column in record if column not in all_columns ] @@ -4592,7 +4507,7 @@ class Table(Queryable): zip(column_names, cast(Sequence[Any], first_record)) ) else: - first_record_dict = cast(Dict[str, Any], first_record) + first_record_dict = cast(dict[str, Any], first_record) if hash_id: self.last_pk = hash_record(first_record_dict, hash_id_columns) elif isinstance(pk, str): @@ -4608,7 +4523,7 @@ class Table(Queryable): # columns so we can report its rowid (and pk if not already # known). Falls back to leaving them unset if the conflict # cannot be resolved to a pk lookup (e.g. a UNIQUE column). - key_cols: Optional[List[str]] = None + key_cols: list[str] | None = None if isinstance(pk, str): key_cols = [pk] elif pk: @@ -4625,12 +4540,10 @@ class Table(Queryable): key_values = None if key_values is not None: where = " and ".join( - "{} = ?".format(quote_identifier(c)) for c in key_cols + f"{quote_identifier(c)} = ?" for c in key_cols ) existing = self.db.execute( - "select rowid from {} where {} limit 1".format( - quote_identifier(self.name), where - ), + f"select rowid from {quote_identifier(self.name)} where {where} limit 1", key_values, ).fetchone() if existing is not None: @@ -4650,7 +4563,9 @@ class Table(Queryable): rowid_pk = isinstance(pk, str) and pk.lower() in ROWID_ALIASES if (hash_id or (pk and not rowid_pk)) and self.last_rowid: # Set self.last_pk to the pk(s) for that rowid - row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] + row = next( + iter(self.rows_where("rowid = ?", [self.last_rowid])) + ) if hash_id: self.last_pk = row[hash_id] elif isinstance(pk, str): @@ -4680,7 +4595,7 @@ class Table(Queryable): for p in pk ) else: - first_record_dict = cast(Dict[str, Any], first_record) + first_record_dict = cast(dict[str, Any], first_record) if hash_id: self.last_pk = hash_record(first_record_dict, hash_id_columns) else: @@ -4738,10 +4653,7 @@ class Table(Queryable): def upsert_all( self, - records: Union[ - Iterable[Dict[str, Any]], - Iterable[Sequence[Any]], - ], + records: Iterable[dict[str, Any]] | Iterable[Sequence[Any]], pk=DEFAULT, foreign_keys=DEFAULT, column_order=DEFAULT, @@ -4779,7 +4691,7 @@ class Table(Queryable): strict=strict, ) - def add_missing_columns(self, records: Iterable[Dict[str, Any]]) -> "Table": + def add_missing_columns(self, records: Iterable[dict[str, Any]]) -> "Table": needed_columns = suggest_column_types(records) current_columns = {c.lower() for c in self.columns_dict} for col_name, col_type in needed_columns.items(): @@ -4789,17 +4701,17 @@ class Table(Queryable): def lookup( self, - lookup_values: Dict[str, Any], - extra_values: Optional[Dict[str, Any]] = None, - pk: Optional[str] = "id", - foreign_keys: Optional[ForeignKeysType] = None, - column_order: Optional[List[str]] = None, - not_null: Optional[Iterable[str]] = None, - defaults: Optional[Dict[str, Any]] = None, - extracts: Optional[Union[Dict[str, str], List[str]]] = None, - conversions: Optional[Dict[str, str]] = None, - columns: Optional[Dict[str, Any]] = None, - strict: Optional[bool] = False, + lookup_values: dict[str, Any], + extra_values: dict[str, Any] | None = None, + pk: str | None = "id", + foreign_keys: ForeignKeysType | None = None, + column_order: list[str] | None = None, + not_null: Iterable[str] | None = None, + defaults: dict[str, Any] | None = None, + extracts: dict[str, str] | list[str] | None = None, + conversions: dict[str, str] | None = None, + columns: dict[str, Any] | None = None, + strict: bool | None = False, ): """ Create or populate a lookup table with the specified values. @@ -4825,7 +4737,7 @@ class Table(Queryable): :param strict: Boolean, apply STRICT mode if creating the table. """ if not isinstance(lookup_values, dict): - raise ValueError("lookup_values must be a dictionary") + raise ValueError("lookup_values must be a dictionary") # noqa: TRY004 if pk is None: raise ValueError("pk cannot be None") if extra_values is not None and not isinstance(extra_values, dict): @@ -4843,9 +4755,7 @@ class Table(Queryable): } not in unique_column_sets: self.create_index(lookup_values.keys(), unique=True) # IS rather than = so that null values are matched correctly - wheres = [ - "{} IS ?".format(quote_identifier(column)) for column in lookup_values - ] + wheres = [f"{quote_identifier(column)} IS ?" for column in lookup_values] rows = list( self.rows_where( " and ".join(wheres), [value for _, value in lookup_values.items()] @@ -4885,12 +4795,10 @@ class Table(Queryable): def m2m( self, other_table: Union[str, "Table"], - record_or_iterable: Optional[ - Union[Iterable[Dict[str, Any]], Dict[str, Any]] - ] = None, - pk: Optional[Union[Any, Default]] = DEFAULT, - lookup: Optional[Dict[str, Any]] = None, - m2m_table: Optional[str] = None, + record_or_iterable: Iterable[dict[str, Any]] | dict[str, Any] | None = None, + pk: Any | Default | None = DEFAULT, + lookup: dict[str, Any] | None = None, + m2m_table: str | None = None, alter: bool = False, ): """ @@ -4923,8 +4831,8 @@ class Table(Queryable): raise ValueError("Provide lookup= or record, not both") elif record_or_iterable is None: raise ValueError("Provide lookup= or record, not both") - tables = list(sorted([self.name, other_table.name])) - columns = ["{}_id".format(t) for t in tables] + tables = sorted([self.name, other_table.name]) + columns = [f"{t}_id" for t in tables] if m2m_table is not None: m2m_table_name = m2m_table else: @@ -4934,9 +4842,7 @@ class Table(Queryable): m2m_table_name = candidates[0] elif len(candidates) > 1: raise NoObviousTable( - "No single obvious m2m table for {}, {} - use m2m_table= parameter".format( - self.name, other_table.name - ) + f"No single obvious m2m table for {self.name}, {other_table.name} - use m2m_table= parameter" ) else: # If not, create a new table @@ -4947,7 +4853,7 @@ class Table(Queryable): if isinstance(record_or_iterable, Mapping): records = [record_or_iterable] else: - records = cast(List, record_or_iterable) + records = cast(list, record_or_iterable) # Ensure each record exists in other table for record in records: id = other_table.insert( @@ -4955,8 +4861,8 @@ class Table(Queryable): ).last_pk m2m_table_obj.insert( { - "{}_id".format(other_table.name): id, - "{}_id".format(self.name): our_id, + f"{other_table.name}_id": id, + f"{self.name}_id": our_id, }, replace=True, ) @@ -4964,8 +4870,8 @@ class Table(Queryable): id = other_table.lookup(lookup) m2m_table_obj.insert( { - "{}_id".format(other_table.name): id, - "{}_id".format(self.name): our_id, + f"{other_table.name}_id": id, + f"{self.name}_id": our_id, }, replace=True, ) @@ -5012,21 +4918,19 @@ class Table(Queryable): table_quoted = quote_identifier(table) column_quoted = quote_identifier(column) num_null = db.execute( - "select count(*) from {} where {} is null".format( - table_quoted, column_quoted - ) + f"select count(*) from {table_quoted} where {column_quoted} is null" ).fetchone()[0] num_blank = db.execute( - "select count(*) from {} where {} = ''".format(table_quoted, column_quoted) + f"select count(*) from {table_quoted} where {column_quoted} = ''" ).fetchone()[0] num_distinct = db.execute( - "select count(distinct {}) from {}".format(column_quoted, table_quoted) + f"select count(distinct {column_quoted}) from {table_quoted}" ).fetchone()[0] most_common_results = None least_common_results = None if num_distinct == 1: value = db.execute( - "select {} from {} limit 1".format(column_quoted, table_quoted) + f"select {column_quoted} from {table_quoted} limit 1" ).fetchone()[0] most_common_results = [(truncate(value), total_rows)] elif num_distinct != total_rows: @@ -5038,13 +4942,10 @@ class Table(Queryable): most_common_results = [ (truncate(r[0]), r[1]) for r in db.execute( - "select {}, count(*) from {} group by {} order by count(*) desc, {} limit {}".format( - column_quoted, - table_quoted, - column_quoted, - column_quoted, - common_limit, - ) + f"select {column_quoted}, count(*) " + f"from {table_quoted} group by {column_quoted} " + f"order by count(*) desc, {column_quoted} " + f"limit {common_limit}" ).fetchall() ] most_common_results.sort(key=lambda p: (p[1], p[0]), reverse=True) @@ -5056,13 +4957,10 @@ class Table(Queryable): least_common_results = [ (truncate(r[0]), r[1]) for r in db.execute( - "select {}, count(*) from {} group by {} order by count(*), {} desc limit {}".format( - column_quoted, - table_quoted, - column_quoted, - column_quoted, - common_limit, - ) + f"select {column_quoted}, count(*) " + f"from {table_quoted} group by {column_quoted} " + f"order by count(*), {column_quoted} desc " + f"limit {common_limit}" ).fetchall() ] least_common_results.sort(key=lambda p: (p[1], p[0])) @@ -5179,7 +5077,7 @@ class View(Queryable): """ try: - self.db.execute("DROP VIEW {}".format(quote_identifier(self.name))) + self.db.execute(f"DROP VIEW {quote_identifier(self.name)}") except sqlite3.OperationalError: if not ignore: raise @@ -5192,16 +5090,14 @@ def jsonify_if_needed(value: object) -> object: return json.dumps(value, default=repr, ensure_ascii=False) elif isinstance(value, (datetime.time, datetime.date, datetime.datetime)): return value.isoformat() - elif isinstance(value, datetime.timedelta): - return str(value) - elif isinstance(value, uuid.UUID): + elif isinstance(value, (datetime.timedelta, uuid.UUID)): return str(value) else: return value def resolve_extracts( - extracts: Optional[Union[Dict[str, str], List[str], Tuple[str]]], + extracts: dict[str, str] | list[str] | tuple[str] | None, ) -> dict: if extracts is None: extracts = {} diff --git a/sqlite_utils/hookspecs.py b/sqlite_utils/hookspecs.py index a746619..73d1acc 100644 --- a/sqlite_utils/hookspecs.py +++ b/sqlite_utils/hookspecs.py @@ -1,8 +1,7 @@ import sqlite3 import click -from pluggy import HookimplMarker -from pluggy import HookspecMarker +from pluggy import HookimplMarker, HookspecMarker hookspec = HookspecMarker("sqlite_utils") hookimpl = HookimplMarker("sqlite_utils") diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py index 00d0fa5..69397ba 100644 --- a/sqlite_utils/migrations.py +++ b/sqlite_utils/migrations.py @@ -1,19 +1,28 @@ -from collections.abc import Iterable -from dataclasses import dataclass import datetime -from typing import Callable, cast, TYPE_CHECKING +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol, TypeVar, cast if TYPE_CHECKING: from sqlite_utils.db import Database, Table +class _MigrationFunction(Protocol): + __name__: str + + def __call__(self, db: "Database", /) -> None: ... + + +_MigrationFunctionT = TypeVar("_MigrationFunctionT", bound=_MigrationFunction) + + class Migrations: migrations_table = "_sqlite_migrations" @dataclass class _Migration: name: str - fn: Callable + fn: _MigrationFunction transactional: bool = True @dataclass @@ -32,7 +41,7 @@ class Migrations: def __call__( self, *, name: str | None = None, transactional: bool = True - ) -> Callable: + ) -> Callable[[_MigrationFunctionT], _MigrationFunctionT]: """ :param name: The name to use for this migration - if not provided, the name of the function will be used. @@ -43,13 +52,11 @@ class Migrations: example those that execute ``VACUUM``. """ - def inner(func: Callable) -> Callable: - migration_name = name or getattr(func, "__name__") + def inner(func: _MigrationFunctionT) -> _MigrationFunctionT: + migration_name = name or func.__name__ if any(m.name == migration_name for m in self._migrations): raise ValueError( - "Migration '{}' is already registered in set '{}'".format( - migration_name, self.name - ) + f"Migration '{migration_name}' is already registered in set '{self.name}'" ) self._migrations.append( self._Migration(migration_name, func, transactional) diff --git a/sqlite_utils/plugins.py b/sqlite_utils/plugins.py index 0aff7ff..10815b4 100644 --- a/sqlite_utils/plugins.py +++ b/sqlite_utils/plugins.py @@ -1,7 +1,7 @@ -from typing import Dict, List, Union +import sys import pluggy -import sys + from . import hookspecs pm: pluggy.PluginManager = pluggy.PluginManager("sqlite_utils") @@ -17,13 +17,13 @@ def ensure_plugins_loaded() -> None: _plugins_loaded = True -def get_plugins() -> List[Dict[str, Union[str, List[str]]]]: +def get_plugins() -> list[dict[str, str | list[str]]]: ensure_plugins_loaded() - plugins: List[Dict[str, Union[str, List[str]]]] = [] + plugins: list[dict[str, str | list[str]]] = [] plugin_to_distinfo = dict(pm.list_plugin_distinfo()) for plugin in pm.get_plugins(): hookcallers = pm.get_hookcallers(plugin) or [] - plugin_info: Dict[str, Union[str, List[str]]] = { + plugin_info: dict[str, str | list[str]] = { "name": plugin.__name__, "hooks": [h.name for h in hookcallers], } diff --git a/sqlite_utils/recipes.py b/sqlite_utils/recipes.py index 55b55a4..d28a099 100644 --- a/sqlite_utils/recipes.py +++ b/sqlite_utils/recipes.py @@ -1,9 +1,9 @@ from __future__ import annotations -from typing import Callable, Optional +import json +from collections.abc import Callable from dateutil import parser -import json IGNORE: object = object() SET_NULL: object = object() @@ -13,8 +13,8 @@ def parsedate( value: str, dayfirst: bool = False, yearfirst: bool = False, - errors: Optional[object] = None, -) -> Optional[str]: + errors: object | None = None, +) -> str | None: """ Parse a date and convert it to ISO date format: yyyy-mm-dd \b @@ -44,8 +44,8 @@ def parsedatetime( value: str, dayfirst: bool = False, yearfirst: bool = False, - errors: Optional[object] = None, -) -> Optional[str]: + errors: object | None = None, +) -> str | None: """ Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS \b diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index b39b117..ed5a558 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -9,20 +9,11 @@ import itertools import json import os import sys +from collections.abc import Callable, Generator, Iterable, Iterator from typing import ( + TYPE_CHECKING, Any, BinaryIO, - Callable, - Dict, - Generator, - Iterable, - Iterator, - List, - Optional, - Set, - Tuple, - Type, - TYPE_CHECKING, TypeVar, Union, cast, @@ -33,8 +24,8 @@ import click from . import recipes if TYPE_CHECKING: - import sqlite3 # noqa: F401 - from sqlite3 import dbapi2 # noqa: F401 + import sqlite3 + from sqlite3 import dbapi2 OperationalError = dbapi2.OperationalError else: @@ -44,7 +35,7 @@ else: OperationalError = dbapi2.OperationalError except ImportError: import sqlite3 # noqa: F401 - from sqlite3 import dbapi2 # noqa: F401 + from sqlite3 import dbapi2 OperationalError = dbapi2.OperationalError @@ -61,8 +52,8 @@ SPATIALITE_PATHS = ( ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit() # Type alias for row dictionaries - values can be various SQLite-compatible types -RowValue = Union[None, int, float, str, bytes, bool, List[str]] -Row = Dict[str, RowValue] +RowValue = None | int | float | str | bytes | bool | list[str] +Row = dict[str, RowValue] T = TypeVar("T") @@ -103,7 +94,7 @@ def maximize_csv_field_size_limit() -> None: field_size_limit = int(field_size_limit / 10) -def find_spatialite() -> Optional[str]: +def find_spatialite() -> str | None: """ The ``find_spatialite()`` function searches for the `SpatiaLite `__ SQLite extension in some common places. It returns a string path to the location, or ``None`` if SpatiaLite was not found. @@ -132,9 +123,9 @@ def find_spatialite() -> Optional[str]: def suggest_column_types( - records: Iterable[Dict[str, Any]], -) -> Dict[str, type]: - all_column_types: Dict[str, Set[type]] = {} + records: Iterable[dict[str, Any]], +) -> dict[str, type]: + all_column_types: dict[str, set[type]] = {} for record in records: for key, value in record.items(): all_column_types.setdefault(key, set()).add(type(value)) @@ -142,9 +133,9 @@ def suggest_column_types( def types_for_column_types( - all_column_types: Dict[str, Set[type]], -) -> Dict[str, type]: - column_types: Dict[str, type] = {} + all_column_types: dict[str, set[type]], +) -> dict[str, type]: + column_types: dict[str, type] = {} for key, types in all_column_types.items(): # Ignore null values if at least one other type present: if len(types) > 1: @@ -153,7 +144,7 @@ def types_for_column_types( if {None.__class__} == types: t = str elif len(types) == 1: - t = list(types)[0] + t = next(iter(types)) # But if it's a subclass of list / tuple / dict, use str # instead as we will be storing it as JSON in the table for superclass in (list, tuple, dict): @@ -190,7 +181,7 @@ def column_affinity(column_type: str) -> type: return float -def decode_base64_values(doc: Dict[str, Any]) -> Dict[str, Any]: +def decode_base64_values(doc: dict[str, Any]) -> dict[str, Any]: # Looks for '{"$base64": true..., "encoded": ...}' values and decodes them to_fix = [ k @@ -263,9 +254,9 @@ class RowError(Exception): def _extra_key_strategy( - reader: Iterable[Dict[Optional[str], object]], - ignore_extras: Optional[bool] = False, - extras_key: Optional[str] = None, + reader: Iterable[dict[str | None, object]], + ignore_extras: bool | None = False, + extras_key: str | None = None, ) -> Iterable[Row]: # Logic for handling CSV rows with more values than there are headings for row in reader: @@ -279,9 +270,7 @@ def _extra_key_strategy( yield cast(Row, row) elif not extras_key: extras = row.pop(None) - raise RowError( - "Row {} contained these extra values: {}".format(row, extras) - ) + raise RowError(f"Row {row} contained these extra values: {extras}") else: extras_value = row.pop(None) row_out = cast(Row, row) @@ -291,12 +280,12 @@ def _extra_key_strategy( def rows_from_file( fp: BinaryIO, - format: Optional[Format] = None, - dialect: Optional[Type[csv.Dialect]] = None, - encoding: Optional[str] = None, - ignore_extras: Optional[bool] = False, - extras_key: Optional[str] = None, -) -> Tuple[Iterable[Row], Format]: + format: Format | None = None, + dialect: type[csv.Dialect] | None = None, + encoding: str | None = None, + ignore_extras: bool | None = False, + extras_key: str | None = None, +) -> tuple[Iterable[Row], Format]: """ Load a sequence of dictionaries from a file-like object containing one of four different formats. @@ -363,7 +352,7 @@ def rows_from_file( ) return ( _extra_key_strategy( - cast(Iterable[Dict[Optional[str], object]], rows), + cast(Iterable[dict[str | None, object]], rows), ignore_extras, extras_key, ), @@ -379,7 +368,7 @@ def rows_from_file( raise TypeError( "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO" ) - if first_bytes.startswith(b"[") or first_bytes.startswith(b"{"): + if first_bytes.startswith((b"[", b"{")): # TODO: Detect newline-JSON return rows_from_file(buffered, format=Format.JSON) else: @@ -393,7 +382,7 @@ def rows_from_file( detected_format = Format.TSV if dialect.delimiter == "\t" else Format.CSV return ( _extra_key_strategy( - cast(Iterable[Dict[Optional[str], object]], rows), + cast(Iterable[dict[str | None, object]], rows), ignore_extras, extras_key, ), @@ -425,9 +414,9 @@ class TypeTracker: """ def __init__(self) -> None: - self.trackers: Dict[str, "ValueTracker"] = {} + self.trackers: dict[str, ValueTracker] = {} - def wrap(self, iterator: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]: + def wrap(self, iterator: Iterable[dict[str, Any]]) -> Iterable[dict[str, Any]]: """ Use this to loop through an existing iterator, tracking the column types as part of the iteration. @@ -441,7 +430,7 @@ class TypeTracker: yield row @property - def types(self) -> Dict[str, str]: + def types(self) -> dict[str, str]: """ A dictionary mapping column names to their detected types. This can be passed to the ``db[table_name].transform(types=tracker.types)`` method. @@ -450,17 +439,15 @@ class TypeTracker: class ValueTracker: - couldbe: Dict[str, Callable[[object], bool]] + couldbe: dict[str, Callable[[object], bool]] def __init__(self) -> None: self.couldbe = {key: getattr(self, "test_" + key) for key in self.get_tests()} @classmethod - def get_tests(cls) -> List[str]: + def get_tests(cls) -> list[str]: return [ - key.split("test_")[-1] - for key in cls.__dict__.keys() - if key.startswith("test_") + key.split("test_")[-1] for key in cls.__dict__ if key.startswith("test_") ] def test_integer(self, value: object) -> bool: @@ -492,7 +479,7 @@ class ValueTracker: def evaluate(self, value: object) -> None: if not value or not self.couldbe: return - not_these: List[str] = [] + not_these: list[str] = [] for name, test in self.couldbe.items(): if not test(value): not_these.append(name) @@ -524,14 +511,14 @@ def progressbar(*args: Iterable[T], **kwargs: Any) -> Generator[Any, None, None] def _compile_code( code: str, imports: Iterable[str], variable: str = "value" ) -> Callable[..., Any]: - globals_dict: Dict[str, Any] = {"r": recipes, "recipes": recipes} + globals_dict: dict[str, Any] = {"r": recipes, "recipes": recipes} # Handle imports first so they're available for all approaches for import_ in imports: globals_dict[import_.split(".")[0]] = __import__(import_) # If user defined a convert() function, return that try: - exec(code, globals_dict) + exec(code, globals_dict) # noqa: S102 return cast(Callable[..., object], globals_dict["convert"]) except (AttributeError, SyntaxError, NameError, KeyError, TypeError): pass @@ -542,20 +529,20 @@ def _compile_code( fn = eval(code, globals_dict) if callable(fn): return cast(Callable[..., object], fn) - except Exception: + except Exception: # noqa: BLE001, S110 pass # Try compiling their code as a function instead body_variants = [code] # If single line and no 'return', try adding the return if "\n" not in code and not code.strip().startswith("return "): - body_variants.insert(0, "return {}".format(code)) + body_variants.insert(0, f"return {code}") code_o = None for variant in body_variants: - new_code = ["def fn({}):".format(variable)] + new_code = [f"def fn({variable}):"] for line in variant.split("\n"): - new_code.append(" {}".format(line)) + new_code.append(f" {line}") try: code_o = compile("\n".join(new_code), "", "exec") break @@ -566,7 +553,7 @@ def _compile_code( if code_o is None: raise SyntaxError("Could not compile code") - exec(code_o, globals_dict) + exec(code_o, globals_dict) # noqa: S102 return cast(Callable[..., object], globals_dict["fn"]) @@ -582,7 +569,7 @@ def chunks(sequence: Iterable[T], size: int) -> Iterable[Iterable[T]]: yield itertools.chain([item], itertools.islice(iterator, size - 1)) -def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> str: +def hash_record(record: dict[str, Any], keys: Iterable[str] | None = None) -> str: """ ``record`` should be a Python dictionary. Returns a sha1 hash of the keys and values in that record. @@ -603,7 +590,7 @@ def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> :param record: Record to generate a hash for :param keys: Subset of keys to use for that hash """ - to_hash: Dict[str, Any] = record + to_hash: dict[str, Any] = record if keys is not None: to_hash = {key: record[key] for key in keys} return hashlib.sha1( @@ -613,7 +600,7 @@ def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> ).hexdigest() -def dedupe_keys(keys: Iterable[str]) -> List[str]: +def dedupe_keys(keys: Iterable[str]) -> list[str]: """ Rename duplicates in a list of column names so every name is unique, by appending ``_2``, ``_3``... to later occurrences - skipping any @@ -636,7 +623,7 @@ def dedupe_keys(keys: Iterable[str]) -> List[str]: new_key = key suffix = 2 while new_key in seen or new_key in taken: - new_key = "{}_{}".format(key, suffix) + new_key = f"{key}_{suffix}" suffix += 1 key = new_key seen.add(key) @@ -644,7 +631,7 @@ def dedupe_keys(keys: Iterable[str]) -> List[str]: return result -def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]: +def _flatten(d: dict[str, Any]) -> Generator[tuple[str, Any], None, None]: for key, value in d.items(): if isinstance(value, dict): for key2, value2 in _flatten(value): @@ -653,7 +640,7 @@ def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]: yield key, value -def flatten(row: Dict[str, Any]) -> Dict[str, Any]: +def flatten(row: dict[str, Any]) -> dict[str, Any]: """ Turn a nested dict e.g. ``{"a": {"b": 1}}`` into a flat dict: ``{"a_b": 1}`` diff --git a/tests/conftest.py b/tests/conftest.py index 728db7b..a4eb860 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ +import pytest + from sqlite_utils import Database from sqlite_utils.utils import sqlite3 -import pytest CREATE_TABLES = """ create table Gosh (c1 text, c2 text, c3 text); @@ -55,7 +56,7 @@ def close_all_databases(): for db in databases: try: db.close() - except Exception: + except sqlite3.Error: pass diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index a2ce585..a51bba6 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -1,9 +1,11 @@ -from sqlite_utils.db import Database, ColumnDetails -from sqlite_utils import cli -from click.testing import CliRunner -import pytest import sqlite3 +import pytest +from click.testing import CliRunner + +from sqlite_utils import cli +from sqlite_utils.db import ColumnDetails, Database + @pytest.fixture def db_to_analyze(fresh_db): diff --git a/tests/test_atomic.py b/tests/test_atomic.py index c3fd02f..ba16ca5 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -28,11 +28,13 @@ from sqlite_utils.utils import sqlite3 END; """, [ - "CREATE TRIGGER t_ai AFTER INSERT ON t\n" - " BEGIN\n" - " UPDATE t SET value = 'a;b' WHERE id = new.id;\n" - " INSERT INTO log VALUES ('x;y');\n" - " END;" + ( + "CREATE TRIGGER t_ai AFTER INSERT ON t\n" + " BEGIN\n" + " UPDATE t SET value = 'a;b' WHERE id = new.id;\n" + " INSERT INTO log VALUES ('x;y');\n" + " END;" + ) ], ), ), @@ -49,10 +51,9 @@ def test_atomic_commits(fresh_db): def test_atomic_rolls_back(fresh_db): - with pytest.raises(RuntimeError): - with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") - raise RuntimeError("boom") + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + raise RuntimeError("boom") assert not fresh_db["dogs"].exists() @@ -62,10 +63,9 @@ def test_nested_atomic_rolls_back_to_savepoint(fresh_db): with fresh_db.atomic(): fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}) - with pytest.raises(RuntimeError): - with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) - raise RuntimeError("boom") + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + raise RuntimeError("boom") fresh_db["dogs"].insert({"id": 3, "name": "Marnie"}) assert list(fresh_db["dogs"].rows) == [ @@ -75,20 +75,18 @@ def test_nested_atomic_rolls_back_to_savepoint(fresh_db): def test_outer_atomic_rolls_back_released_savepoint(fresh_db): - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") - with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) - raise RuntimeError("boom") + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + raise RuntimeError("boom") assert not fresh_db["dogs"].exists() def test_executescript_does_not_commit_open_atomic_block(fresh_db): - with pytest.raises(RuntimeError): - with fresh_db.atomic(): - fresh_db.executescript(""" + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db.executescript(""" CREATE TABLE dogs(id INTEGER PRIMARY KEY, name TEXT); CREATE TRIGGER dogs_ai AFTER INSERT ON dogs BEGIN @@ -97,7 +95,7 @@ def test_executescript_does_not_commit_open_atomic_block(fresh_db): -- This comment has a semicolon; INSERT INTO dogs VALUES (1, 'Cleo; the first'); """) - raise RuntimeError("boom") + raise RuntimeError("boom") assert not fresh_db["dogs"].exists() @@ -105,11 +103,10 @@ def test_executescript_does_not_commit_open_atomic_block(fresh_db): def test_transform_does_not_commit_open_atomic_block(fresh_db): fresh_db["dogs"].insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") - with pytest.raises(RuntimeError): - with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"}) - fresh_db["dogs"].transform(rename={"age": "dog_age"}) - raise RuntimeError("boom") + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"}) + fresh_db["dogs"].transform(rename={"age": "dog_age"}) + raise RuntimeError("boom") assert ( fresh_db["dogs"].schema @@ -149,10 +146,9 @@ def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db): foreign_keys={"author_id"}, ) - with pytest.raises(RuntimeError): - with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "full_name"}) - raise RuntimeError("boom") + with pytest.raises(RuntimeError), fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "full_name"}) + raise RuntimeError("boom") assert ( fresh_db["authors"].schema @@ -354,9 +350,11 @@ def test_atomic_preserves_error_from_transaction_destroying_trigger(fresh_db): # with "cannot rollback - no transaction is active" fresh_db.execute("create table t (id integer primary key, v text)") fresh_db.execute(TRIGGER_SQL) - with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): - with fresh_db.atomic(): - fresh_db.execute("insert into t (v) values ('bad')") + with ( + pytest.raises(sqlite3.IntegrityError, match="trigger says no"), + fresh_db.atomic(), + ): + fresh_db.execute("insert into t (v) values ('bad')") assert not fresh_db.conn.in_transaction @@ -367,16 +365,17 @@ def test_nested_atomic_preserves_error_from_transaction_destroying_trigger( # "no such savepoint" from ROLLBACK TO SAVEPOINT fresh_db.execute("create table t (id integer primary key, v text)") fresh_db.execute(TRIGGER_SQL) - with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): - with fresh_db.atomic(): - with fresh_db.atomic(): - fresh_db.execute("insert into t (v) values ('bad')") + with ( + pytest.raises(sqlite3.IntegrityError, match="trigger says no"), + fresh_db.atomic(), + fresh_db.atomic(), + ): + fresh_db.execute("insert into t (v) values ('bad')") assert not fresh_db.conn.in_transaction def test_atomic_preserves_error_from_insert_or_rollback(fresh_db): fresh_db["t"].insert({"id": 1}, pk="id") - with pytest.raises(sqlite3.IntegrityError): - with fresh_db.atomic(): - fresh_db.execute("insert or rollback into t (id) values (1)") + with pytest.raises(sqlite3.IntegrityError), fresh_db.atomic(): + fresh_db.execute("insert or rollback into t (id) values (1)") assert not fresh_db.conn.in_transaction diff --git a/tests/test_cli.py b/tests/test_cli.py index a2135b0..a1e072f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,14 +1,16 @@ -from sqlite_utils import cli, Database -from sqlite_utils.db import Index, ForeignKey -from click.testing import CliRunner -from pathlib import Path -import subprocess -import sqlite3 -import sys import json import os -import pytest +import sqlite3 +import subprocess +import sys import textwrap +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli +from sqlite_utils.db import ForeignKey, Index def write_json(file_path, data): @@ -21,7 +23,7 @@ def _supports_pragma_function_list(): try: db.execute("select * from pragma_function_list()") return True - except Exception: + except sqlite3.DatabaseError: return False finally: db.close() @@ -184,9 +186,9 @@ def test_output_table(db_path, options, expected): db["rows"].insert_all( [ { - "c1": "verb{}".format(i), - "c2": "noun{}".format(i), - "c3": "adjective{}".format(i), + "c1": f"verb{i}", + "c2": f"noun{i}", + "c3": f"adjective{i}", } for i in range(4) ] @@ -678,9 +680,9 @@ def test_optimize(db_path, tables): db[table].insert_all( [ { - "c1": "verb{}".format(i), - "c2": "noun{}".format(i), - "c3": "adjective{}".format(i), + "c1": f"verb{i}", + "c2": f"noun{i}", + "c3": f"adjective{i}", } for i in range(10000) ] @@ -704,9 +706,9 @@ def test_rebuild_fts_fixes_docsize_error(db_path): db = Database(db_path, recursive_triggers=False) records = [ { - "c1": "verb{}".format(i), - "c2": "noun{}".format(i), - "c3": "adjective{}".format(i), + "c1": f"verb{i}", + "c2": f"noun{i}", + "c3": f"adjective{i}", } for i in range(10000) ] @@ -1019,16 +1021,14 @@ def test_query_json_binary(db_path): "data": { "$base64": True, "encoded": ( - ( - "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH" - "8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+" - "DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I" - "/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uI" - "jnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3f" - "iCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8" - "IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984A" - "Mf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9" - ) + "eJzt0c1xAyEMBeC7q1ABHleR3HxNAQrIjmb4M0gelx+RTY7p4N2WBYT0vmufUknH" + "8kq5lz5pqRFXsTOl3pYkE/NJnHXoStruJEVjc0mOCyTqq/ZMJnXEZW1Js2ZvRm5U+" + "DPKk9hRWqjyvTFx0YfzhT6MpGmN2lR1fzxjyfVMD9dFrS+bnkleMpMam/ZGXgrX1I" + "/K+5Au3S/9lNQRh0k4Gq/RUz8GiKfsQm+7JLsJ6fTo5JhVG00ZU76kZZkxePx49uI" + "jnpNoJyYlWUsoaSl/CcVATje/Kxu13RANnrHweaH3V5Jh4jvGyKCnxJLiXPKhmW3f" + "iCnG7Jql7RR3UvFo8jJ4z039dtOkTFmWzL1be9lt8A5II471m6vXy+l0BR/4wAc+8" + "IEPfOADH/jABz7wgQ984AMf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984A" + "Mf+MAHPvCBD3zgAx/4wAc+8IEPfOADH/jABz7wgQ984PuP7xubBoN9" ), }, } @@ -2114,11 +2114,13 @@ _common_other_schema = ( ), ( ["--rename", "name", "name2"], - 'CREATE TABLE "trees" (\n' - ' "id" INTEGER PRIMARY KEY,\n' - ' "address" TEXT,\n' - ' "species_id" INTEGER REFERENCES "species"("id")\n' - ")", + ( + 'CREATE TABLE "trees" (\n' + ' "id" INTEGER PRIMARY KEY,\n' + ' "address" TEXT,\n' + ' "species_id" INTEGER REFERENCES "species"("id")\n' + ")" + ), 'CREATE TABLE "species" (\n "id" INTEGER PRIMARY KEY,\n "species" TEXT\n)', ), ], @@ -2137,9 +2139,9 @@ def test_extract(db_path, args, expected_table_schema, expected_other_schema): assert result.exit_code == 0 schema = db["trees"].schema assert schema == expected_table_schema - other_schema = [t for t in db.tables if t.name not in ("trees", "Gosh", "Gosh2")][ - 0 - ].schema + other_schema = next( + t for t in db.tables if t.name not in ("trees", "Gosh", "Gosh2") + ).schema assert other_schema == expected_other_schema @@ -2431,7 +2433,7 @@ def test_long_csv_column_value(tmpdir): with open(csv_path, "w") as csv_file: long_string = "a" * 131073 csv_file.write("id,text\n") - csv_file.write("1,{}\n".format(long_string)) + csv_file.write(f"1,{long_string}\n") result = CliRunner().invoke( cli.cli, ["insert", db_path, "bigtable", csv_path, "--csv"], @@ -2457,8 +2459,8 @@ def test_import_no_headers(tmpdir, args, tsv): csv_path = str(tmpdir / "test.csv") with open(csv_path, "w") as csv_file: sep = "\t" if tsv else "," - csv_file.write("Cleo{sep}Dog{sep}5\n".format(sep=sep)) - csv_file.write("Tracy{sep}Spider{sep}7\n".format(sep=sep)) + csv_file.write(f"Cleo{sep}Dog{sep}5\n") + csv_file.write(f"Tracy{sep}Spider{sep}7\n") result = CliRunner().invoke( cli.cli, ["insert", db_path, "creatures", csv_path] + args + ["--no-detect-types"], @@ -2690,7 +2692,9 @@ def test_integer_overflow_error(tmpdir): def test_python_dash_m(): "Tool can be run using python -m sqlite_utils" result = subprocess.run( - [sys.executable, "-m", "sqlite_utils", "--help"], stdout=subprocess.PIPE + [sys.executable, "-m", "sqlite_utils", "--help"], + stdout=subprocess.PIPE, + check=False, ) assert result.returncode == 0 assert b"Commands for interacting with a SQLite database" in result.stdout @@ -2830,14 +2834,14 @@ def test_load_extension(entrypoint, should_pass, should_fail): for func in should_pass: result = CliRunner().invoke( cli.cli, - ["memory", "select {}()".format(func), "--load-extension", ext], + ["memory", f"select {func}()", "--load-extension", ext], catch_exceptions=False, ) assert result.exit_code == 0 for func in should_fail: result = CliRunner().invoke( cli.cli, - ["memory", "select {}()".format(func), "--load-extension", ext], + ["memory", f"select {func}()", "--load-extension", ext], catch_exceptions=False, ) assert result.exit_code == 1 diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py index 514f4ac..932269b 100644 --- a/tests/test_cli_bulk.py +++ b/tests/test_cli_bulk.py @@ -1,11 +1,13 @@ -from click.testing import CliRunner -from sqlite_utils import cli, Database import pathlib -import pytest import subprocess import sys import time +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli + @pytest.fixture def test_db_and_path(tmpdir): diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 6c3f5c5..65543b1 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -1,10 +1,12 @@ -from click.testing import CliRunner -from sqlite_utils import cli -import sqlite_utils import json -import textwrap import pathlib +import textwrap + import pytest +from click.testing import CliRunner + +import sqlite_utils +from sqlite_utils import cli @pytest.fixture @@ -50,7 +52,7 @@ def test_convert_code(fresh_db_and_path, code): cli.cli, ["convert", db_path, "t", "text", code], catch_exceptions=False ) assert result.exit_code == 0, result.output - value = list(db["t"].rows)[0]["text"] + value = next(iter(db["t"].rows))["text"] assert value == "Spooktober" @@ -442,7 +444,7 @@ def test_recipe_jsonsplit(tmpdir, delimiter): ) code = "r.jsonsplit(value)" if delimiter: - code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter) + code = f'recipes.jsonsplit(value, delimiter="{delimiter}")' args = ["convert", db_path, "example", "tags", code] result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0, result.output @@ -470,7 +472,7 @@ def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array): ) code = "r.jsonsplit(value)" if type: - code = "recipes.jsonsplit(value, type={})".format(type) + code = f"recipes.jsonsplit(value, type={type})" args = ["convert", db_path, "example", "records", code] result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0, result.output diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index df6f80c..eefb3fa 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -1,11 +1,13 @@ -from sqlite_utils import cli, Database -from click.testing import CliRunner import json -import pytest import subprocess import sys import time +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli + def test_insert_simple(tmpdir): json_path = str(tmpdir / "dog.json") @@ -99,7 +101,7 @@ def test_insert_with_primary_keys(db_path, tmpdir, args, expected_pks): def test_insert_multiple_with_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") - dogs = [{"id": i, "name": "Cleo {}".format(i), "age": i + 3} for i in range(1, 21)] + dogs = [{"id": i, "name": f"Cleo {i}", "age": i + 3} for i in range(1, 21)] with open(json_path, "w") as fp: fp.write(json.dumps(dogs)) result = CliRunner().invoke( @@ -114,7 +116,7 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir): def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") dogs = [ - {"breed": "mixed", "id": i, "name": "Cleo {}".format(i), "age": i + 3} + {"breed": "mixed", "id": i, "name": f"Cleo {i}", "age": i + 3} for i in range(1, 21) ] with open(json_path, "w") as fp: @@ -140,8 +142,7 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): def test_insert_not_null_default(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") dogs = [ - {"id": i, "name": "Cleo {}".format(i), "age": i + 3, "score": 10} - for i in range(1, 21) + {"id": i, "name": f"Cleo {i}", "age": i + 3, "score": 10} for i in range(1, 21) ] with open(json_path, "w") as fp: fp.write(json.dumps(dogs)) @@ -587,7 +588,7 @@ def test_insert_streaming_batch_size_1(db_path): return tries += 1 if tries > 10: - assert False, "Expected {}, got {}".format(expected, rows) + assert False, f"Expected {expected}, got {rows}" time.sleep(tries * 0.1) try_until([{"name": "Azi"}]) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index 2ed4aaa..4fb4fb3 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -1,5 +1,6 @@ -import click import json + +import click import pytest from click.testing import CliRunner @@ -28,7 +29,7 @@ def test_memory_csv(tmpdir, sql_from, use_stdin): fp.write(content) result = CliRunner().invoke( cli.cli, - ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + ["memory", csv_path, f"select * from {sql_from}", "--nl"], input=input, ) assert result.exit_code == 0 @@ -53,7 +54,7 @@ def test_memory_tsv(tmpdir, use_stdin): sql_from = "chickens" result = CliRunner().invoke( cli.cli, - ["memory", path, "select * from {}".format(sql_from)], + ["memory", path, f"select * from {sql_from}"], input=input, ) assert result.exit_code == 0, result.output @@ -79,7 +80,7 @@ def test_memory_json(tmpdir, use_stdin): sql_from = "chickens" result = CliRunner().invoke( cli.cli, - ["memory", path, "select * from {}".format(sql_from)], + ["memory", path, f"select * from {sql_from}"], input=input, ) assert result.exit_code == 0, result.output @@ -105,7 +106,7 @@ def test_memory_json_nl(tmpdir, use_stdin): sql_from = "chickens" result = CliRunner().invoke( cli.cli, - ["memory", path, "select * from {}".format(sql_from)], + ["memory", path, f"select * from {sql_from}"], input=input, ) assert result.exit_code == 0, result.output @@ -135,7 +136,7 @@ def test_memory_csv_encoding(tmpdir, use_stdin): CliRunner() .invoke( cli.cli, - ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], + ["memory", csv_path, f"select * from {sql_from}", "--nl"], input=input, ) .exit_code diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index 0f29e36..f49ef10 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -1,7 +1,8 @@ import pathlib -from click.testing import CliRunner import pytest +from click.testing import CliRunner + import sqlite_utils import sqlite_utils.cli diff --git a/tests/test_column_affinity.py b/tests/test_column_affinity.py index fb8f340..fa23345 100644 --- a/tests/test_column_affinity.py +++ b/tests/test_column_affinity.py @@ -1,4 +1,5 @@ import pytest + from sqlite_utils.utils import column_affinity EXAMPLES = [ @@ -41,5 +42,5 @@ def test_column_affinity(column_def, expected_type): @pytest.mark.parametrize("column_def,expected_type", EXAMPLES) def test_columns_dict(fresh_db, column_def, expected_type): - fresh_db.execute("create table foo (col {})".format(column_def)) + fresh_db.execute(f"create table foo (col {column_def})") assert {"col": expected_type} == fresh_db["foo"].columns_dict diff --git a/tests/test_constructor.py b/tests/test_constructor.py index a619fba..4282969 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -1,8 +1,10 @@ +import sys + +import pytest + from sqlite_utils import Database from sqlite_utils.db import TransactionError from sqlite_utils.utils import sqlite3 -import pytest -import sys def test_recursive_triggers(): diff --git a/tests/test_convert.py b/tests/test_convert.py index ea3fd96..879267a 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -1,6 +1,7 @@ -from sqlite_utils.db import BadMultiValues import pytest +from sqlite_utils.db import BadMultiValues + @pytest.mark.parametrize( "columns,fn,expected", diff --git a/tests/test_create.py b/tests/test_create.py index d281eb4..40746bf 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,26 +1,28 @@ -from sqlite_utils.db import ( - Index, - Database, - DescIndex, - AlterError, - InvalidColumns, - NoObviousTable, - OperationalError, - ForeignKey, - Table, - View, - NoTable, - NoView, -) -from sqlite_utils.utils import hash_record, sqlite3 import collections import datetime import decimal import json import pathlib -import pytest import uuid +import pytest + +from sqlite_utils.db import ( + AlterError, + Database, + DescIndex, + ForeignKey, + Index, + InvalidColumns, + NoObviousTable, + NoTable, + NoView, + OperationalError, + Table, + View, +) +from sqlite_utils.utils import hash_record, sqlite3 + try: import pandas as pd # type: ignore except ImportError: @@ -699,7 +701,7 @@ def test_bulk_insert_more_than_999_values(fresh_db): "num_columns,should_error", ((900, False), (999, False), (1000, True)) ) def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): - record = dict([("c{}".format(i), i) for i in range(num_columns)]) + record = {f"c{i}": i for i in range(num_columns)} if should_error: with pytest.raises(ValueError): fresh_db["big"].insert(record) @@ -718,17 +720,9 @@ def test_columns_not_in_first_record_should_not_cause_batch_to_be_too_large(fres records = [ {"c0": "first record"}, # one column in first record -> batch size = 999 # fill out the batch with 99 records with enough columns to exceed THRESHOLD - *[ - dict([("c{}".format(i), j) for i in range(extra_columns)]) - for j in range(batch_size - 1) - ], + *[{f"c{i}": j for i in range(extra_columns)} for j in range(batch_size - 1)], ] - try: - fresh_db["too_many_columns"].insert_all( - records, alter=True, batch_size=batch_size - ) - except sqlite3.OperationalError: - raise + fresh_db["too_many_columns"].insert_all(records, alter=True, batch_size=batch_size) @pytest.mark.parametrize( @@ -910,7 +904,7 @@ def test_insert_list_nested_unicode(fresh_db): def test_insert_uuid(fresh_db): uuid4 = uuid.uuid4() fresh_db["test"].insert({"uuid": uuid4}) - row = list(fresh_db["test"].rows)[0] + row = next(iter(fresh_db["test"].rows)) assert {"uuid"} == row.keys() assert isinstance(row["uuid"], str) assert row["uuid"] == str(uuid4) @@ -918,16 +912,14 @@ def test_insert_uuid(fresh_db): def test_insert_memoryview(fresh_db): fresh_db["test"].insert({"data": memoryview(b"hello")}) - row = list(fresh_db["test"].rows)[0] + row = next(iter(fresh_db["test"].rows)) assert {"data"} == row.keys() assert isinstance(row["data"], bytes) assert row["data"] == b"hello" def test_insert_thousands_using_generator(fresh_db): - fresh_db["test"].insert_all( - {"i": i, "word": "word_{}".format(i)} for i in range(10000) - ) + fresh_db["test"].insert_all({"i": i, "word": f"word_{i}"} for i in range(10000)) assert [{"name": "i", "type": "INTEGER"}, {"name": "word", "type": "TEXT"}] == [ {"name": col.name, "type": col.type} for col in fresh_db["test"].columns ] @@ -938,7 +930,7 @@ def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fr # https://github.com/simonw/sqlite-utils/issues/139 with pytest.raises(Exception, match="table test has no column named extra"): fresh_db["test"].insert_all( - [{"i": i, "word": "word_{}".format(i)} for i in range(100)] + [{"i": i, "word": f"word_{i}"} for i in range(100)] + [{"i": 101, "extra": "This extra column should cause an exception"}], ) @@ -946,7 +938,7 @@ def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fr def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db): # https://github.com/simonw/sqlite-utils/issues/139 fresh_db["test"].insert_all( - [{"i": i, "word": "word_{}".format(i)} for i in range(100)] + [{"i": i, "word": f"word_{i}"} for i in range(100)] + [{"i": 101, "extra": "Should trigger ALTER"}], alter=True, ) @@ -958,7 +950,7 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): # https://github.com/simonw/sqlite-utils/issues/732 fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") - rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + rows = [{"a": f"x{i}", "b": i} for i in range(num_rows)] with pytest.raises(InvalidColumns) as ex: fresh_db["t"].insert_all(rows, pk="not_a_column") @@ -975,7 +967,7 @@ def test_insert_all_pk_not_in_records_alter_raises(fresh_db, num_rows): # known - a pk column that is in neither the table nor the records # still raises fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") - rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + rows = [{"a": f"x{i}", "b": i} for i in range(num_rows)] with pytest.raises(InvalidColumns) as ex: fresh_db["t"].insert_all(rows, pk="not_a_column", alter=True) @@ -1146,7 +1138,7 @@ def test_insert_hash_id_columns(fresh_db, use_table_factory): insert_kwargs = {} else: dogs = fresh_db["dogs"] - insert_kwargs = dict(hash_id_columns=("name", "twitter")) + insert_kwargs = {"hash_id_columns": ("name", "twitter")} id = dogs.insert( {"name": "Cleo", "twitter": "cleopaws", "age": 5}, @@ -1654,7 +1646,7 @@ def test_upsert_uses_pk_from_prior_insert_655(fresh_db): # Upsert should work without specifying pk again table.upsert({"id": 1, "name": "Alice Updated"}) assert table.count == 1 - assert list(table.rows)[0]["name"] == "Alice Updated" + assert next(iter(table.rows))["name"] == "Alice Updated" def test_upsert_all_uses_pk_from_prior_insert_655(fresh_db): diff --git a/tests/test_create_view.py b/tests/test_create_view.py index 056e246..2b70099 100644 --- a/tests/test_create_view.py +++ b/tests/test_create_view.py @@ -1,4 +1,5 @@ import pytest + from sqlite_utils.utils import OperationalError diff --git a/tests/test_default_value.py b/tests/test_default_value.py index 3724d99..2815180 100644 --- a/tests/test_default_value.py +++ b/tests/test_default_value.py @@ -31,7 +31,7 @@ EXAMPLES = [ @pytest.mark.parametrize("column_def,initial_value,expected_value", EXAMPLES) def test_quote_default_value(fresh_db, column_def, initial_value, expected_value): - fresh_db.execute("create table foo (col {})".format(column_def)) + fresh_db.execute(f"create table foo (col {column_def})") assert initial_value == fresh_db["foo"].columns[0].default_value assert expected_value == fresh_db.quote_default_value( fresh_db["foo"].columns[0].default_value diff --git a/tests/test_delete.py b/tests/test_delete.py index a2d93aa..dffb6bb 100644 --- a/tests/test_delete.py +++ b/tests/test_delete.py @@ -3,7 +3,7 @@ import sqlite_utils def test_delete_rowid_table(fresh_db): table = fresh_db["table"] - table.insert({"foo": 1}).last_pk + table.insert({"foo": 1}) rowid = table.insert({"foo": 2}).last_pk table.delete(rowid) assert [{"foo": 1}] == list(table.rows) diff --git a/tests/test_docs.py b/tests/test_docs.py index f657416..6bc06c8 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,8 +1,10 @@ -from click.testing import CliRunner -from sqlite_utils import cli, recipes -from pathlib import Path -import pytest import re +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from sqlite_utils import cli, recipes docs_path = Path(__file__).parent.parent / "docs" commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+)") @@ -34,7 +36,7 @@ def test_commands_are_documented(documented_commands, command): @pytest.mark.parametrize("command", cli.cli.commands.values()) def test_commands_have_help(command): - assert command.help, "{} is missing its help".format(command) + assert command.help, f"{command} is missing its help" def test_convert_help(): diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py index 28961d2..ad853a5 100644 --- a/tests/test_duplicate.py +++ b/tests/test_duplicate.py @@ -1,7 +1,9 @@ -from sqlite_utils.db import NoTable import datetime + import pytest +from sqlite_utils.db import NoTable + def test_duplicate(fresh_db): # Create table using native Sqlite statement: @@ -12,7 +14,7 @@ def test_duplicate(fresh_db): "bool_col" INTEGER, "datetime_col" TEXT)""") # Insert one row of mock data: - dt = datetime.datetime.now() + dt = datetime.datetime.now(datetime.timezone.utc) data = { "text_col": "Cleo", "real_col": 3.14, diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index 2f6b0db..71a8936 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -1,14 +1,14 @@ -from sqlite_utils import Database -from sqlite_utils import cli -from click.testing import CliRunner import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli def test_enable_counts_specific_table(fresh_db): foo = fresh_db["foo"] assert fresh_db.table_names() == [] for i in range(10): - foo.insert({"name": "item {}".format(i)}) + foo.insert({"name": f"item {i}"}) assert fresh_db.table_names() == ["foo"] assert foo.count == 10 # Now enable counts @@ -44,7 +44,7 @@ def test_enable_counts_specific_table(fresh_db): assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}] # Add some items to test the triggers for i in range(5): - foo.insert({"name": "item {}".format(10 + i)}) + foo.insert({"name": f"item {10 + i}"}) assert foo.count == 15 assert list(fresh_db["_counts"].rows) == [{"count": 15, "table": "foo"}] # Delete some items diff --git a/tests/test_extract.py b/tests/test_extract.py index c73ee7a..915e6e1 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1,19 +1,21 @@ -from sqlite_utils.db import InvalidColumns import itertools + import pytest +from sqlite_utils.db import InvalidColumns + @pytest.mark.parametrize("table", [None, "Species"]) @pytest.mark.parametrize("fk_column", [None, "species"]) def test_extract_single_column(fresh_db, table, fk_column): expected_table = table or "species" - expected_fk = fk_column or "{}_id".format(expected_table) + expected_fk = fk_column or f"{expected_table}_id" iter_species = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) fresh_db["tree"].insert_all( ( { "id": i, - "name": "Tree {}".format(i), + "name": f"Tree {i}", "species": next(iter_species), "end": 1, } @@ -26,13 +28,12 @@ def test_extract_single_column(fresh_db, table, fk_column): 'CREATE TABLE "tree" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT,\n' - ' "{}" INTEGER REFERENCES "{}"("id"),\n'.format(expected_fk, expected_table) + f' "{expected_fk}" INTEGER REFERENCES "{expected_table}"("id"),\n' + ' "end" INTEGER\n' + ")" ) assert fresh_db[expected_table].schema == ( - 'CREATE TABLE "{}" (\n'.format(expected_table) - + ' "id" INTEGER PRIMARY KEY,\n' + f'CREATE TABLE "{expected_table}" (\n' + ' "id" INTEGER PRIMARY KEY,\n' ' "species" TEXT\n' ")" ) @@ -57,7 +58,7 @@ def test_extract_multiple_columns_with_rename(fresh_db): ( { "id": i, - "name": "Tree {}".format(i), + "name": f"Tree {i}", "common_name": next(iter_common), "latin_name": next(iter_latin), } diff --git a/tests/test_extracts.py b/tests/test_extracts.py index 7add79a..9519b91 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -1,13 +1,14 @@ -from sqlite_utils.db import Index import pytest +from sqlite_utils.db import Index + @pytest.mark.parametrize( "kwargs,expected_table", [ - (dict(extracts={"species_id": "Species"}), "Species"), - (dict(extracts=["species_id"]), "species_id"), - (dict(extracts=("species_id",)), "species_id"), + ({"extracts": {"species_id": "Species"}}, "Species"), + ({"extracts": ["species_id"]}, "species_id"), + ({"extracts": ("species_id",)}, "species_id"), ], ) @pytest.mark.parametrize("use_table_factory", [True, False]) @@ -30,15 +31,11 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): # Should now have two tables: Trees and Species assert {expected_table, "Trees"} == set(fresh_db.table_names()) assert ( - 'CREATE TABLE "{}" (\n "id" INTEGER PRIMARY KEY,\n "value" TEXT\n)'.format( - expected_table - ) + f'CREATE TABLE "{expected_table}" (\n "id" INTEGER PRIMARY KEY,\n "value" TEXT\n)' == fresh_db[expected_table].schema ) assert ( - 'CREATE TABLE "Trees" (\n "id" INTEGER,\n "species_id" INTEGER REFERENCES "{}"("id")\n)'.format( - expected_table - ) + f'CREATE TABLE "Trees" (\n "id" INTEGER,\n "species_id" INTEGER REFERENCES "{expected_table}"("id")\n)' == fresh_db["Trees"].schema ) # Should have a foreign key reference @@ -51,7 +48,7 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): assert [ Index( seq=0, - name="idx_{}_value".format(expected_table), + name=f"idx_{expected_table}_value", unique=1, origin="c", partial=0, diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index b37d374..45f4f35 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -1,6 +1,7 @@ """Tests for compound (multi-column) foreign keys - issue #594.""" import pytest + from sqlite_utils import Database from sqlite_utils.db import AlterError, ForeignKey from sqlite_utils.utils import sqlite3 @@ -64,7 +65,7 @@ def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): fresh_db["books"].add_foreign_key("author_id", "authors", "id") fk = fresh_db["books"].foreign_keys[0] with pytest.raises(TypeError): - table, column, other_table, other_column = fk + _table, _column, _other_table, _other_column = fk with pytest.raises(TypeError): fk[0] diff --git a/tests/test_fts.py b/tests/test_fts.py index 64ec645..50c1770 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -1,7 +1,9 @@ +from unittest.mock import ANY + import pytest + from sqlite_utils import Database from sqlite_utils.utils import sqlite3 -from unittest.mock import ANY search_records = [ { @@ -103,9 +105,10 @@ def test_search_limit_offset(fresh_db): table.enable_fts(["text", "country"], fts_version="FTS4") assert len(list(table.search("are"))) == 2 assert len(list(table.search("are", limit=1))) == 1 - assert list(table.search("are", limit=1, order_by="rowid"))[0]["rowid"] == 1 + assert next(iter(table.search("are", limit=1, order_by="rowid")))["rowid"] == 1 assert ( - list(table.search("are", limit=1, offset=1, order_by="rowid"))[0]["rowid"] == 2 + next(iter(table.search("are", limit=1, offset=1, order_by="rowid")))["rowid"] + == 2 ) @@ -223,20 +226,20 @@ def test_populate_fts_escape_table_names(fresh_db): @pytest.mark.parametrize("fts_version", ("4", "5")) def test_fts_tokenize(fresh_db, fts_version): - table_name = "searchable_{}".format(fts_version) + table_name = f"searchable_{fts_version}" table = fresh_db[table_name] table.insert_all(search_records) # Test without porter stemming table.enable_fts( ["text", "country"], - fts_version="FTS{}".format(fts_version), + fts_version=f"FTS{fts_version}", ) assert [] == list(table.search("bite")) # Test WITH stemming table.disable_fts() table.enable_fts( ["text", "country"], - fts_version="FTS{}".format(fts_version), + fts_version=f"FTS{fts_version}", tokenize="porter", ) rows = list(table.search("bite", order_by="rowid")) @@ -251,10 +254,10 @@ def test_fts_tokenize(fresh_db, fts_version): def test_optimize_fts(fresh_db): for fts_version in ("4", "5"): - table_name = "searchable_{}".format(fts_version) + table_name = f"searchable_{fts_version}" table = fresh_db[table_name] table.insert_all(search_records) - table.enable_fts(["text", "country"], fts_version="FTS{}".format(fts_version)) + table.enable_fts(["text", "country"], fts_version=f"FTS{fts_version}") # You can call optimize successfully against the tables OR their _fts equivalents: for table_name in ( "searchable_4", @@ -310,12 +313,12 @@ def test_disable_fts(fresh_db, create_triggers): expected_triggers = {"searchable_ai", "searchable_ad", "searchable_au"} else: expected_triggers = set() - assert expected_triggers == set( + assert expected_triggers == { r[0] for r in fresh_db.execute( "select name from sqlite_master where type = 'trigger'" ).fetchall() - ) + } # Now run .disable_fts() and confirm it worked table.disable_fts() assert ( @@ -424,7 +427,7 @@ def test_enable_fts_replace(kwargs): db["books"].enable_fts(**kwargs, replace=True) # Check that the new configuration is correct if should_have_changed_columns: - assert db["books_fts"].columns_dict.keys() == set(["title"]) + assert db["books_fts"].columns_dict.keys() == {"title"} if "create_triggers" in kwargs: assert db["books"].triggers if "fts_version" in kwargs: @@ -741,6 +744,7 @@ def test_enable_fts_cli_on_view_errors(tmpdir): db.create_view("v", "select * from t") db.close() from click.testing import CliRunner + from sqlite_utils import cli as cli_module result = CliRunner().invoke(cli_module.cli, ["enable-fts", db_path, "v", "text"]) diff --git a/tests/test_get.py b/tests/test_get.py index 63c4a2e..3cdaed8 100644 --- a/tests/test_get.py +++ b/tests/test_get.py @@ -1,4 +1,5 @@ import pytest + from sqlite_utils.db import NotFoundError diff --git a/tests/test_gis.py b/tests/test_gis.py index f39554e..8b41d22 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -1,7 +1,8 @@ import json -import pytest +import pytest from click.testing import CliRunner + from sqlite_utils.cli import cli from sqlite_utils.db import Database from sqlite_utils.utils import find_spatialite, sqlite3 @@ -104,7 +105,7 @@ def test_query_load_extension(use_spatialite_shortcut): [ ":memory:", "select spatialite_version()", - "--load-extension={}".format(load_extension), + f"--load-extension={load_extension}", ], ) assert result.exit_code == 0, result.stdout diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index f12f865..ab652c7 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -1,5 +1,6 @@ -from hypothesis import given import hypothesis.strategies as st +from hypothesis import given + import sqlite_utils diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 88e49a8..1724d2d 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -1,10 +1,12 @@ -from sqlite_utils import cli, Database -from click.testing import CliRunner import os import pathlib -import pytest import sys +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli + @pytest.mark.parametrize("silent", (False, True)) @pytest.mark.parametrize( @@ -44,7 +46,7 @@ def test_insert_files(silent, pk_args, expected_pks): ) cols = [] for coltype in coltypes: - cols += ["-c", "{}:{}".format(coltype, coltype)] + cols += ["-c", f"{coltype}:{coltype}"] result = runner.invoke( cli.cli, ["insert-files", db_path, "files", str(tmpdir)] @@ -142,7 +144,7 @@ def test_insert_files_stdin(use_text, encoding, input, expected): ) assert result.exit_code == 0, result.stdout db = Database(db_path) - row = list(db["files"].rows)[0] + row = next(iter(db["files"].rows)) key = "content" if use_text: key = "content_text" @@ -167,5 +169,5 @@ def test_insert_files_bad_text_encoding_error(): ) assert result.exit_code == 1, result.output assert result.output.strip().startswith( - "Error: Could not read file '{}' as text".format(str(latin.resolve())) + f"Error: Could not read file '{latin.resolve()!s}' as text" ) diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 8b6765d..385c052 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,6 +1,7 @@ -from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn import pytest +from sqlite_utils.db import Database, Index, View, XIndex, XIndexColumn + def _check_supports_strict(): """Check if SQLite supports strict tables without leaking the database.""" @@ -57,8 +58,8 @@ def test_detect_fts_similar_tables(fresh_db, reverse_order): fresh_db[table2].insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" ) - assert fresh_db[table1].detect_fts() == "{}_fts".format(table1) - assert fresh_db[table2].detect_fts() == "{}_fts".format(table2) + assert fresh_db[table1].detect_fts() == f"{table1}_fts" + assert fresh_db[table2].detect_fts() == f"{table2}_fts" def test_tables(existing_db): diff --git a/tests/test_list_mode.py b/tests/test_list_mode.py index 746c9c1..646098e 100644 --- a/tests/test_list_mode.py +++ b/tests/test_list_mode.py @@ -3,6 +3,7 @@ Tests for list-based iteration in insert_all and upsert_all """ import pytest + from sqlite_utils import Database diff --git a/tests/test_lookup.py b/tests/test_lookup.py index da4f18b..c93d1ed 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -1,6 +1,7 @@ -from sqlite_utils.db import Index import pytest +from sqlite_utils.db import Index + def test_lookup_new_table(fresh_db): species = fresh_db["species"] diff --git a/tests/test_m2m.py b/tests/test_m2m.py index d613bb9..4fca918 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -1,6 +1,7 @@ -from sqlite_utils.db import ForeignKey, NoObviousTable import pytest +from sqlite_utils.db import ForeignKey, NoObviousTable + def test_insert_m2m_single(fresh_db): dogs = fresh_db["dogs"] @@ -65,8 +66,7 @@ def test_insert_m2m_iterable(fresh_db): iterable_records = ({"id": 1, "name": "Phineas"}, {"id": 2, "name": "Ferb"}) def iterable(): - for record in iterable_records: - yield record + yield from iterable_records platypuses = fresh_db["platypuses"] platypuses.insert({"id": 1, "name": "Perry"}, pk="id").m2m( diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 04185fc..3f3dfea 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -1,4 +1,5 @@ import pytest + import sqlite_utils from sqlite_utils import Migrations @@ -154,10 +155,9 @@ def test_non_transactional_migration_allows_vacuum(tmpdir): def test_apply_composes_inside_outer_transaction(migrations): db = sqlite_utils.Database(memory=True) - with pytest.raises(ZeroDivisionError): - with db.atomic(): - migrations.apply(db) - raise ZeroDivisionError + with pytest.raises(ZeroDivisionError), db.atomic(): + migrations.apply(db) + raise ZeroDivisionError # The outer transaction rolled back, taking the migrations with it assert db.table_names() == [] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index c793e32..ef202be 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,9 +1,12 @@ -from click.testing import CliRunner -import click import importlib -import pytest +import sqlite3 import sys -from sqlite_utils import cli, Database, hookimpl, plugins + +import click +import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli, hookimpl, plugins def _supports_pragma_function_list(): @@ -11,7 +14,7 @@ def _supports_pragma_function_list(): try: db.execute("select * from pragma_function_list()") return True - except Exception: + except sqlite3.DatabaseError: return False finally: db.close() diff --git a/tests/test_query.py b/tests/test_query.py index 06847da..9d79755 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1,6 +1,7 @@ -import pytest import types +import pytest + from sqlite_utils.utils import sqlite3 diff --git a/tests/test_recipes.py b/tests/test_recipes.py index a7c7ef7..c6222a3 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -1,7 +1,9 @@ +import json + +import pytest + from sqlite_utils import recipes from sqlite_utils.utils import sqlite3 -import json -import pytest @pytest.fixture diff --git a/tests/test_recreate.py b/tests/test_recreate.py index bce53d5..09e237e 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -1,8 +1,10 @@ -from sqlite_utils import Database -import sqlite3 import pathlib +import sqlite3 + import pytest +from sqlite_utils import Database + def test_recreate_ignored_for_in_memory(): # None of these should raise an exception: diff --git a/tests/test_rows_from_file.py b/tests/test_rows_from_file.py index a19fed6..8c080d6 100644 --- a/tests/test_rows_from_file.py +++ b/tests/test_rows_from_file.py @@ -1,7 +1,9 @@ -from sqlite_utils.utils import rows_from_file, Format, RowError from io import BytesIO, StringIO + import pytest +from sqlite_utils.utils import Format, RowError, rows_from_file + @pytest.mark.parametrize( "input,expected_format", @@ -29,7 +31,7 @@ def test_rows_from_file_detect_format(input, expected_format): ) def test_rows_from_file_extra_fields_strategies(ignore_extras, extras_key, expected): try: - rows, format = rows_from_file( + rows, _format = rows_from_file( BytesIO(b"id,name\r\n1,Cleo,oops"), format=Format.CSV, ignore_extras=ignore_extras, diff --git a/tests/test_sniff.py b/tests/test_sniff.py index 4bbdb66..7149978 100644 --- a/tests/test_sniff.py +++ b/tests/test_sniff.py @@ -1,7 +1,9 @@ -from sqlite_utils import cli, Database -from click.testing import CliRunner import pathlib + import pytest +from click.testing import CliRunner + +from sqlite_utils import Database, cli sniff_dir = pathlib.Path(__file__).parent / "sniff" diff --git a/tests/test_suggest_column_types.py b/tests/test_suggest_column_types.py index e36c58f..d4f28d3 100644 --- a/tests/test_suggest_column_types.py +++ b/tests/test_suggest_column_types.py @@ -1,5 +1,7 @@ -import pytest from collections import OrderedDict + +import pytest + from sqlite_utils.utils import suggest_column_types diff --git a/tests/test_tracer.py b/tests/test_tracer.py index d14697d..ec81f2f 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -53,16 +53,18 @@ def test_with_tracer(): assert len(collected) == 4 assert collected == [ ( - "SELECT name FROM sqlite_master\n" - " WHERE rootpage = 0\n" - " AND (\n" - " sql LIKE :like\n" - " OR sql LIKE :like2\n" - " OR (\n" - " tbl_name = :table\n" - " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" - " )\n" - " )", + ( + "SELECT name FROM sqlite_master\n" + " WHERE rootpage = 0\n" + " AND (\n" + " sql LIKE :like\n" + " OR sql LIKE :like2\n" + " OR (\n" + " tbl_name = :table\n" + " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" + " )\n" + " )" + ), { "like": "%VIRTUAL TABLE%USING FTS%content=[dogs]%", "like2": '%VIRTUAL TABLE%USING FTS%content="dogs"%', @@ -72,21 +74,23 @@ def test_with_tracer(): ("select name from sqlite_master where type = 'view'", None), ("select sql from sqlite_master where name = ?", ("dogs_fts",)), ( - 'with "original" as (\n' - " select\n" - " rowid,\n" - " *\n" - ' from "dogs"\n' - ")\n" - "select\n" - ' "original".*\n' - "from\n" - ' "original"\n' - ' join "dogs_fts" on "original".rowid = "dogs_fts".rowid\n' - "where\n" - ' "dogs_fts" match :query\n' - "order by\n" - ' "dogs_fts".rank', + ( + 'with "original" as (\n' + " select\n" + " rowid,\n" + " *\n" + ' from "dogs"\n' + ")\n" + "select\n" + ' "original".*\n' + "from\n" + ' "original"\n' + ' join "dogs_fts" on "original".rowid = "dogs_fts".rowid\n' + "where\n" + ' "dogs_fts" match :query\n' + "order by\n" + ' "dogs_fts".rank' + ), {"query": "Cleopaws"}, ), ] diff --git a/tests/test_transform.py b/tests/test_transform.py index 362f1ca..b9ee126 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1,8 +1,9 @@ import sqlite3 +import pytest + from sqlite_utils.db import ForeignKey, TransactionError, TransformError from sqlite_utils.utils import OperationalError -import pytest @pytest.mark.parametrize( @@ -113,7 +114,7 @@ def test_transform_sql_table_with_primary_key( if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") - sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) + sql = dogs.transform_sql(**{**params, "tmp_suffix": "suffix"}) assert sql == expected_sql # Check that .transform() runs without exceptions: with fresh_db.tracer(tracer): @@ -186,7 +187,7 @@ def test_transform_sql_table_with_no_primary_key( if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) - sql = dogs.transform_sql(**{**params, **{"tmp_suffix": "suffix"}}) + sql = dogs.transform_sql(**{**params, "tmp_suffix": "suffix"}) assert sql == expected_sql # Check that .transform() runs without exceptions: with fresh_db.tracer(tracer): @@ -476,23 +477,22 @@ def test_transform_in_transaction_refuses_destructive_on_delete(fresh_db, on_del # keys inside an open transaction would fire those actions when the old # table is dropped - transform() should refuse instead fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db.executescript(""" + fresh_db.executescript(f""" CREATE TABLE authors (id INTEGER PRIMARY KEY, name TEXT); CREATE TABLE books ( id INTEGER PRIMARY KEY, title TEXT, - author_id INTEGER REFERENCES authors(id) ON DELETE {} + author_id INTEGER REFERENCES authors(id) ON DELETE {on_delete} ); - """.format(on_delete)) + """) fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) previous_schema = fresh_db["authors"].schema - with fresh_db.atomic(): - with pytest.raises(TransactionError) as excinfo: - fresh_db["authors"].transform(rename={"name": "author_name"}) + with fresh_db.atomic(), pytest.raises(TransactionError) as excinfo: + fresh_db["authors"].transform(rename={"name": "author_name"}) message = str(excinfo.value) assert "books" in message - assert "ON DELETE {}".format(on_delete.upper()) in message + assert f"ON DELETE {on_delete.upper()}" in message # Nothing should have changed assert fresh_db["authors"].schema == previous_schema assert list(fresh_db["books"].rows) == [ @@ -518,9 +518,8 @@ def test_transform_in_transaction_refuses_self_referential_cascade(fresh_db): {"id": 2, "name": "Science Fiction", "parent_id": 1}, ] ) - with fresh_db.atomic(): - with pytest.raises(TransactionError) as excinfo: - fresh_db["categories"].transform(rename={"name": "title"}) + with fresh_db.atomic(), pytest.raises(TransactionError) as excinfo: + fresh_db["categories"].transform(rename={"name": "title"}) assert "categories" in str(excinfo.value) assert fresh_db["categories"].count == 2 @@ -715,15 +714,15 @@ def test_transform_preserves_rowids(fresh_db, table_type): # Now delete and insert a row to mix up the `rowid` sequence fresh_db["places"].delete_where("id = ?", ["2"]) fresh_db["places"].insert({"id": "4", "name": "London", "country": "UK"}) - previous_rows = list( + previous_rows = [ tuple(row) for row in fresh_db.execute("select rowid, id, name from places") - ) + ] # Transform it fresh_db["places"].transform(column_order=("country", "name")) # Should be the same - next_rows = list( + next_rows = [ tuple(row) for row in fresh_db.execute("select rowid, id, name from places") - ) + ] assert previous_rows == next_rows diff --git a/tests/test_update.py b/tests/test_update.py index 03bec11..e6ae7d8 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -43,7 +43,7 @@ def test_update_compound_pk_table(fresh_db): ) def test_update_invalid_pk(fresh_db, pk, update_pk): table = fresh_db["table"] - table.insert({"id1": 5, "id2": 3, "v": 1}, pk=pk).last_pk + table.insert({"id1": 5, "id2": 3, "v": 1}, pk=pk) with pytest.raises(NotFoundError): table.update(update_pk, {"v": 2}) diff --git a/tests/test_upsert.py b/tests/test_upsert.py index a782b26..0eaae9b 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -1,7 +1,8 @@ -from sqlite_utils.db import PrimaryKeyRequired -from sqlite_utils import Database import pytest +from sqlite_utils import Database +from sqlite_utils.db import PrimaryKeyRequired + @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_upsert(use_old_upsert): diff --git a/tests/test_utils.py b/tests/test_utils.py index 3de5e94..360a443 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,8 +1,10 @@ -from sqlite_utils import utils import csv import io + import pytest +from sqlite_utils import utils + @pytest.mark.parametrize( "input,expected,should_be_is", @@ -57,7 +59,7 @@ def test_maximize_csv_field_size_limit(): # Reset to default in case other tests have changed it csv.field_size_limit(utils.ORIGINAL_CSV_FIELD_SIZE_LIMIT) long_value = "a" * 131073 - long_csv = "id,text\n1,{}".format(long_value) + long_csv = f"id,text\n1,{long_value}" fp = io.BytesIO(long_csv.encode("utf-8")) # Using rows_from_file should error with pytest.raises(csv.Error): diff --git a/tests/test_wal.py b/tests/test_wal.py index 2ddcf54..35318f8 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -1,4 +1,5 @@ import pytest + from sqlite_utils import Database from sqlite_utils.db import TransactionError @@ -11,7 +12,7 @@ def db_path_tmpdir(tmpdir): def test_enable_disable_wal(db_path_tmpdir): - db, path, tmpdir = db_path_tmpdir + db, _path, tmpdir = db_path_tmpdir assert len(tmpdir.listdir()) == 1 assert "delete" == db.journal_mode assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()] @@ -25,12 +26,11 @@ def test_enable_disable_wal(db_path_tmpdir): def test_enable_wal_inside_transaction_raises(db_path_tmpdir): - db, path, tmpdir = db_path_tmpdir + db, _path, _tmpdir = db_path_tmpdir db["test"].insert({"id": 1}, pk="id") - with pytest.raises(TransactionError): - with db.atomic(): - db["test"].insert({"id": 2}, pk="id") - db.enable_wal() + with pytest.raises(TransactionError), db.atomic(): + db["test"].insert({"id": 2}, pk="id") + db.enable_wal() # The atomic() block must have rolled back cleanly and the # journal mode must be unchanged assert db.journal_mode == "delete" @@ -38,19 +38,18 @@ def test_enable_wal_inside_transaction_raises(db_path_tmpdir): def test_disable_wal_inside_transaction_raises(db_path_tmpdir): - db, path, tmpdir = db_path_tmpdir + db, _path, _tmpdir = db_path_tmpdir db.enable_wal() db["test"].insert({"id": 1}, pk="id") - with pytest.raises(TransactionError): - with db.atomic(): - db["test"].insert({"id": 2}, pk="id") - db.disable_wal() + with pytest.raises(TransactionError), db.atomic(): + db["test"].insert({"id": 2}, pk="id") + db.disable_wal() assert db.journal_mode == "wal" assert [r["id"] for r in db["test"].rows] == [1] def test_ensure_autocommit_on(db_path_tmpdir): - db, path, tmpdir = db_path_tmpdir + db, _path, _tmpdir = db_path_tmpdir previous_isolation_level = db.conn.isolation_level assert previous_isolation_level is not None with db.ensure_autocommit_on(): @@ -63,7 +62,7 @@ def test_ensure_autocommit_on(db_path_tmpdir): def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): # Calling enable_wal() when WAL is already enabled is a no-op, # so it is fine inside a transaction - db, path, tmpdir = db_path_tmpdir + db, _path, _tmpdir = db_path_tmpdir db.enable_wal() with db.atomic(): db["test"].insert({"id": 1}, pk="id") @@ -75,13 +74,12 @@ def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): # Setting isolation_level commits any pending transaction as a side # effect, silently breaking the caller's rollback guarantee - so # entering autocommit mode with a transaction open is an error - db, path, tmpdir = db_path_tmpdir + db, _path, _tmpdir = db_path_tmpdir db["test"].insert({"id": 1}, pk="id") db.begin() db.execute("insert into test (id) values (2)") - with pytest.raises(TransactionError): - with db.ensure_autocommit_on(): - pass + with pytest.raises(TransactionError), db.ensure_autocommit_on(): + pass # The transaction is still open and can still be rolled back assert db.conn.in_transaction db.rollback() From c621499ed1e3572989087c19a2f9d13bfff45021 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 14:53:46 -0700 Subject: [PATCH 085/110] codespell should check sqlite_utils as well It did in CI but did not in the Justfile --- Justfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Justfile b/Justfile index 5caa120..be41523 100644 --- a/Justfile +++ b/Justfile @@ -16,6 +16,7 @@ uv run ty check sqlite_utils uv run cog --check README.md docs/*.rst uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt + uv run --group docs codespell sqlite_utils --ignore-words docs/codespell-ignore-words.txt # Rebuild docs with cog @cog: From a7b734946f95341f52d866321c183e3532ac0aad Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 25 Jul 2026 21:50:59 -0700 Subject: [PATCH 086/110] Changelog entry for 3.39.1 Refs #815 Copied from e1d55de8f84a486f5f1178f377e80d23a08a404b --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c868f4..a853aa2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _v3_39_1: + +3.39.1 (2026-07-25) +------------------- + +- Fixed a bug where ``table.delete_where()`` left the connection in an open transaction, causing deleted rows to be silently restored when the connection was closed. (:issue:`815`) + .. _v4_1_1: 4.1.1 (2026-07-12) From 6a456830ca33eb5edaa634a9b0febe5d71bea2be Mon Sep 17 00:00:00 2001 From: ikatyal2110 <134458944+ikatyal2110@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:52:04 -0500 Subject: [PATCH 087/110] Fix _decode_default_value to unescape doubled single quotes in string defaults (#811) * Fix _decode_default_value to unescape doubled single quotes in string defaults SQLite stores string defaults with single quotes doubled (e.g. DEFAULT 'O''Brien' is stored as the literal "'O''Brien'" in sqlite_master). The previous code stripped the outer quotes with value[1:-1] but never converted '' back to ', so default_values returned the raw escaped form instead of the true string value. * Test for doubled single quotes in string defaults --- sqlite_utils/db.py | 4 ++-- tests/test_introspect.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9a00123..713b110 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -5108,8 +5108,8 @@ def resolve_extracts( def _decode_default_value(value: str) -> object: if value.startswith("'") and value.endswith("'"): - # It's a string - return value[1:-1] + # It's a string; unescape doubled single quotes + return value[1:-1].replace("''", "'") if value.isdigit(): # It's an integer return int(value) diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 385c052..b7e8fc2 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -312,6 +312,7 @@ def test_table_strict(fresh_db, create_table, expected_strict): 1, 1.3, "foo", + "O'Brien", True, b"binary", ), @@ -324,6 +325,16 @@ def test_table_default_values(fresh_db, value): assert default_values == {"value": value} +def test_table_default_values_escaped_quotes(fresh_db): + # SQLite stores string defaults with single quotes doubled, so + # introspection needs to unescape them again + fresh_db.execute( + "create table t (id integer primary key, name text default 'O''Brien')" + ) + assert "default 'O''Brien'" in fresh_db["t"].schema + assert fresh_db["t"].default_values == {"name": "O'Brien"} + + def test_pks_use_primary_key_declaration_order(fresh_db): # PRIMARY KEY (a, b) declared against columns stored in order (b, a) - # pks must follow the declaration order, which is what SQLite uses to From f726ea4a65c3ce9eaff67057908ee8f2fe7f81e0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Aug 2026 20:48:11 -0700 Subject: [PATCH 088/110] transform() now works for tables referenced by views (#832) Closes #831 --- docs/changelog.rst | 8 ++ docs/cli.rst | 4 + docs/python-api.rst | 11 +++ sqlite_utils/db.py | 21 +++++- tests/test_transform.py | 162 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 204 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a853aa2..4ae53a6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _unreleased: + +Unreleased +---------- + +- ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) + .. _v3_39_1: 3.39.1 (2026-07-25) @@ -18,6 +25,7 @@ - ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`) - The :ref:`CLI ` and :ref:`Python API ` documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`) + .. _v4_1: 4.1 (2026-07-11) diff --git a/docs/cli.rst b/docs/cli.rst index 2e506dd..cf241aa 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2288,7 +2288,11 @@ If you want to see the SQL that will be executed to make the change without actu INSERT INTO "roadside_attractions_new_4033a60276b9" ("longitude", "latitude", "id", "name") SELECT "longitude", "latitude", "pk", "name" FROM "roadside_attractions"; DROP TABLE "roadside_attractions"; + PRAGMA legacy_alter_table=ON; ALTER TABLE "roadside_attractions_new_4033a60276b9" RENAME TO "roadside_attractions"; + PRAGMA legacy_alter_table=OFF; + +Tables that are referenced by views can be transformed - the view definitions are left unchanged, see :ref:`python_api_transform_views` for details. .. note:: In Python: :ref:`table.transform() ` CLI reference: :ref:`sqlite-utils transform ` diff --git a/docs/python-api.rst b/docs/python-api.rst index 43b734d..53a47dd 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1986,6 +1986,17 @@ A bare column name drops any foreign key that column participates in, including Renaming a column with ``rename=`` updates any foreign keys that use it, and dropping a column with ``drop=`` also drops any foreign keys it participates in - for a compound foreign key this removes the whole constraint. +.. _python_api_transform_views: + +Tables referenced by views +-------------------------- + +Tables that are referenced by views can be safely transformed - the view definitions are left byte-for-byte unchanged, and views continue to read from the live table even when ``keep_table=`` is used to keep a copy of the original around. + +A view that references a column which the transform renamed or dropped will remain defined but will raise a ``no such column`` error when it is next queried. This is inherent to SQLite views, whose SQL is stored as text - if you rename or drop columns that a view depends on you should update that view definition yourself. + +To achieve this, the SQL produced by ``transform_sql()`` turns on ``PRAGMA legacy_alter_table`` for its ``ALTER TABLE ... RENAME TO`` statements, then restores the pragma to the value it had when the SQL was generated - without this, SQLite would attempt to rewrite references to the renamed table in every view definition, which fails when a view references the table that was just dropped. + .. _python_api_transform_sql: Custom transformations with .transform_sql() diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 713b110..5307931 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2824,17 +2824,34 @@ class Table(Queryable): new_cols=", ".join(quote_identifier(col) for col in new_cols), ) sqls.append(copy_sql) - # Drop (or keep) the old table + # Drop (or keep) the old table, then rename the new one into place. + # Since SQLite 3.25 ALTER TABLE ... RENAME TO rewrites references to + # the renamed table in every view definition, which fails if a view + # references the table that was just dropped - and with keep_table= + # would silently repoint views at the backup table. These renames are + # an implementation detail of transform(), so use legacy_alter_table + # to leave view definitions untouched, restoring the connection's + # current value afterwards. + legacy_alter_table_row = self.db.execute("PRAGMA legacy_alter_table").fetchone() + legacy_alter_table_was_on = bool( + legacy_alter_table_row and legacy_alter_table_row[0] + ) if keep_table: + sqls.append("PRAGMA legacy_alter_table=ON;") sqls.append( f"ALTER TABLE {quote_identifier(self.name)} RENAME TO {quote_identifier(keep_table)};" ) else: sqls.append(f"DROP TABLE {quote_identifier(self.name)};") - # Rename the new one + sqls.append("PRAGMA legacy_alter_table=ON;") sqls.append( f"ALTER TABLE {quote_identifier(new_table_name)} RENAME TO {quote_identifier(self.name)};" ) + sqls.append( + "PRAGMA legacy_alter_table={};".format( + "ON" if legacy_alter_table_was_on else "OFF" + ) + ) # Re-add existing indexes for index in self.indexes: if index.origin != "pk": diff --git a/tests/test_transform.py b/tests/test_transform.py index b9ee126..7874421 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -16,7 +16,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change column type @@ -26,7 +28,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Rename a column @@ -36,7 +40,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Drop a column @@ -46,7 +52,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n SELECT "rowid", "id", "name" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Convert type AND rename column @@ -56,7 +64,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" INTEGER\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change primary key @@ -66,7 +76,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT PRIMARY KEY\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change primary key to a compound pk @@ -76,7 +88,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT,\n PRIMARY KEY ("age", "name")\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Remove primary key, creating a rowid table @@ -86,7 +100,9 @@ from sqlite_utils.utils import OperationalError 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Keeping the table @@ -95,8 +111,10 @@ from sqlite_utils.utils import OperationalError [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n SELECT "rowid", "id", "name" FROM "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs" RENAME TO "kept_table";', 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), ], @@ -139,7 +157,9 @@ def test_transform_sql_table_with_primary_key( 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Change column type @@ -149,7 +169,9 @@ def test_transform_sql_table_with_primary_key( 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" INTEGER\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Rename a column @@ -159,7 +181,9 @@ def test_transform_sql_table_with_primary_key( 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "dog_age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), # Make ID a primary key @@ -169,7 +193,9 @@ def test_transform_sql_table_with_primary_key( 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n);', 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', 'DROP TABLE "dogs";', + "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', + "PRAGMA legacy_alter_table=OFF;", ], ), ], @@ -903,3 +929,139 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db): "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation." in str(excinfo.value) ) + + +def test_transform_preserves_view(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/831 + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + view_sql_before = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + dogs.transform(rename={"name": "title"}) + view_sql_after = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert view_sql_before == view_sql_after + + +@pytest.mark.parametrize( + "transform_params", + [ + {"types": {"name": int}}, + {"pk": "name"}, + {"add_foreign_keys": [("other_id", "other", "id")]}, + {"drop_foreign_keys": ["other_id"]}, + ], +) +def test_transform_variants_preserve_view(fresh_db, transform_params): + # Covers retyping, changing primary key and foreign key modifications, + # with a view whose columns are untouched by the transform + fresh_db["other"].insert({"id": 1}, pk="id") + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo", "other_id": 1}, pk="id") + if "drop_foreign_keys" in transform_params: + dogs.transform(add_foreign_keys=[("other_id", "other", "id")]) + fresh_db.execute("create view dogs_view as select id, name from dogs") + view_sql_before = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + dogs.transform(**transform_params) + view_sql_after = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert view_sql_before == view_sql_after + assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}] + + +def test_transform_view_referencing_renamed_column(fresh_db): + # The view survives but querying it raises "no such column" - inherent + # to SQLite views, whose SQL is stored as text + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + dogs.transform(rename={"name": "title"}) + with pytest.raises(OperationalError, match="no such column"): + fresh_db.execute("select * from dogs_view") + + +def test_transform_view_on_view(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view v1 as select id, name from dogs") + fresh_db.execute("create view v2 as select name from v1") + sqls_before = fresh_db.execute( + "select sql from sqlite_master where type = 'view' order by name" + ).fetchall() + dogs.transform(types={"id": str}) + sqls_after = fresh_db.execute( + "select sql from sqlite_master where type = 'view' order by name" + ).fetchall() + assert sqls_before == sqls_after + assert list(fresh_db["v2"].rows) == [{"name": "Cleo"}] + + +def test_transform_keep_table_does_not_repoint_view(fresh_db): + # Without legacy_alter_table the ALTER TABLE dogs RENAME TO dogs_backup + # step would rewrite the view to select from "dogs_backup" + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + dogs.transform(types={"name": str}, keep_table="dogs_backup") + view_sql = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert "dogs_backup" not in view_sql + # View reads from the live table, not the frozen backup + dogs.insert({"id": 2, "name": "Pancakes"}) + assert list(fresh_db["dogs_view"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Pancakes"}, + ] + + +def test_transform_sql_standalone_statements_work_with_view(fresh_db): + # The documented "run these statements yourself" workflow should be + # standalone-correct, so the pragmas must come from transform_sql() + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + sqls = dogs.transform_sql(types={"name": str}, tmp_suffix="suffix") + assert sqls[-3] == "PRAGMA legacy_alter_table=ON;" + assert sqls[-2] == 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";' + assert sqls[-1] == "PRAGMA legacy_alter_table=OFF;" + for sql in sqls: + fresh_db.execute(sql) + assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}] + + +def test_transform_with_view_in_open_transaction(fresh_db): + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute("create view dogs_view as select id, name from dogs") + with fresh_db.conn: + fresh_db.execute("insert into dogs (id, name) values (2, 'Pancakes')") + dogs.transform(rename={"name": "title"}) + assert dogs.columns_dict == {"id": int, "title": str} + view_sql = fresh_db.execute( + "select sql from sqlite_master where name = 'dogs_view'" + ).fetchone()[0] + assert view_sql == "CREATE VIEW dogs_view as select id, name from dogs" + + +def test_transform_restores_legacy_alter_table_setting(fresh_db): + if sqlite3.sqlite_version_info < (3, 25, 0): + pytest.skip("legacy_alter_table pragma requires SQLite 3.25 or higher") + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id") + # Default is OFF, reset to OFF afterwards + dogs.transform(types={"name": str}) + assert fresh_db.execute("PRAGMA legacy_alter_table").fetchone()[0] == 0 + # If the connection has it ON, it should be restored to ON + fresh_db.execute("PRAGMA legacy_alter_table=ON") + sqls = dogs.transform_sql(types={"name": str}, tmp_suffix="suffix") + assert sqls[-1] == "PRAGMA legacy_alter_table=ON;" + dogs.transform(types={"name": str}) + assert fresh_db.execute("PRAGMA legacy_alter_table").fetchone()[0] == 1 From 3db0c57a3bc9d8468db430ebe0ffd0da213fdda3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Aug 2026 21:58:00 -0700 Subject: [PATCH 089/110] table.checks, table.column_checks, table.table_checks, closes #834 Refs #762 --- docs/changelog.rst | 1 + docs/python-api.rst | 37 ++ sqlite_utils/create_table_parser.py | 551 ++++++++++++++++++++++++++++ sqlite_utils/db.py | 22 ++ tests/test_create_table_parser.py | 138 +++++++ tests/test_introspect.py | 27 +- 6 files changed, 775 insertions(+), 1 deletion(-) create mode 100644 sqlite_utils/create_table_parser.py create mode 100644 tests/test_create_table_parser.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 4ae53a6..77f644c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Unreleased ---------- +- New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) - ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) .. _v3_39_1: diff --git a/docs/python-api.rst b/docs/python-api.rst index 53a47dd..93f8a3d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2480,6 +2480,43 @@ Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly False +.. _python_api_introspection_checks: + +.checks +------- + +The ``.checks`` property returns the column-level and table-level ``CHECK`` constraints defined on a table, as a list of ``Check`` objects. Each object has ``check`` (the expression inside ``CHECK (...)``), ``name``, ``column`` and ``options`` attributes. ``column`` is an empty string for a table-level check. ``options`` contains a list of values only when a column check consists entirely of ``column IN (literal, ...)``. The original constraint fragment is available as ``sql``; ``start`` and ``end`` are its offsets within ``table.schema``. + +.. code-block:: python + + >>> db["scores"].checks + [Check(check='score > 0', name='positive', column='score', options=None), + Check(check='score <= maximum', name='within_maximum', column='', options=None)] + +.. _python_api_introspection_column_checks: + +.column_checks +-------------- + +The ``.column_checks`` property returns the column-level checks grouped by column name: + +.. code-block:: python + + >>> db["scores"].column_checks + {'score': [Check(check='score > 0', name='positive', column='score', options=None)]} + +.. _python_api_introspection_table_checks: + +.table_checks +------------- + +The ``.table_checks`` property returns only the table-level checks: + +.. code-block:: python + + >>> db["scores"].table_checks + [Check(check='score <= maximum', name='within_maximum', column='', options=None)] + .. _python_api_introspection_foreign_keys: .foreign_keys diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py new file mode 100644 index 0000000..9c0a9aa --- /dev/null +++ b/sqlite_utils/create_table_parser.py @@ -0,0 +1,551 @@ +"""Helpers for parsing CHECK constraints from SQLite CREATE TABLE SQL. + +SQLite does not expose CHECK constraints through a pragma, so preserving them +across a table rebuild requires reading ``sqlite_schema.sql``. This module is +deliberately small, but it uses a real lexer: strings, quoted identifiers and +comments are opaque, every token retains its source span and malformed input is +reported instead of being silently under-parsed. +""" + +import re +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class Check: + check: str + name: str = "" + column: str = "" + options: list[Any] | None = None + # Source details are excluded from equality and repr so callers can compare + # semantic constraints while still having the original SQL available for + # diagnostics or future lossless edits. + sql: str = field(default="", compare=False, repr=False) + start: int = field(default=-1, compare=False, repr=False) + end: int = field(default=-1, compare=False, repr=False) + + +class ParseError(ValueError): + pass + + +@dataclass(frozen=True) +class _Token: + kind: str + text: str + start: int + end: int + + def is_keyword(self, keyword: str) -> bool: + return self.kind == "word" and self.text.upper() == keyword + + +_PUNCTUATION = frozenset("(),.;+-*/%<>=!~|&?:") +_TRIVIA = frozenset(("whitespace", "comment")) +_TABLE_CONSTRAINT_KEYWORDS = frozenset(("PRIMARY", "UNIQUE", "CHECK", "FOREIGN")) +_OTHER_COLUMN_CONSTRAINT_KEYWORDS = frozenset( + ("PRIMARY", "UNIQUE", "REFERENCES", "DEFAULT", "NOT", "COLLATE", "GENERATED") +) +_SQLITE_KEYWORDS = frozenset( + ( + "ABORT", + "ACTION", + "ADD", + "AFTER", + "ALL", + "ALTER", + "ANALYZE", + "AND", + "AS", + "ASC", + "ATTACH", + "AUTOINCREMENT", + "BEFORE", + "BEGIN", + "BETWEEN", + "BY", + "CASCADE", + "CASE", + "CAST", + "CHECK", + "COLLATE", + "COLUMN", + "COMMIT", + "CONFLICT", + "CONSTRAINT", + "CREATE", + "CROSS", + "CURRENT_DATE", + "CURRENT_TIME", + "CURRENT_TIMESTAMP", + "DATABASE", + "DEFAULT", + "DEFERRABLE", + "DEFERRED", + "DELETE", + "DESC", + "DETACH", + "DISTINCT", + "DO", + "DROP", + "EACH", + "ELSE", + "END", + "ESCAPE", + "EXCEPT", + "EXCLUDE", + "EXCLUSIVE", + "EXISTS", + "EXPLAIN", + "FAIL", + "FALSE", + "FILTER", + "FIRST", + "FOLLOWING", + "FOR", + "FOREIGN", + "FROM", + "FULL", + "GENERATED", + "GLOB", + "GROUP", + "GROUPS", + "HAVING", + "IF", + "IGNORE", + "IMMEDIATE", + "IN", + "INDEX", + "INDEXED", + "INITIALLY", + "INNER", + "INSERT", + "INSTEAD", + "INTERSECT", + "INTO", + "IS", + "ISNULL", + "JOIN", + "KEY", + "LAST", + "LEFT", + "LIKE", + "LIMIT", + "MATCH", + "MATERIALIZED", + "NATURAL", + "NO", + "NOT", + "NOTHING", + "NOTNULL", + "NULL", + "NULLS", + "OF", + "OFFSET", + "ON", + "OR", + "ORDER", + "OTHERS", + "OUTER", + "OVER", + "PARTITION", + "PLAN", + "PRAGMA", + "PRECEDING", + "PRIMARY", + "QUERY", + "RAISE", + "RANGE", + "RECURSIVE", + "REFERENCES", + "REGEXP", + "REINDEX", + "RELEASE", + "RENAME", + "REPLACE", + "RESTRICT", + "RETURNING", + "RIGHT", + "ROLLBACK", + "ROW", + "ROWS", + "SAVEPOINT", + "SELECT", + "SET", + "STRICT", + "TABLE", + "TEMP", + "TEMPORARY", + "THEN", + "TIES", + "TO", + "TRANSACTION", + "TRIGGER", + "TRUE", + "UNBOUNDED", + "UNION", + "UNIQUE", + "UPDATE", + "USING", + "VACUUM", + "VALUES", + "VIEW", + "VIRTUAL", + "WHEN", + "WHERE", + "WINDOW", + "WITH", + "WITHOUT", + ) +) +_INTEGER_RE = re.compile(r"[+-]?(?:0[xX][0-9a-fA-F]+|[0-9]+)\Z") +_FLOAT_RE = re.compile( + r"[+-]?(?:(?:[0-9]+\.[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?|" + r"[0-9]+[eE][+-]?[0-9]+)\Z" +) + + +def _lex(sql: str) -> list[_Token]: + tokens: list[_Token] = [] + i = 0 + while i < len(sql): + start = i + char = sql[i] + if char.isspace(): + i += 1 + while i < len(sql) and sql[i].isspace(): + i += 1 + tokens.append(_Token("whitespace", sql[start:i], start, i)) + continue + if sql.startswith("--", i): + newline = sql.find("\n", i + 2) + i = len(sql) if newline == -1 else newline + 1 + tokens.append(_Token("comment", sql[start:i], start, i)) + continue + if sql.startswith("/*", i): + end = sql.find("*/", i + 2) + if end == -1: + raise ParseError("Unterminated SQL comment") + i = end + 2 + tokens.append(_Token("comment", sql[start:i], start, i)) + continue + if char in ("'", '"', "`"): + quote = char + i += 1 + while i < len(sql): + if sql[i] == quote: + if i + 1 < len(sql) and sql[i + 1] == quote: + i += 2 + continue + i += 1 + break + i += 1 + else: + raise ParseError(f"Unterminated {quote} quoted token") + kind = "string" if quote == "'" else "identifier" + tokens.append(_Token(kind, sql[start:i], start, i)) + continue + if char == "[": + end = sql.find("]", i + 1) + if end == -1: + raise ParseError("Unterminated [ quoted identifier") + i = end + 1 + tokens.append(_Token("identifier", sql[start:i], start, i)) + continue + if char in _PUNCTUATION: + i += 1 + tokens.append(_Token("punct", char, start, i)) + continue + # SQLite accepts any character >= U+0080 in a bare identifier. More + # generally, consume until a lexical delimiter rather than relying on + # Python's narrower definition of an alphanumeric character. + i += 1 + while i < len(sql): + if sql[i].isspace() or sql[i] in _PUNCTUATION or sql[i] in "'\"`[": + break + i += 1 + tokens.append(_Token("word", sql[start:i], start, i)) + return tokens + + +def _meaningful(tokens: list[_Token]) -> list[_Token]: + return [token for token in tokens if token.kind not in _TRIVIA] + + +def _unquote(token: str) -> str: + if len(token) >= 2 and token[0] in ("'", '"', "`") and token[-1] == token[0]: + return token[1:-1].replace(token[0] * 2, token[0]) + if len(token) >= 2 and token[0] == "[" and token[-1] == "]": + return token[1:-1] + return token + + +def _matching_paren(tokens: list[_Token], open_index: int) -> int: + if tokens[open_index].text != "(": + raise ParseError("Expected an opening parenthesis") + depth = 0 + for index in range(open_index, len(tokens)): + if tokens[index].text == "(": + depth += 1 + elif tokens[index].text == ")": + depth -= 1 + if depth == 0: + return index + raise ParseError("Unbalanced parentheses") + + +def _split_spans(sql: str, tokens: list[_Token]) -> list[tuple[str, int, int]]: + if not tokens: + return [] + items: list[tuple[str, int, int]] = [] + depth = 0 + start = tokens[0].start + for token in tokens: + if token.text == "(": + depth += 1 + elif token.text == ")": + depth -= 1 + if depth < 0: + raise ParseError("Unbalanced parentheses") + elif token.text == "," and depth == 0: + raw = sql[start : token.start] + item = raw.strip() + if item: + item_start = start + len(raw) - len(raw.lstrip()) + items.append((item, item_start, item_start + len(item))) + start = token.end + if depth: + raise ParseError("Unbalanced parentheses") + raw = sql[start : tokens[-1].end] + item = raw.strip() + if item: + item_start = start + len(raw) - len(raw.lstrip()) + items.append((item, item_start, item_start + len(item))) + return items + + +def _split_ranges(sql: str, tokens: list[_Token]) -> list[str]: + return [item for item, _, _ in _split_spans(sql, tokens)] + + +def _strip_outer_parens(tokens: list[_Token]) -> list[_Token]: + while tokens and tokens[0].text == "(": + close = _matching_paren(tokens, 0) + if close != len(tokens) - 1: + break + tokens = tokens[1:-1] + return tokens + + +_NO_LITERAL = object() + + +def _literal_value(text: str) -> Any: + tokens = _meaningful(_lex(text)) + if len(tokens) == 1 and tokens[0].kind == "string": + return _unquote(tokens[0].text) + raw = "".join(token.text for token in tokens) + if raw.upper() == "NULL": + return None + if raw.upper() == "TRUE": + return True + if raw.upper() == "FALSE": + return False + if _INTEGER_RE.fullmatch(raw): + try: + return ( + int(raw, 16) if raw.lower().lstrip("+-").startswith("0x") else int(raw) + ) + except ValueError: + return _NO_LITERAL + if _FLOAT_RE.fullmatch(raw): + try: + return float(raw) + except ValueError: + return _NO_LITERAL + return _NO_LITERAL + + +def _ascii_fold(identifier: str) -> str: + return identifier.translate( + str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") + ) + + +def _parse_options(expression: str, column: str) -> list[Any] | None: + tokens = _strip_outer_parens(_meaningful(_lex(expression))) + if len(tokens) < 4: + return None + lhs = tokens[0] + if lhs.kind not in ("word", "identifier"): + return None + if column and _ascii_fold(_unquote(lhs.text)) != _ascii_fold(column): + return None + if not tokens[1].is_keyword("IN") or tokens[2].text != "(": + return None + close = _matching_paren(tokens, 2) + if close != len(tokens) - 1: + return None + inner = expression[tokens[2].end : tokens[close].start] + inner_tokens = _lex(inner) + if not _meaningful(inner_tokens): + return [] + values = [] + for item in _split_ranges(inner, inner_tokens): + value = _literal_value(item) + if value is _NO_LITERAL: + return None + values.append(value) + return values + + +def _check_after( + item: str, + tokens: list[_Token], + check_index: int, + name: str, + column: str, + constraint_start: int, + base_offset: int, +) -> tuple[Check, int]: + if check_index + 1 >= len(tokens) or tokens[check_index + 1].text != "(": + raise ParseError("CHECK must be followed by a parenthesized expression") + close = _matching_paren(tokens, check_index + 1) + expression = item[tokens[check_index + 1].end : tokens[close].start].strip() + source_start = tokens[constraint_start].start + source_end = tokens[close].end + return ( + Check( + expression, + name=name, + column=column, + options=_parse_options(expression, column), + sql=item[source_start:source_end], + start=base_offset + source_start, + end=base_offset + source_end, + ), + close + 1, + ) + + +def _column_checks( + item: str, tokens: list[_Token], column: str, base_offset: int +) -> list[Check]: + checks: list[Check] = [] + pending_name = "" + pending_start: int | None = None + index = 1 + while index < len(tokens): + token = tokens[index] + if token.text == "(": + index = _matching_paren(tokens, index) + 1 + continue + if token.is_keyword("CONSTRAINT"): + if index + 1 >= len(tokens): + raise ParseError("CONSTRAINT is missing its name") + pending_name = _unquote(tokens[index + 1].text) + pending_start = index + index += 2 + continue + if token.is_keyword("CHECK"): + check, index = _check_after( + item, + tokens, + index, + pending_name, + column, + pending_start if pending_start is not None else index, + base_offset, + ) + checks.append(check) + pending_name = "" + pending_start = None + continue + if ( + token.kind == "word" + and token.text.upper() in _OTHER_COLUMN_CONSTRAINT_KEYWORDS + ): + pending_name = "" + pending_start = None + index += 1 + return checks + + +def parse_checks(create_sql: str) -> list[Check]: + """Return CHECK constraints from a valid SQLite CREATE TABLE statement.""" + all_tokens = _lex(create_sql) + tokens = _meaningful(all_tokens) + if not tokens or not tokens[0].is_keyword("CREATE"): + raise ParseError("Expected CREATE TABLE") + index = 1 + if index < len(tokens) and ( + tokens[index].is_keyword("TEMP") or tokens[index].is_keyword("TEMPORARY") + ): + index += 1 + if index < len(tokens) and tokens[index].is_keyword("VIRTUAL"): + return [] + if index >= len(tokens) or not tokens[index].is_keyword("TABLE"): + raise ParseError("Expected CREATE TABLE") + index += 1 + if ( + index + 2 < len(tokens) + and tokens[index].is_keyword("IF") + and tokens[index + 1].is_keyword("NOT") + and tokens[index + 2].is_keyword("EXISTS") + ): + index += 3 + if index >= len(tokens): + raise ParseError("CREATE TABLE is missing its table name") + index += 1 + if index + 1 < len(tokens) and tokens[index].text == ".": + index += 2 + if index < len(tokens) and tokens[index].is_keyword("AS"): + return [] + if index >= len(tokens) or tokens[index].text != "(": + raise ParseError("CREATE TABLE is missing its column list") + close = _matching_paren(tokens, index) + trailing = tokens[close + 1 :] + allowed_trailing = {"STRICT", "WITHOUT", "ROWID", ",", ";"} + if any(token.text.upper() not in allowed_trailing for token in trailing): + raise ParseError("Unexpected SQL after CREATE TABLE column list") + + body_start = tokens[index].end + body_end = tokens[close].start + body = create_sql[body_start:body_end] + body_tokens = _lex(body) + checks: list[Check] = [] + for item, item_start, _ in _split_spans(body, body_tokens): + item_tokens = _meaningful(_lex(item)) + if not item_tokens: + continue + item_index = 0 + constraint_name = "" + if item_tokens[item_index].is_keyword("CONSTRAINT"): + if len(item_tokens) < 2: + raise ParseError("CONSTRAINT is missing its name") + constraint_name = _unquote(item_tokens[1].text) + item_index = 2 + head = item_tokens[item_index] if item_index < len(item_tokens) else None + if ( + head + and head.kind == "word" + and head.text.upper() in _TABLE_CONSTRAINT_KEYWORDS + ): + if head.is_keyword("CHECK"): + check, _ = _check_after( + item, + item_tokens, + item_index, + constraint_name, + "", + 0, + body_start + item_start, + ) + checks.append(check) + continue + column = _unquote(item_tokens[0].text) + checks.extend( + _column_checks(item, item_tokens, column, body_start + item_start) + ) + return checks diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 5307931..d85ca41 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -27,6 +27,7 @@ from typing_extensions import Self from sqlite_utils.plugins import ensure_plugins_loaded, pm +from .create_table_parser import Check, parse_checks from .utils import ( OperationalError, chunks, @@ -2196,6 +2197,27 @@ class Table(Queryable): "Does this table use ``rowid`` for its primary key (no other primary keys are specified)?" return not any(column for column in self.columns if column.is_pk) + @property + def checks(self) -> list[Check]: + "List of column-level and table-level CHECK constraints on this table." + if not self.exists() or self.virtual_table_using is not None: + return [] + return parse_checks(self.schema) + + @property + def column_checks(self) -> dict[str, list[Check]]: + "CHECK constraints grouped by the column on which they are defined." + checks: dict[str, list[Check]] = {} + for check in self.checks: + if check.column: + checks.setdefault(check.column, []).append(check) + return checks + + @property + def table_checks(self) -> list[Check]: + "Table-level CHECK constraints on this table." + return [check for check in self.checks if not check.column] + def get(self, pk_values: list | tuple | str | int) -> dict: """ Return row (as dictionary) for the specified primary key. diff --git a/tests/test_create_table_parser.py b/tests/test_create_table_parser.py new file mode 100644 index 0000000..e7ab3e8 --- /dev/null +++ b/tests/test_create_table_parser.py @@ -0,0 +1,138 @@ +import sqlite3 + +import hypothesis.strategies as st +import pytest +from hypothesis import given + +from sqlite_utils.create_table_parser import Check, ParseError, parse_checks + + +def test_parse_column_and_table_checks(): + sql = """ + CREATE TABLE people ( + age INTEGER CONSTRAINT positive CHECK (age > 0), + status TEXT CHECK(status IN ('active', 'inactive')), + CONSTRAINT adult CHECK(age >= 18) + ) + """ + assert parse_checks(sql) == [ + Check("age > 0", name="positive", column="age"), + Check( + "status IN ('active', 'inactive')", + column="status", + options=["active", "inactive"], + ), + Check("age >= 18", name="adult"), + ] + checks = parse_checks(sql) + assert checks[0].sql == "CONSTRAINT positive CHECK (age > 0)" + assert sql[checks[0].start : checks[0].end] == checks[0].sql + assert checks[1].sql == "CHECK(status IN ('active', 'inactive'))" + assert sql[checks[2].start : checks[2].end] == checks[2].sql + + +def test_comments_are_trivia_not_constraints(): + sql = """ + CREATE /* fake CHECK (nope), ( */ TABLE t ( + a INTEGER /* CHECK (a < 0), phantom */, + b INTEGER CHECK /* between keyword and expression */ (b > 0), + /* CHECK (also_fake) */ CONSTRAINT upper CHECK(b < 10) + ) + """ + sqlite3.connect(":memory:").execute(sql) + assert parse_checks(sql) == [ + Check("b > 0", column="b"), + Check("b < 10", name="upper"), + ] + + +@pytest.mark.parametrize( + "expression,expected", + [ + ("value IN ('one', 'two')", ["one", "two"]), + ("((value IN ('one', 'two')))", ["one", "two"]), + ("value NOT IN ('one', 'two')", None), + ("value IN ('one', 'two') OR enabled", None), + ("other IN ('one', 'two')", None), + ("value IN (lower('one'), 'two')", None), + ('value IN ("other")', None), + ], +) +def test_options_only_for_exact_literal_in_check(expression, expected): + sql = f"CREATE TABLE t(value TEXT CHECK({expression}), enabled INTEGER, other TEXT)" + sqlite3.connect(":memory:").execute(sql) + assert parse_checks(sql)[0].options == expected + + +@pytest.mark.parametrize("column", ["💩x", "e\u0301"]) +def test_unquoted_unicode_identifiers(column): + sql = f"CREATE TABLE t({column} INTEGER CHECK({column} > 0))" + sqlite3.connect(":memory:").execute(sql) + assert parse_checks(sql) == [Check(f"{column} > 0", column=column)] + + +@pytest.mark.parametrize( + "sql", + [ + "SELECT CHECK(x > 0)", + "CREATE TABLE t(x INTEGER CHECK(x > 0)", + "CREATE TABLE t(x TEXT CHECK(x != 'unterminated))", + "CREATE TABLE t(x INTEGER /* unterminated)", + ], +) +def test_invalid_sql_raises_parse_error(sql): + with pytest.raises(ParseError): + parse_checks(sql) + + +def test_virtual_table_has_no_checks(): + assert ( + parse_checks("CREATE /* comment */ VIRTUAL TABLE search USING fts5(text)") == [] + ) + + +comment_or_space = st.sampled_from( + [ + " ", + "\n ", + "/* comment with , ( ) and CHECK(fake) */", + "-- comment with , ( ) and CHECK(fake)\n", + ] +) + + +@given(gaps=st.lists(comment_or_space, min_size=5, max_size=5)) +def test_comments_and_whitespace_can_separate_check_tokens(gaps): + sql = ( + f"CREATE{gaps[0]}TABLE{gaps[1]}t{gaps[2]}(" + f"value INTEGER CHECK{gaps[3]}(value{gaps[4]}> 0))" + ) + connection = sqlite3.connect(":memory:") + connection.execute(sql) + stored_sql = connection.execute( + "select sql from sqlite_schema where name = 't'" + ).fetchone()[0] + assert parse_checks(stored_sql) == [Check(f"value{gaps[4]}> 0", column="value")] + + +safe_string_text = st.text( + alphabet=st.characters( + blacklist_categories=("Cc", "Cs"), + blacklist_characters=("'",), + ), + max_size=40, +) + + +@given(value=safe_string_text) +def test_check_like_text_inside_strings_is_opaque(value): + sql = f"CREATE TABLE t(value TEXT CHECK(value != '{value}'))" + connection = sqlite3.connect(":memory:") + connection.execute(sql) + stored_sql = connection.execute( + "select sql from sqlite_schema where name = 't'" + ).fetchone()[0] + checks = parse_checks(stored_sql) + assert len(checks) == 1 + assert checks[0].column == "value" + assert checks[0].check == f"value != '{value}'" diff --git a/tests/test_introspect.py b/tests/test_introspect.py index b7e8fc2..2a8d579 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,6 +1,6 @@ import pytest -from sqlite_utils.db import Database, Index, View, XIndex, XIndexColumn +from sqlite_utils.db import Check, Database, Index, View, XIndex, XIndexColumn def _check_supports_strict(): @@ -177,6 +177,31 @@ def test_pks(fresh_db, pk, expected): assert expected == fresh_db["foo"].pks +def test_checks(fresh_db): + fresh_db.execute(""" + CREATE TABLE scores ( + score INTEGER CONSTRAINT positive CHECK(score > 0), + maximum INTEGER, + CONSTRAINT within_maximum CHECK(score <= maximum) + ) + """) + scores = fresh_db["scores"] + expected_column = Check("score > 0", name="positive", column="score") + expected_table = Check("score <= maximum", name="within_maximum") + assert scores.checks == [expected_column, expected_table] + assert scores.column_checks == {"score": [expected_column]} + assert scores.table_checks == [expected_table] + assert scores.checks[0].sql == "CONSTRAINT positive CHECK(score > 0)" + + +def test_checks_nonexistent_and_virtual_tables(fresh_db): + assert fresh_db["does_not_exist"].checks == [] + fresh_db["searchable"].insert({"text": "hello"}).enable_fts( + ["text"], fts_version="FTS5" + ) + assert fresh_db["searchable_fts"].checks == [] + + def test_triggers_and_triggers_dict(fresh_db): assert [] == fresh_db.triggers authors = fresh_db["authors"] From 2303b80aef69abe418636dc980c5350958d55411 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Aug 2026 21:59:57 -0700 Subject: [PATCH 090/110] .transform() preserves check constraints, refs #762 --- docs/changelog.rst | 1 + docs/python-api.rst | 9 ++ sqlite_utils/create_table_parser.py | 83 +++++++++++++++ sqlite_utils/db.py | 65 +++++++++++- tests/test_mutator_transactions.py | 154 ++++++++++++++++++++++++++++ tests/test_transform.py | 137 ++++++++++++++++++++++++- 6 files changed, 447 insertions(+), 2 deletions(-) create mode 100644 tests/test_mutator_transactions.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 77f644c..938e9db 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) +- ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in them without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) - ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) .. _v3_39_1: diff --git a/docs/python-api.rst b/docs/python-api.rst index 93f8a3d..75900b6 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1986,6 +1986,15 @@ A bare column name drops any foreign key that column participates in, including Renaming a column with ``rename=`` updates any foreign keys that use it, and dropping a column with ``drop=`` also drops any foreign keys it participates in - for a compound foreign key this removes the whole constraint. +.. _python_api_transform_check_constraints: + +CHECK constraints +----------------- + +``.transform()`` preserves both column-level and table-level ``CHECK`` constraints. If a column is renamed, references to that column in the check expression are renamed too. + +A column-level check is removed if its owning column is dropped. Dropping a column referenced by any remaining check raises ``TransformError`` instead of creating an invalid or unexpectedly weakened schema. + .. _python_api_transform_views: Tables referenced by views diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py index 9c0a9aa..082c2c8 100644 --- a/sqlite_utils/create_table_parser.py +++ b/sqlite_utils/create_table_parser.py @@ -549,3 +549,86 @@ def parse_checks(create_sql: str) -> list[Check]: _column_checks(item, item_tokens, column, body_start + item_start) ) return checks + + +def _is_identifier_token(tokens: list[_Token], index: int) -> bool: + token = tokens[index] + if index + 1 < len(tokens) and tokens[index + 1].text in ("(", "."): + return False + if index and ( + tokens[index - 1].is_keyword("COLLATE") or tokens[index - 1].is_keyword("AS") + ): + return False + if token.kind == "identifier": + return True + if token.kind != "word" or token.text.upper() in _SQLITE_KEYWORDS: + return False + return True + + +def check_references_identifier(expression: str, identifier: str) -> bool: + tokens = _meaningful(_lex(expression)) + folded = _ascii_fold(identifier) + return any( + _is_identifier_token(tokens, index) + and _ascii_fold(_unquote(token.text)) == folded + for index, token in enumerate(tokens) + ) + + +def check_expression_ends_in_line_comment(expression: str) -> bool: + """Return True if appended SQL would be swallowed by a ``--`` comment.""" + tokens = _lex(expression) + if not tokens: + return False + final = tokens[-1] + return ( + final.kind == "comment" + and final.text.startswith("--") + and not final.text.endswith(("\n", "\r")) + ) + + +def _valid_bare_identifier(identifier: str) -> bool: + if not identifier or identifier.upper() in _SQLITE_KEYWORDS: + return False + first = identifier[0] + if not (first == "_" or first.isalpha() or ord(first) >= 0x80): + return False + return all( + char == "_" or char == "$" or char.isalnum() or ord(char) >= 0x80 + for char in identifier[1:] + ) + + +def _quote_replacement(original: str, replacement: str) -> str: + if original.startswith('"'): + return '"{}"'.format(replacement.replace('"', '""')) + if original.startswith("`"): + return "`{}`".format(replacement.replace("`", "``")) + if original.startswith("[") and "]" not in replacement: + return f"[{replacement}]" + if _valid_bare_identifier(replacement): + return replacement + return '"{}"'.format(replacement.replace('"', '""')) + + +def rewrite_check_expression(expression: str, rename: dict[str, str]) -> str: + """Rewrite column identifiers in a CHECK expression, preserving trivia.""" + if not rename: + return expression + tokens = _lex(expression) + meaningful = _meaningful(tokens) + replacements = {_ascii_fold(key): value for key, value in rename.items()} + edits: list[tuple[int, int, str]] = [] + for index, token in enumerate(meaningful): + if not _is_identifier_token(meaningful, index): + continue + replacement = replacements.get(_ascii_fold(_unquote(token.text))) + if replacement is not None: + edits.append( + (token.start, token.end, _quote_replacement(token.text, replacement)) + ) + for start, end, replacement in reversed(edits): + expression = expression[:start] + replacement + expression[end:] + return expression diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d85ca41..949f574 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -27,7 +27,14 @@ from typing_extensions import Self from sqlite_utils.plugins import ensure_plugins_loaded, pm -from .create_table_parser import Check, parse_checks +from .create_table_parser import ( + Check, + ParseError, + check_expression_ends_in_line_comment, + check_references_identifier, + parse_checks, + rewrite_check_expression, +) from .utils import ( OperationalError, chunks, @@ -86,6 +93,12 @@ def quote_identifier(identifier: str) -> str: return '"{}"'.format(identifier.replace('"', '""')) +def _check_constraint_sql(check: Check) -> str: + prefix = f"CONSTRAINT {quote_identifier(check.name)} " if check.name else "" + newline = "\n" if check_expression_ends_in_line_comment(check.check) else "" + return f"{prefix}CHECK ({check.check}{newline})" + + _IDENTIFIER_CASEFOLD = str.maketrans( "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" ) @@ -1380,6 +1393,7 @@ class Database: extracts: dict[str, str] | list[str] | None = None, if_not_exists: bool = False, strict: bool = False, + _checks: Iterable[Check] | None = None, ) -> str: """ Returns the SQL ``CREATE TABLE`` statement for creating the specified table. @@ -1425,6 +1439,19 @@ class Database: defaults = {resolve_casing(n, columns): v for n, v in (defaults or {}).items()} if column_order is not None: column_order = [resolve_casing(c, columns) for c in column_order] + checks = list(_checks or ()) + checks_by_column: dict[str, list[Check]] = {} + table_checks: list[Check] = [] + for check in checks: + if check.column: + column = resolve_casing(check.column, columns) + if column not in columns: + raise AlterError( + f"No such column for CHECK constraint: {check.column}" + ) + checks_by_column.setdefault(column, []).append(check) + else: + table_checks.append(check) if not columns: raise ValueError("Tables must have at least one column") if not all(n in columns for n in not_null): @@ -1481,6 +1508,10 @@ class Database: column_extras.append( f"REFERENCES {quote_identifier(fk.other_table)}({quote_identifier(cast(str, fk.other_column))}){_fk_actions_sql(fk)}" ) + column_extras.extend( + _check_constraint_sql(check) + for check in checks_by_column.get(column_name, ()) + ) column_type_str = COLUMN_TYPE_MAPPING[column_type] # Special case for strict tables to map FLOAT to REAL # Refs https://github.com/simonw/sqlite-utils/issues/644 @@ -1520,6 +1551,9 @@ class Database: actions=_fk_actions_sql(fk), ) ) + column_defs.extend( + f" {_check_constraint_sql(check)}" for check in table_checks + ) columns_sql = ",\n".join(column_defs) sql = """CREATE TABLE {if_not_exists}{table} ( {columns_sql}{extra_pk} @@ -2677,6 +2711,34 @@ class Table(Queryable): if column_order is not None: column_order = [resolve_casing(c, existing_columns) for c in column_order] + try: + existing_checks = self.checks + except ParseError as ex: + raise TransformError( + f"Could not parse CHECK constraints for table {self.name!r}: {ex}" + ) from ex + create_table_checks: list[Check] = [] + for check in existing_checks: + owner = ( + resolve_casing(check.column, existing_columns) if check.column else "" + ) + # A column-level constraint disappears with the column that owns it. + if owner and owner in drop: + continue + for dropped_column in drop: + if check_references_identifier(check.check, dropped_column): + raise TransformError( + f"Cannot drop column {dropped_column!r}: it is used by " + f"CHECK constraint {check.name or check.check!r}" + ) + create_table_checks.append( + Check( + rewrite_check_expression(check.check, rename), + name=check.name, + column=rename.get(owner) or owner, + ) + ) + create_table_foreign_keys: list[ForeignKeyIndicator] = [] if foreign_keys is not None: @@ -2826,6 +2888,7 @@ class Table(Queryable): foreign_keys=create_table_foreign_keys, column_order=column_order, strict=self.strict if strict is None else strict, + _checks=create_table_checks, ).strip() ) diff --git a/tests/test_mutator_transactions.py b/tests/test_mutator_transactions.py new file mode 100644 index 0000000..37ae1b6 --- /dev/null +++ b/tests/test_mutator_transactions.py @@ -0,0 +1,154 @@ +import pytest + +from sqlite_utils import Database +from sqlite_utils.utils import sqlite3 + +BASELINE_ROWS = [(1, "one"), (2, "two")] + + +def insert(table): + table.insert({"id": 3, "value": "three"}, pk="id") + + +def insert_all(table): + table.insert_all( + [ + {"id": 3, "value": "three"}, + {"id": 4, "value": "four"}, + ], + pk="id", + batch_size=1, + ) + + +def upsert(table): + table.upsert({"id": 2, "value": "TWO"}, pk="id") + + +def upsert_all(table): + table.upsert_all( + [ + {"id": 2, "value": "TWO"}, + {"id": 3, "value": "three"}, + ], + pk="id", + batch_size=1, + ) + + +def update(table): + table.update(2, {"value": "TWO"}) + + +def delete(table): + table.delete(2) + + +def delete_where(table): + table.delete_where("id > ?", [1]) + + +MUTATOR_CASES = ( + pytest.param( + insert, + [(1, "one"), (2, "two"), (3, "three")], + id="insert", + ), + pytest.param( + insert_all, + [(1, "one"), (2, "two"), (3, "three"), (4, "four")], + id="insert_all", + ), + pytest.param( + upsert, + [(1, "one"), (2, "TWO")], + id="upsert", + ), + pytest.param( + upsert_all, + [(1, "one"), (2, "TWO"), (3, "three")], + id="upsert_all", + ), + pytest.param( + update, + [(1, "one"), (2, "TWO")], + id="update", + ), + pytest.param(delete, [(1, "one")], id="delete"), + pytest.param(delete_where, [(1, "one")], id="delete_where"), +) + + +class RollbackTest(Exception): + pass + + +def seed_database(path): + conn = sqlite3.connect(str(path)) + try: + conn.execute("create table items (id integer primary key, value text)") + conn.executemany("insert into items values (?, ?)", BASELINE_ROWS) + conn.commit() + finally: + conn.close() + return Database(path) + + +def current_rows(db): + return db.conn.execute("select id, value from items order by id").fetchall() + + +def persisted_rows(path): + conn = sqlite3.connect(str(path)) + try: + return conn.execute("select id, value from items order by id").fetchall() + finally: + conn.close() + + +@pytest.mark.parametrize("mutate,expected_rows", MUTATOR_CASES) +def test_mutator_commits_by_default(tmp_path, mutate, expected_rows): + path = tmp_path / "default.db" + db = seed_database(path) + + assert not db.conn.in_transaction + mutate(db["items"]) + assert current_rows(db) == expected_rows + assert not db.conn.in_transaction + + db.close() + assert persisted_rows(path) == expected_rows + + +@pytest.mark.parametrize("mutate,expected_rows", MUTATOR_CASES) +def test_mutator_commits_with_outer_atomic(tmp_path, mutate, expected_rows): + path = tmp_path / "atomic.db" + db = seed_database(path) + + with db.atomic(): + assert db.conn.in_transaction + mutate(db["items"]) + assert current_rows(db) == expected_rows + assert db.conn.in_transaction + + assert current_rows(db) == expected_rows + assert not db.conn.in_transaction + db.close() + assert persisted_rows(path) == expected_rows + + +@pytest.mark.parametrize("mutate,expected_rows", MUTATOR_CASES) +def test_mutator_rolls_back_outer_atomic(tmp_path, mutate, expected_rows): + path = tmp_path / "rollback.db" + db = seed_database(path) + + with pytest.raises(RollbackTest), db.atomic(): + mutate(db["items"]) + assert current_rows(db) == expected_rows + assert db.conn.in_transaction + raise RollbackTest + + assert current_rows(db) == BASELINE_ROWS + assert not db.conn.in_transaction + db.close() + assert persisted_rows(path) == BASELINE_ROWS diff --git a/tests/test_transform.py b/tests/test_transform.py index 7874421..44fec99 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -2,7 +2,7 @@ import sqlite3 import pytest -from sqlite_utils.db import ForeignKey, TransactionError, TransformError +from sqlite_utils.db import Check, ForeignKey, TransactionError, TransformError from sqlite_utils.utils import OperationalError @@ -1065,3 +1065,138 @@ def test_transform_restores_legacy_alter_table_setting(fresh_db): assert sqls[-1] == "PRAGMA legacy_alter_table=ON;" dogs.transform(types={"name": str}) assert fresh_db.execute("PRAGMA legacy_alter_table").fetchone()[0] == 1 + + +def test_transform_preserves_check_constraints(fresh_db): + fresh_db.execute(""" + CREATE TABLE scores ( + id INTEGER PRIMARY KEY, + score INTEGER CONSTRAINT valid_score CHECK(score BETWEEN 0 AND 100), + CONSTRAINT nonzero_id CHECK(id != 0) + ) + """) + scores = fresh_db["scores"] + scores.insert({"id": 1, "score": 50}) + scores.transform() + assert scores.checks == [ + Check("score BETWEEN 0 AND 100", name="valid_score", column="score"), + Check("id != 0", name="nonzero_id"), + ] + with pytest.raises(sqlite3.IntegrityError, match="CHECK constraint failed"): + scores.insert({"id": 2, "score": 101}) + + +def test_transform_preserves_check_ending_in_line_comment(fresh_db): + fresh_db.execute(""" + CREATE TABLE inventory ( + quantity INTEGER, + CHECK ( + quantity >= 0 -- Quantity cannot be negative + ) + ) + """) + inventory = fresh_db["inventory"] + inventory.transform(types={"quantity": float}) + assert inventory.checks == [Check("quantity >= 0 -- Quantity cannot be negative")] + with pytest.raises(sqlite3.IntegrityError, match="CHECK constraint failed"): + inventory.insert({"quantity": -1}) + + +def test_transform_renames_columns_inside_check_constraints(fresh_db): + fresh_db.execute(""" + CREATE TABLE inventory ( + quantity INTEGER CONSTRAINT positive + CHECK(quantity > 0 AND 'quantity' != ''), + maximum INTEGER, + CONSTRAINT within_maximum CHECK(quantity <= maximum) + ) + """) + inventory = fresh_db["inventory"] + inventory.insert({"quantity": 2, "maximum": 3}) + inventory.transform(rename={"quantity": "amount"}) + assert inventory.checks == [ + Check( + "amount > 0 AND 'quantity' != ''", + name="positive", + column="amount", + ), + Check("amount <= maximum", name="within_maximum"), + ] + with pytest.raises(sqlite3.IntegrityError, match="CHECK constraint failed"): + inventory.insert({"amount": 4, "maximum": 3}) + + +def test_transform_check_rewrite_preserves_functions_and_quotes(fresh_db): + fresh_db.execute(""" + CREATE TABLE items ( + length TEXT, + "old name" TEXT, + CHECK(length("old name") > 0 AND length != '') + ) + """) + items = fresh_db["items"] + items.insert({"length": "label", "old name": "hello"}) + items.transform(rename={"length": "description", "old name": "new name"}) + assert items.checks == [Check("length(\"new name\") > 0 AND description != ''")] + + +def test_transform_check_rewrite_quotes_keyword_column(fresh_db): + fresh_db.execute("CREATE TABLE t(old_name TEXT CHECK(old_name != ''))") + fresh_db["t"].insert({"old_name": "value"}) + fresh_db["t"].transform(rename={"old_name": "select"}) + assert fresh_db["t"].checks == [Check("\"select\" != ''", column="select")] + + +def test_transform_check_rewrite_does_not_rename_collations_or_cast_types(fresh_db): + fresh_db.execute(""" + CREATE TABLE t ( + nocase TEXT, + kind TEXT, + other TEXT, + CHECK( + other COLLATE nocase != '' + AND CAST(other AS kind) != '' + AND nocase != '' + AND kind != '' + ) + ) + """) + fresh_db["t"].insert({"nocase": "n", "kind": "k", "other": "o"}) + fresh_db["t"].transform(rename={"nocase": "label", "kind": "category"}) + check = fresh_db["t"].checks[0].check + assert "COLLATE nocase" in check + assert "AS kind" in check + assert "AND label != ''" in check + assert "AND category != ''" in check + + +def test_transform_drops_check_owned_by_dropped_column(fresh_db): + fresh_db.execute(""" + CREATE TABLE t ( + id INTEGER, + obsolete INTEGER CHECK(obsolete > 0), + CHECK(id > 0) + ) + """) + fresh_db["t"].insert({"id": 1, "obsolete": 2}) + fresh_db["t"].transform(drop={"obsolete"}) + assert fresh_db["t"].checks == [Check("id > 0")] + + +def test_transform_refuses_to_drop_column_used_by_remaining_check(fresh_db): + fresh_db.execute(""" + CREATE TABLE ranges ( + minimum INTEGER, + maximum INTEGER, + CHECK(minimum <= maximum) + ) + """) + ranges = fresh_db["ranges"] + ranges.insert({"minimum": 1, "maximum": 2}) + schema_before = ranges.schema + with pytest.raises( + TransformError, + match="Cannot drop column 'maximum'.*CHECK constraint", + ): + ranges.transform(drop={"maximum"}) + assert ranges.schema == schema_before From b432e686ca3d3449df74899393ab05cb4f40f6d7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Aug 2026 22:02:28 -0700 Subject: [PATCH 091/110] Use sqlite_master not sqlite_schema for older SQLite compatibility --- tests/test_create_table_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_create_table_parser.py b/tests/test_create_table_parser.py index e7ab3e8..3b878f3 100644 --- a/tests/test_create_table_parser.py +++ b/tests/test_create_table_parser.py @@ -110,7 +110,7 @@ def test_comments_and_whitespace_can_separate_check_tokens(gaps): connection = sqlite3.connect(":memory:") connection.execute(sql) stored_sql = connection.execute( - "select sql from sqlite_schema where name = 't'" + "select sql from sqlite_master where name = 't'" ).fetchone()[0] assert parse_checks(stored_sql) == [Check(f"value{gaps[4]}> 0", column="value")] @@ -130,7 +130,7 @@ def test_check_like_text_inside_strings_is_opaque(value): connection = sqlite3.connect(":memory:") connection.execute(sql) stored_sql = connection.execute( - "select sql from sqlite_schema where name = 't'" + "select sql from sqlite_master where name = 't'" ).fetchone()[0] checks = parse_checks(stored_sql) assert len(checks) == 1 From b37b8cf8c83515a022e7ec0b1599004bb2f7eb55 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 11 Aug 2026 22:37:13 -0700 Subject: [PATCH 092/110] Preserve column before/after comments through .transform() Refs #762 The before comment comes before the column definition - the after comment is anything after it but before its trailing comma. --- docs/changelog.rst | 3 +- docs/python-api.rst | 2 ++ sqlite_utils/create_table_parser.py | 56 +++++++++++++++++++++++++---- sqlite_utils/db.py | 52 ++++++++++++++++++++++----- tests/test_create_table_parser.py | 28 ++++++++++++++- tests/test_transform.py | 48 +++++++++++++++++++++++++ 6 files changed, 171 insertions(+), 18 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 938e9db..2950dd4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,8 @@ Unreleased ---------- - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) -- ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in them without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) +- ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) +- ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) - ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) .. _v3_39_1: diff --git a/docs/python-api.rst b/docs/python-api.rst index 75900b6..90c5165 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1995,6 +1995,8 @@ CHECK constraints A column-level check is removed if its owning column is dropped. Dropping a column referenced by any remaining check raises ``TransformError`` instead of creating an invalid or unexpectedly weakened schema. +Comments immediately before or after a column definition are preserved too. They move with that column if it is renamed or reordered, and are removed if the column is dropped. A comment between two column definitions is treated as belonging to the following column. + .. _python_api_transform_views: Tables referenced by views diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py index 082c2c8..2d891ae 100644 --- a/sqlite_utils/create_table_parser.py +++ b/sqlite_utils/create_table_parser.py @@ -26,6 +26,12 @@ class Check: end: int = field(default=-1, compare=False, repr=False) +@dataclass(frozen=True) +class ColumnComments: + before: str = "" + after: str = "" + + class ParseError(ValueError): pass @@ -472,8 +478,7 @@ def _column_checks( return checks -def parse_checks(create_sql: str) -> list[Check]: - """Return CHECK constraints from a valid SQLite CREATE TABLE statement.""" +def _table_body(create_sql: str) -> tuple[str, int] | None: all_tokens = _lex(create_sql) tokens = _meaningful(all_tokens) if not tokens or not tokens[0].is_keyword("CREATE"): @@ -484,7 +489,7 @@ def parse_checks(create_sql: str) -> list[Check]: ): index += 1 if index < len(tokens) and tokens[index].is_keyword("VIRTUAL"): - return [] + return None if index >= len(tokens) or not tokens[index].is_keyword("TABLE"): raise ParseError("Expected CREATE TABLE") index += 1 @@ -501,7 +506,7 @@ def parse_checks(create_sql: str) -> list[Check]: if index + 1 < len(tokens) and tokens[index].text == ".": index += 2 if index < len(tokens) and tokens[index].is_keyword("AS"): - return [] + return None if index >= len(tokens) or tokens[index].text != "(": raise ParseError("CREATE TABLE is missing its column list") close = _matching_paren(tokens, index) @@ -512,7 +517,15 @@ def parse_checks(create_sql: str) -> list[Check]: body_start = tokens[index].end body_end = tokens[close].start - body = create_sql[body_start:body_end] + return create_sql[body_start:body_end], body_start + + +def parse_checks(create_sql: str) -> list[Check]: + """Return CHECK constraints from a valid SQLite CREATE TABLE statement.""" + body_info = _table_body(create_sql) + if body_info is None: + return [] + body, body_start = body_info body_tokens = _lex(body) checks: list[Check] = [] for item, item_start, _ in _split_spans(body, body_tokens): @@ -551,6 +564,35 @@ def parse_checks(create_sql: str) -> list[Check]: return checks +def parse_column_comments(create_sql: str) -> dict[str, ColumnComments]: + """Return comments immediately before and after each column definition.""" + body_info = _table_body(create_sql) + if body_info is None: + return {} + body, _ = body_info + comments: dict[str, ColumnComments] = {} + for item, _, _ in _split_spans(body, _lex(body)): + item_tokens = _meaningful(_lex(item)) + if not item_tokens: + continue + item_index = 0 + if item_tokens[item_index].is_keyword("CONSTRAINT"): + item_index = 2 + head = item_tokens[item_index] if item_index < len(item_tokens) else None + if ( + head + and head.kind == "word" + and head.text.upper() in _TABLE_CONSTRAINT_KEYWORDS + ): + continue + column = _unquote(item_tokens[0].text) + before = item[: item_tokens[0].start].strip() + after = item[item_tokens[-1].end :].strip() + if before or after: + comments[column] = ColumnComments(before=before, after=after) + return comments + + def _is_identifier_token(tokens: list[_Token], index: int) -> bool: token = tokens[index] if index + 1 < len(tokens) and tokens[index + 1].text in ("(", "."): @@ -576,9 +618,9 @@ def check_references_identifier(expression: str, identifier: str) -> bool: ) -def check_expression_ends_in_line_comment(expression: str) -> bool: +def sql_ends_in_line_comment(sql: str) -> bool: """Return True if appended SQL would be swallowed by a ``--`` comment.""" - tokens = _lex(expression) + tokens = _lex(sql) if not tokens: return False final = tokens[-1] diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 949f574..2e9b570 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -29,11 +29,13 @@ from sqlite_utils.plugins import ensure_plugins_loaded, pm from .create_table_parser import ( Check, + ColumnComments, ParseError, - check_expression_ends_in_line_comment, check_references_identifier, parse_checks, + parse_column_comments, rewrite_check_expression, + sql_ends_in_line_comment, ) from .utils import ( OperationalError, @@ -95,10 +97,26 @@ def quote_identifier(identifier: str) -> str: def _check_constraint_sql(check: Check) -> str: prefix = f"CONSTRAINT {quote_identifier(check.name)} " if check.name else "" - newline = "\n" if check_expression_ends_in_line_comment(check.check) else "" + newline = "\n" if sql_ends_in_line_comment(check.check) else "" return f"{prefix}CHECK ({check.check}{newline})" +def _column_definition_with_comments( + definition: str, comments: ColumnComments | None +) -> str: + if comments is None: + return definition + before = textwrap.dedent(comments.before).strip() + after = textwrap.dedent(comments.after).strip() + if before: + definition = f"{textwrap.indent(before, ' ')}\n{definition}" + if after: + definition = f"{definition} {after}" + if sql_ends_in_line_comment(after): + definition += "\n" + return definition + + _IDENTIFIER_CASEFOLD = str.maketrans( "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" ) @@ -1394,6 +1412,7 @@ class Database: if_not_exists: bool = False, strict: bool = False, _checks: Iterable[Check] | None = None, + _column_comments: Mapping[str, ColumnComments] | None = None, ) -> str: """ Returns the SQL ``CREATE TABLE`` statement for creating the specified table. @@ -1439,6 +1458,10 @@ class Database: defaults = {resolve_casing(n, columns): v for n, v in (defaults or {}).items()} if column_order is not None: column_order = [resolve_casing(c, columns) for c in column_order] + column_comments = { + resolve_casing(name, columns): comments + for name, comments in (_column_comments or {}).items() + } checks = list(_checks or ()) checks_by_column: dict[str, list[Check]] = {} table_checks: list[Check] = [] @@ -1517,13 +1540,16 @@ class Database: # Refs https://github.com/simonw/sqlite-utils/issues/644 if strict and column_type_str == "FLOAT": column_type_str = "REAL" + column_definition = " {} {column_type}{column_extras}".format( + quote_identifier(column_name), + column_type=column_type_str, + column_extras=( + (" " + " ".join(column_extras)) if column_extras else "" + ), + ) column_defs.append( - " {} {column_type}{column_extras}".format( - quote_identifier(column_name), - column_type=column_type_str, - column_extras=( - (" " + " ".join(column_extras)) if column_extras else "" - ), + _column_definition_with_comments( + column_definition, column_comments.get(column_name) ) ) extra_pk = "" @@ -2713,9 +2739,10 @@ class Table(Queryable): try: existing_checks = self.checks + existing_column_comments = parse_column_comments(self.schema) except ParseError as ex: raise TransformError( - f"Could not parse CHECK constraints for table {self.name!r}: {ex}" + f"Could not parse table schema for table {self.name!r}: {ex}" ) from ex create_table_checks: list[Check] = [] for check in existing_checks: @@ -2739,6 +2766,12 @@ class Table(Queryable): ) ) + create_table_column_comments: dict[str, ColumnComments] = {} + for column, comments in existing_column_comments.items(): + owner = resolve_casing(column, existing_columns) + if owner not in drop: + create_table_column_comments[rename.get(owner) or owner] = comments + create_table_foreign_keys: list[ForeignKeyIndicator] = [] if foreign_keys is not None: @@ -2889,6 +2922,7 @@ class Table(Queryable): column_order=column_order, strict=self.strict if strict is None else strict, _checks=create_table_checks, + _column_comments=create_table_column_comments, ).strip() ) diff --git a/tests/test_create_table_parser.py b/tests/test_create_table_parser.py index 3b878f3..a7aa0c0 100644 --- a/tests/test_create_table_parser.py +++ b/tests/test_create_table_parser.py @@ -4,7 +4,13 @@ import hypothesis.strategies as st import pytest from hypothesis import given -from sqlite_utils.create_table_parser import Check, ParseError, parse_checks +from sqlite_utils.create_table_parser import ( + Check, + ColumnComments, + ParseError, + parse_checks, + parse_column_comments, +) def test_parse_column_and_table_checks(): @@ -46,6 +52,26 @@ def test_comments_are_trivia_not_constraints(): ] +def test_parse_comments_owned_by_columns(): + sql = """ + CREATE TABLE t ( + -- Before id + id /* Between name and type */ INTEGER /* After id */, + /* Between column definitions */ + value TEXT CHECK(value != '') /* After value */, + /* Before a table constraint, not a column */ + CHECK(value != 'forbidden') + ) + """ + assert parse_column_comments(sql) == { + "id": ColumnComments(before="-- Before id", after="/* After id */"), + "value": ColumnComments( + before="/* Between column definitions */", + after="/* After value */", + ), + } + + @pytest.mark.parametrize( "expression,expected", [ diff --git a/tests/test_transform.py b/tests/test_transform.py index 44fec99..980ee9d 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1102,6 +1102,54 @@ def test_transform_preserves_check_ending_in_line_comment(fresh_db): inventory.insert({"quantity": -1}) +def test_transform_preserves_comments_owned_by_columns(fresh_db): + fresh_db.execute(""" + CREATE TABLE people ( + -- Primary identifier + id INTEGER PRIMARY KEY /* IDs are stable */, + /* Displayed to users */ + name TEXT /* May contain spaces */, + -- Age in years + age INTEGER -- May be NULL + ) + """) + people = fresh_db["people"] + people.insert({"id": 1, "name": "Cleo", "age": 5}) + people.transform( + rename={"name": "display_name"}, + types={"age": float}, + column_order=("age", "id", "name"), + ) + assert people.get(1) == {"age": 5.0, "id": 1, "display_name": "Cleo"} + schema = people.schema + assert schema.index("-- Age in years") < schema.index('"age" REAL') + assert schema.index('"age" REAL') < schema.index("-- May be NULL") + assert schema.index("-- Primary identifier") < schema.index('"id" INTEGER') + assert schema.index('"id" INTEGER') < schema.index("/* IDs are stable */") + assert schema.index("/* Displayed to users */") < schema.index( + '"display_name" TEXT' + ) + assert schema.index('"display_name" TEXT') < schema.index( + "/* May contain spaces */" + ) + + +def test_transform_drops_comments_owned_by_dropped_column(fresh_db): + fresh_db.execute(""" + CREATE TABLE t ( + /* Keep this explanation */ + id INTEGER, + /* Drop this explanation */ + obsolete TEXT /* Drop this too */ + ) + """) + fresh_db["t"].transform(drop={"obsolete"}) + schema = fresh_db["t"].schema + assert "Keep this explanation" in schema + assert "Drop this explanation" not in schema + assert "Drop this too" not in schema + + def test_transform_renames_columns_inside_check_constraints(fresh_db): fresh_db.execute(""" CREATE TABLE inventory ( From 2d3c6b9a1e5068fcee6923c9ed74cbd158ee9db4 Mon Sep 17 00:00:00 2001 From: Bunlong Heng Date: Wed, 12 Aug 2026 01:48:06 -0400 Subject: [PATCH 093/110] Escape tokenize argument in enable_fts (#828) The tokenize value passed to Table.enable_fts() was interpolated directly into the CREATE VIRTUAL TABLE statement inside a single-quoted string literal. A value containing a single quote could break out of that literal and inject arbitrary SQL, which executes via executescript(). This is reachable from the CLI via 'enable-fts --tokenize'. Route the value through the existing Database.quote() helper so SQLite itself escapes it. Legitimate tokenizers such as 'porter' are unaffected. Adds a regression test. --- sqlite_utils/db.py | 4 +++- tests/test_fts.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2e9b570..45482f3 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3514,7 +3514,9 @@ class Table(Queryable): table_fts=quote_identifier(self.name + "_fts"), columns=", ".join(quote_identifier(c) for c in columns), fts_version=fts_version, - tokenize=f"\n tokenize='{tokenize}'," if tokenize else "", + tokenize=( + f"\n tokenize={self.db.quote(tokenize)}," if tokenize else "" + ), ) ) should_recreate = False diff --git a/tests/test_fts.py b/tests/test_fts.py index 50c1770..395fc66 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -252,6 +252,18 @@ def test_fts_tokenize(fresh_db, fts_version): }.items() <= rows[0].items() +def test_fts_tokenize_escaped(fresh_db): + # A malicious tokenize value must not be able to break out of the + # string literal in the CREATE VIRTUAL TABLE statement. + table = fresh_db["searchable"] + table.insert_all(search_records) + malicious = "porter'); CREATE TABLE injected(x); --" + with pytest.raises(Exception): + table.enable_fts(["text"], tokenize=malicious) + # The injected statement must not have executed + assert "injected" not in fresh_db.table_names() + + def test_optimize_fts(fresh_db): for fts_version in ("4", "5"): table_name = f"searchable_{fts_version}" From 43d5d3331f5bd056d20ee61903142b86a2ee0efb Mon Sep 17 00:00:00 2001 From: ethanhawkes-gif Date: Wed, 12 Aug 2026 01:52:43 -0400 Subject: [PATCH 094/110] Emit LIMIT -1 when offset is used without limit (#821) * Emit LIMIT -1 when offset is used without limit, closes #816 SQLite requires a LIMIT clause to appear before OFFSET, so passing offset without limit generated invalid SQL such as: select * from "t" offset 2 which raised OperationalError: near "2": syntax error. A negative limit means "no upper bound" in SQLite, so "limit -1 offset N" returns all rows from position N onwards. Fixed in three places that build LIMIT/OFFSET SQL: - Queryable.rows_where() - also covers pks_and_rows_where() - Table.search_sql() - also covers search() - the "sqlite-utils rows" CLI command * Remove duplicate comments --------- Co-authored-by: ethanhawkes-gif <259455325+ethanhawkes-gif@users.noreply.github.com> --- sqlite_utils/cli.py | 2 ++ sqlite_utils/db.py | 6 ++++++ tests/test_cli.py | 5 +++++ tests/test_fts.py | 11 +++++++++++ tests/test_rows.py | 9 +++++++++ 5 files changed, 33 insertions(+) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index dab4b67..d9c7728 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2478,6 +2478,8 @@ def rows( if limit: sql += f" limit {limit}" if offset: + if not limit: + sql += " limit -1" sql += f" offset {offset}" ctx.invoke( query, diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 45482f3..82a95c1 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2064,6 +2064,10 @@ class Queryable: if limit is not None: sql += f" limit {limit}" if offset is not None: + # SQLite requires a limit clause before offset - a negative limit + # means "no upper bound", so offset works without an explicit limit + if limit is None: + sql += " limit -1" sql += f" offset {offset}" cursor = self.db.execute(sql, where_args or []) columns = dedupe_keys(c[0] for c in cursor.description) @@ -3732,6 +3736,8 @@ class Table(Queryable): if limit is not None: limit_offset += f" limit {limit}" if offset is not None: + if limit is None: + limit_offset += " limit -1" limit_offset += f" offset {offset}" return sql.format( dbtable=quote_identifier(self.name), diff --git a/tests/test_cli.py b/tests/test_cli.py index a1e072f..d3ad228 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1183,6 +1183,11 @@ def test_query_memory_does_not_create_file(tmpdir): ["-c", "name", "--limit", "1", "--offset", "1"], '[{"name": "Pancakes"}]', ), + # --offset without --limit + ( + ["-c", "name", "--offset", "1"], + '[{"name": "Pancakes"}]', + ), # --where ( ["-c", "name", "--where", "id = 1"], diff --git a/tests/test_fts.py b/tests/test_fts.py index 395fc66..79af042 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -112,6 +112,17 @@ def test_search_limit_offset(fresh_db): ) +def test_search_offset_without_limit(fresh_db): + table = fresh_db["t"] + table.insert_all(search_records) + table.enable_fts(["text", "country"], fts_version="FTS4") + assert [row["rowid"] for row in table.search("are", order_by="rowid")] == [1, 2] + assert [ + row["rowid"] for row in table.search("are", offset=1, order_by="rowid") + ] == [2] + assert table.search_sql(offset=1).strip().endswith("limit -1 offset 1") + + @pytest.mark.parametrize("fts_version", ("FTS4", "FTS5")) def test_search_where(fresh_db, fts_version): table = fresh_db["t"] diff --git a/tests/test_rows.py b/tests/test_rows.py index 46d4f53..dccb6ad 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -59,6 +59,9 @@ def test_rows_where_order_by(where, order_by, expected_ids, fresh_db): (None, 3, [1, 2, 3]), (0, 3, [1, 2, 3]), (3, 3, [4, 5, 6]), + # offset without limit should return every remaining row + (97, None, [98, 99, 100]), + (0, None, list(range(1, 101))), ], ) def test_rows_where_offset_limit(fresh_db, offset, limit, expected): @@ -70,6 +73,12 @@ def test_rows_where_offset_limit(fresh_db, offset, limit, expected): ] +def test_pks_and_rows_where_offset_without_limit(fresh_db): + table = fresh_db["rows"] + table.insert_all([{"id": id} for id in range(1, 6)], pk="id") + assert [pk for pk, _ in table.pks_and_rows_where(offset=3, order_by="id")] == [4, 5] + + def test_pks_and_rows_where_rowid(fresh_db): table = fresh_db["rowid_table"] table.insert_all({"number": i + 10} for i in range(3)) From 38fe4667006bc03b0dc22852de3bd891645b5707 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 13:40:55 -0700 Subject: [PATCH 095/110] Use db.table() and db.view() in tests, closes #838 --- tests/test_analyze.py | 20 +- tests/test_analyze_tables.py | 14 +- tests/test_atomic.py | 114 +++++---- tests/test_attach.py | 4 +- tests/test_cli.py | 308 +++++++++++----------- tests/test_cli_bulk.py | 10 +- tests/test_cli_convert.py | 74 +++--- tests/test_cli_insert.py | 68 ++--- tests/test_cli_memory.py | 2 +- tests/test_cli_migrate.py | 76 +++--- tests/test_column_affinity.py | 2 +- tests/test_column_casing.py | 154 +++++------ tests/test_constructor.py | 12 +- tests/test_conversions.py | 10 +- tests/test_convert.py | 20 +- tests/test_create.py | 397 +++++++++++++++-------------- tests/test_default_value.py | 6 +- tests/test_delete.py | 22 +- tests/test_duplicate.py | 4 +- tests/test_enable_counts.py | 40 +-- tests/test_extract.py | 118 ++++----- tests/test_extracts.py | 26 +- tests/test_foreign_keys.py | 220 ++++++++-------- tests/test_fts.py | 112 ++++---- tests/test_get.py | 8 +- tests/test_gis.py | 14 +- tests/test_hypothesis.py | 16 +- tests/test_insert_files.py | 6 +- tests/test_introspect.py | 114 +++++---- tests/test_list_mode.py | 62 ++--- tests/test_lookup.py | 18 +- tests/test_m2m.py | 50 ++-- tests/test_migrations.py | 36 +-- tests/test_mutator_transactions.py | 6 +- tests/test_query.py | 36 +-- tests/test_recipes.py | 40 +-- tests/test_recreate.py | 4 +- tests/test_rows.py | 28 +- tests/test_sniff.py | 2 +- tests/test_transform.py | 238 +++++++++-------- tests/test_update.py | 16 +- tests/test_upsert.py | 26 +- tests/test_wal.py | 22 +- 43 files changed, 1321 insertions(+), 1254 deletions(-) diff --git a/tests/test_analyze.py b/tests/test_analyze.py index a4cd8a2..edd5174 100644 --- a/tests/test_analyze.py +++ b/tests/test_analyze.py @@ -3,11 +3,13 @@ import pytest @pytest.fixture def db(fresh_db): - fresh_db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id") - fresh_db["one_index"].create_index(["name"]) - fresh_db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") - fresh_db["two_indexes"].create_index(["name"]) - fresh_db["two_indexes"].create_index(["species"]) + fresh_db.table("one_index").insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.table("one_index").create_index(["name"]) + fresh_db.table("two_indexes").insert( + {"id": 1, "name": "Cleo", "species": "dog"}, pk="id" + ) + fresh_db.table("two_indexes").create_index(["name"]) + fresh_db.table("two_indexes").create_index(["species"]) return fresh_db @@ -17,7 +19,7 @@ def test_analyze_whole_database(db): assert set(db.table_names()).issuperset( {"one_index", "two_indexes", "sqlite_stat1"} ) - assert list(db["sqlite_stat1"].rows) == [ + assert list(db.table("sqlite_stat1").rows) == [ {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, @@ -30,12 +32,12 @@ def test_analyze_one_table(db, method): if method == "db_method_with_name": db.analyze("one_index") elif method == "table_method": - db["one_index"].analyze() + db.table("one_index").analyze() assert set(db.table_names()).issuperset( {"one_index", "two_indexes", "sqlite_stat1"} ) - assert list(db["sqlite_stat1"].rows) == [ + assert list(db.table("sqlite_stat1").rows) == [ {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"} ] @@ -46,6 +48,6 @@ def test_analyze_index_by_name(db): assert set(db.table_names()).issuperset( {"one_index", "two_indexes", "sqlite_stat1"} ) - assert list(db["sqlite_stat1"].rows) == [ + assert list(db.table("sqlite_stat1").rows) == [ {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, ] diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index a51bba6..9e4799c 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -9,7 +9,7 @@ from sqlite_utils.db import ColumnDetails, Database @pytest.fixture def db_to_analyze(fresh_db): - stuff = fresh_db["stuff"] + stuff = fresh_db.table("stuff") stuff.insert_all( [ {"id": 1, "owner": "Terryterryterry", "size": 5}, @@ -45,7 +45,7 @@ def big_db_to_analyze_path(tmpdir): "all_null": None, } ) - db["stuff"].insert_all(to_insert) + db.table("stuff").insert_all(to_insert) return path @@ -126,7 +126,7 @@ def big_db_to_analyze_path(tmpdir): ) def test_analyze_column(db_to_analyze, column, extra_kwargs, expected): assert ( - db_to_analyze["stuff"].analyze_column( + db_to_analyze.table("stuff").analyze_column( column, common_limit=2, value_truncate=5, **extra_kwargs ) == expected @@ -186,7 +186,7 @@ def test_analyze_table_save(db_to_analyze_path): cli.cli, ["analyze-tables", db_to_analyze_path, "--save"] ) assert result.exit_code == 0 - rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows) + rows = list(Database(db_to_analyze_path).table("_analyze_tables_").rows) assert rows == [ { "table": "stuff", @@ -248,7 +248,7 @@ def test_analyze_table_save_no_most_no_least_options( args.append("--no-least") result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0 - rows = list(Database(big_db_to_analyze_path)["_analyze_tables_"].rows) + rows = list(Database(big_db_to_analyze_path).table("_analyze_tables_").rows) expected = { "table": "stuff", "column": "category", @@ -297,13 +297,13 @@ def test_analyze_table_column_all_nulls(big_db_to_analyze_path): def test_analyze_table_validate_columns(tmpdir, args, expected_error): path = str(tmpdir / "test_validate_columns.db") db = Database(path) - db["one"].insert( + db.table("one").insert( { "id": 1, "name": "one", } ) - db["two"].insert( + db.table("two").insert( { "id": 1, "age": 5, diff --git a/tests/test_atomic.py b/tests/test_atomic.py index ba16ca5..89a318a 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -45,30 +45,30 @@ def test_iter_complete_sql_statements(sql, expected): def test_atomic_commits(fresh_db): with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") - assert list(fresh_db["dogs"].rows) == [{"id": 1, "name": "Cleo"}] + assert list(fresh_db.table("dogs").rows) == [{"id": 1, "name": "Cleo"}] def test_atomic_rolls_back(fresh_db): with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") raise RuntimeError("boom") - assert not fresh_db["dogs"].exists() + assert not fresh_db.table("dogs").exists() def test_nested_atomic_rolls_back_to_savepoint(fresh_db): - fresh_db["dogs"].create({"id": int, "name": str}, pk="id") + fresh_db.table("dogs").create({"id": int, "name": str}, pk="id") with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}) + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo"}) with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + fresh_db.table("dogs").insert({"id": 2, "name": "Pancakes"}) raise RuntimeError("boom") - fresh_db["dogs"].insert({"id": 3, "name": "Marnie"}) + fresh_db.table("dogs").insert({"id": 3, "name": "Marnie"}) - assert list(fresh_db["dogs"].rows) == [ + assert list(fresh_db.table("dogs").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 3, "name": "Marnie"}, ] @@ -76,12 +76,12 @@ def test_nested_atomic_rolls_back_to_savepoint(fresh_db): def test_outer_atomic_rolls_back_released_savepoint(fresh_db): with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") with fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + fresh_db.table("dogs").insert({"id": 2, "name": "Pancakes"}) raise RuntimeError("boom") - assert not fresh_db["dogs"].exists() + assert not fresh_db.table("dogs").exists() def test_executescript_does_not_commit_open_atomic_block(fresh_db): @@ -97,41 +97,41 @@ def test_executescript_does_not_commit_open_atomic_block(fresh_db): """) raise RuntimeError("boom") - assert not fresh_db["dogs"].exists() + assert not fresh_db.table("dogs").exists() def test_transform_does_not_commit_open_atomic_block(fresh_db): - fresh_db["dogs"].insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"}) - fresh_db["dogs"].transform(rename={"age": "dog_age"}) + fresh_db.table("dogs").insert({"id": 2, "name": "Pancakes", "age": "6"}) + fresh_db.table("dogs").transform(rename={"age": "dog_age"}) raise RuntimeError("boom") assert ( - fresh_db["dogs"].schema + fresh_db.table("dogs").schema == 'CREATE TABLE "dogs" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n)' ) - assert list(fresh_db["dogs"].rows) == [ + assert list(fresh_db.table("dogs").rows) == [ {"id": 1, "name": "Cleo", "age": "5"}, ] def test_transform_parent_table_with_foreign_keys_in_atomic(fresh_db): fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") - fresh_db["books"].insert( + fresh_db.table("authors").insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db.table("books").insert( {"id": 1, "title": "Book", "author_id": 1}, pk="id", foreign_keys={"author_id"}, ) with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "full_name"}) + fresh_db.table("authors").transform(rename={"name": "full_name"}) assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] assert ( - fresh_db["authors"].schema + fresh_db.table("authors").schema == 'CREATE TABLE "authors" (\n "id" INTEGER PRIMARY KEY,\n "full_name" TEXT\n)' ) assert fresh_db.execute("PRAGMA foreign_key_check").fetchall() == [] @@ -139,19 +139,19 @@ def test_transform_parent_table_with_foreign_keys_in_atomic(fresh_db): def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db): fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") - fresh_db["books"].insert( + fresh_db.table("authors").insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db.table("books").insert( {"id": 1, "title": "Book", "author_id": 1}, pk="id", foreign_keys={"author_id"}, ) with pytest.raises(RuntimeError), fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "full_name"}) + fresh_db.table("authors").transform(rename={"name": "full_name"}) raise RuntimeError("boom") assert ( - fresh_db["authors"].schema + fresh_db.table("authors").schema == 'CREATE TABLE "authors" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n)' ) assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] @@ -160,49 +160,51 @@ def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db): def test_transform_detects_foreign_key_check_violations(fresh_db): fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 2}, pk="id") + fresh_db.table("authors").insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db.table("books").insert({"id": 1, "author_id": 2}, pk="id") with pytest.raises(sqlite3.IntegrityError): - fresh_db["books"].transform(add_foreign_keys=(("author_id", "authors", "id"),)) + fresh_db.table("books").transform( + add_foreign_keys=(("author_id", "authors", "id"),) + ) - assert fresh_db["books"].foreign_keys == [] + assert fresh_db.table("books").foreign_keys == [] assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] def test_atomic_inside_manual_transaction_uses_savepoint(fresh_db): - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.execute("begin") with fresh_db.atomic(): - fresh_db["t"].insert({"id": 2}, pk="id") + fresh_db.table("t").insert({"id": 2}, pk="id") # Nothing is committed until the user's own transaction commits assert fresh_db.conn.in_transaction fresh_db.rollback() - assert [r["id"] for r in fresh_db["t"].rows] == [1] + assert [r["id"] for r in fresh_db.table("t").rows] == [1] # And with a commit instead, the atomic block's writes persist fresh_db.execute("begin") with fresh_db.atomic(): - fresh_db["t"].insert({"id": 3}, pk="id") + fresh_db.table("t").insert({"id": 3}, pk="id") fresh_db.commit() - assert [r["id"] for r in fresh_db["t"].rows] == [1, 3] + assert [r["id"] for r in fresh_db.table("t").rows] == [1, 3] def test_begin_commit_rollback(tmpdir): path = str(tmpdir / "test.db") db = Database(path) - db["t"].insert({"id": 1}, pk="id") + db.table("t").insert({"id": 1}, pk="id") db.begin() - db["t"].insert({"id": 2}, pk="id") + db.table("t").insert({"id": 2}, pk="id") assert db.conn.in_transaction db.rollback() assert not db.conn.in_transaction - assert [r["id"] for r in db["t"].rows] == [1] + assert [r["id"] for r in db.table("t").rows] == [1] db.begin() - db["t"].insert({"id": 3}, pk="id") + db.table("t").insert({"id": 3}, pk="id") db.commit() db.close() db2 = Database(path) - assert [r["id"] for r in db2["t"].rows] == [1, 3] + assert [r["id"] for r in db2.table("t").rows] == [1, 3] db2.close() @@ -222,7 +224,7 @@ def test_commit_and_rollback_without_transaction_are_noops(fresh_db): def test_execute_write_commits_immediately(tmpdir): path = str(tmpdir / "test.db") db = Database(path) - db["t"].insert({"id": 1}, pk="id") + db.table("t").insert({"id": 1}, pk="id") db.execute("insert into t (id) values (2)") # No implicit transaction is left open assert not db.conn.in_transaction @@ -234,24 +236,24 @@ def test_execute_write_commits_immediately(tmpdir): def test_execute_write_respects_explicit_transaction(fresh_db): - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.begin() fresh_db.execute("insert into t (id) values (2)") # Still inside the explicit transaction - not committed assert fresh_db.conn.in_transaction fresh_db.rollback() - assert [r["id"] for r in fresh_db["t"].rows] == [1] + assert [r["id"] for r in fresh_db.table("t").rows] == [1] def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): # A BEGIN hidden behind a leading comment must not be auto-committed # out from under the caller - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.execute("-- start a transaction\nbegin") assert fresh_db.conn.in_transaction fresh_db.execute("insert into t (id) values (2)") fresh_db.rollback() - assert [r["id"] for r in fresh_db["t"].rows] == [1] + assert [r["id"] for r in fresh_db.table("t").rows] == [1] def _sqlite_accepts_bom(): @@ -269,12 +271,12 @@ def test_execute_prefixed_begin_leaves_transaction_open(fresh_db, begin_sql): # out from under the caller if begin_sql.startswith("\ufeff") and not _sqlite_accepts_bom(): pytest.skip("This SQLite version rejects a leading byte order mark") - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.execute(begin_sql) assert fresh_db.conn.in_transaction fresh_db.execute("insert into t (id) values (2)") fresh_db.rollback() - assert [r["id"] for r in fresh_db["t"].rows] == [1] + assert [r["id"] for r in fresh_db.table("t").rows] == [1] def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir): @@ -282,40 +284,40 @@ def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir): # that would silently disable auto-commit for every subsequent write path = str(tmpdir / "test.db") db = Database(path) - db["t"].insert({"id": 1}, pk="id") + db.table("t").insert({"id": 1}, pk="id") with pytest.raises(sqlite3.IntegrityError): db.execute("insert into t (id) values (1)") assert not db.conn.in_transaction # Subsequent writes commit as normal and survive closing the connection - db["other"].insert({"id": 2}) + db.table("other").insert({"id": 2}) db.close() db2 = Database(path) - assert db2["other"].exists() + assert db2.table("other").exists() db2.close() def test_execute_failed_write_preserves_explicit_transaction(fresh_db): # A failed write inside an explicit transaction must not roll back # the caller's earlier work - only the caller decides that - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.begin() fresh_db.execute("insert into t (id) values (2)") with pytest.raises(sqlite3.IntegrityError): fresh_db.execute("insert into t (id) values (1)") assert fresh_db.conn.in_transaction fresh_db.commit() - assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] + assert [r["id"] for r in fresh_db.table("t").rows] == [1, 2] def test_execute_failed_write_inside_atomic_preserves_block(fresh_db): # A caught failure inside an atomic() block must leave the block's # transaction open so its other work still commits - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") with fresh_db.atomic(): fresh_db.execute("insert into t (id) values (2)") with pytest.raises(sqlite3.IntegrityError): fresh_db.execute("insert into t (id) values (1)") - assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] + assert [r["id"] for r in fresh_db.table("t").rows] == [1, 2] def test_query_returning_commits_after_iteration(tmpdir): @@ -325,7 +327,7 @@ def test_query_returning_commits_after_iteration(tmpdir): _pytest.skip("RETURNING requires SQLite 3.35.0 or higher") path = str(tmpdir / "test.db") db = Database(path) - db["t"].insert({"id": 1}, pk="id") + db.table("t").insert({"id": 1}, pk="id") rows = list(db.query("insert into t (id) values (2) returning id")) assert rows == [{"id": 2}] assert not db.conn.in_transaction @@ -375,7 +377,7 @@ def test_nested_atomic_preserves_error_from_transaction_destroying_trigger( def test_atomic_preserves_error_from_insert_or_rollback(fresh_db): - fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.table("t").insert({"id": 1}, pk="id") with pytest.raises(sqlite3.IntegrityError), fresh_db.atomic(): fresh_db.execute("insert or rollback into t (id) values (1)") assert not fresh_db.conn.in_transaction diff --git a/tests/test_attach.py b/tests/test_attach.py index b594b3b..2b11e36 100644 --- a/tests/test_attach.py +++ b/tests/test_attach.py @@ -6,10 +6,10 @@ def test_attach(tmpdir): bar_path = str(tmpdir / "bar.db") db = Database(foo_path) with db.conn: - db["foo"].insert({"id": 1, "text": "foo"}) + db.table("foo").insert({"id": 1, "text": "foo"}) db2 = Database(bar_path) with db2.conn: - db2["bar"].insert({"id": 1, "text": "bar"}) + db2.table("bar").insert({"id": 1, "text": "bar"}) db.attach("bar", bar_path) assert db.execute( "select * from foo union all select * from bar.bar" diff --git a/tests/test_cli.py b/tests/test_cli.py index d3ad228..012900c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -72,13 +72,13 @@ def test_views(db_path): def test_tables_fts4(db_path): - Database(db_path)["Gosh"].enable_fts(["c2"], fts_version="FTS4") + Database(db_path).table("Gosh").enable_fts(["c2"], fts_version="FTS4") result = CliRunner().invoke(cli.cli, ["tables", "--fts4", db_path]) assert '[{"table": "Gosh_fts"}]' == result.output.strip() def test_tables_fts5(db_path): - Database(db_path)["Gosh"].enable_fts(["c2"], fts_version="FTS5") + Database(db_path).table("Gosh").enable_fts(["c2"], fts_version="FTS5") result = CliRunner().invoke(cli.cli, ["tables", "--fts5", db_path]) assert '[{"table": "Gosh_fts"}]' == result.output.strip() @@ -86,7 +86,7 @@ def test_tables_fts5(db_path): def test_tables_counts_and_columns(db_path): db = Database(db_path) with db.conn: - db["lots"].insert_all([{"id": i, "age": i + 1} for i in range(30)]) + db.table("lots").insert_all([{"id": i, "age": i + 1} for i in range(30)]) result = CliRunner().invoke(cli.cli, ["tables", "--counts", "--columns", db_path]) assert ( '[{"table": "Gosh", "count": 0, "columns": ["c1", "c2", "c3"]},\n' @@ -121,7 +121,7 @@ def test_tables_counts_and_columns(db_path): def test_tables_counts_and_columns_csv(db_path, format, expected): db = Database(db_path) with db.conn: - db["lots"].insert_all([{"id": i, "age": i + 1} for i in range(30)]) + db.table("lots").insert_all([{"id": i, "age": i + 1} for i in range(30)]) result = CliRunner().invoke( cli.cli, ["tables", "--counts", "--columns", format, db_path] ) @@ -131,7 +131,7 @@ def test_tables_counts_and_columns_csv(db_path, format, expected): def test_tables_schema(db_path): db = Database(db_path) with db.conn: - db["lots"].insert_all([{"id": i, "age": i + 1} for i in range(30)]) + db.table("lots").insert_all([{"id": i, "age": i + 1} for i in range(30)]) result = CliRunner().invoke(cli.cli, ["tables", "--schema", db_path]) assert ( '[{"table": "Gosh", "schema": "CREATE TABLE Gosh (c1 text, c2 text, c3 text)"},\n' @@ -183,7 +183,7 @@ def test_tables_schema(db_path): def test_output_table(db_path, options, expected): db = Database(db_path) with db.conn: - db["rows"].insert_all( + db.table("rows").insert_all( [ { "c1": f"verb{i}", @@ -207,7 +207,7 @@ def test_output_table_no_headers(db_path, fmt_option): # tabulate formats and the column names were always printed. db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "name": "Cleo", "age": 4}, {"id": 2, "name": "Pancakes", "age": 2}, @@ -244,14 +244,14 @@ def test_output_table_no_headers(db_path, fmt_option): def test_create_index(db_path): db = Database(db_path) - assert [] == db["Gosh"].indexes + assert [] == db.table("Gosh").indexes result = CliRunner().invoke(cli.cli, ["create-index", db_path, "Gosh", "c1"]) assert result.exit_code == 0 assert [ Index( seq=0, name="idx_Gosh_c1", unique=0, origin="c", partial=0, columns=["c1"] ) - ] == db["Gosh"].indexes + ] == db.table("Gosh").indexes # Try with a custom name result = CliRunner().invoke( cli.cli, ["create-index", db_path, "Gosh", "c2", "--name", "blah"] @@ -262,7 +262,7 @@ def test_create_index(db_path): Index( seq=1, name="idx_Gosh_c1", unique=0, origin="c", partial=0, columns=["c1"] ), - ] == db["Gosh"].indexes + ] == db.table("Gosh").indexes # Try a two-column unique index create_index_unique_args = [ "create-index", @@ -283,7 +283,7 @@ def test_create_index(db_path): partial=0, columns=["c1", "c2"], ) - ] == db["Gosh2"].indexes + ] == db.table("Gosh2").indexes # Trying to create the same index should fail assert CliRunner().invoke(cli.cli, create_index_unique_args).exit_code != 0 # ... unless we use --if-not-exists or --ignore @@ -296,11 +296,11 @@ def test_create_index(db_path): def test_drop_index(db_path): db = Database(db_path) - db["Gosh"].create_index(["c1"]) - assert [index.name for index in db["Gosh"].indexes] == ["idx_Gosh_c1"] + db.table("Gosh").create_index(["c1"]) + assert [index.name for index in db.table("Gosh").indexes] == ["idx_Gosh_c1"] result = CliRunner().invoke(cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1"]) assert result.exit_code == 0 - assert db["Gosh"].indexes == [] + assert db.table("Gosh").indexes == [] result = CliRunner().invoke(cli.cli, ["drop-index", db_path, "Gosh", "idx_Gosh_c1"]) assert result.exit_code == 1 @@ -315,7 +315,7 @@ def test_drop_index(db_path): def test_create_index_analyze(db_path): db = Database(db_path) assert "sqlite_stat1" not in db.table_names() - assert [] == db["Gosh"].indexes + assert [] == db.table("Gosh").indexes result = CliRunner().invoke( cli.cli, ["create-index", db_path, "Gosh", "c1", "--analyze"] ) @@ -325,7 +325,7 @@ def test_create_index_analyze(db_path): def test_create_index_desc(db_path): db = Database(db_path) - assert [] == db["Gosh"].indexes + assert [] == db.table("Gosh").indexes result = CliRunner().invoke(cli.cli, ["create-index", db_path, "Gosh", "--", "-c1"]) assert result.exit_code == 0 assert ( @@ -361,12 +361,12 @@ def test_create_index_desc(db_path): def test_add_column(db_path, col_name, col_type, expected_schema): db = Database(db_path) db.create_table("dogs", {"name": str}) - assert db["dogs"].schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' + assert db.table("dogs").schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' args = ["add-column", db_path, "dogs", col_name] if col_type is not None: args.append(col_type) assert CliRunner().invoke(cli.cli, args).exit_code == 0 - assert db["dogs"].schema == expected_schema + assert db.table("dogs").schema == expected_schema @pytest.mark.parametrize("ignore", (True, False)) @@ -385,7 +385,7 @@ def test_add_column_ignore(db_path, ignore): def test_add_column_not_null_default(db_path): db = Database(db_path) db.create_table("dogs", {"name": str}) - assert db["dogs"].schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' + assert db.table("dogs").schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' args = [ "add-column", db_path, @@ -395,7 +395,7 @@ def test_add_column_not_null_default(db_path): "dogs'dawg", ] assert CliRunner().invoke(cli.cli, args).exit_code == 0 - assert db["dogs"].schema == ( + assert db.table("dogs").schema == ( 'CREATE TABLE "dogs" (\n' ' "name" TEXT\n' ", \"nickname\" TEXT NOT NULL DEFAULT 'dogs''dawg')" @@ -415,10 +415,10 @@ def test_add_column_not_null_default(db_path): ) def test_add_foreign_key(db_path, args, assert_message): db = Database(db_path) - db["authors"].insert_all( + db.table("authors").insert_all( [{"id": 1, "name": "Sally"}, {"id": 2, "name": "Asheesh"}], pk="id" ) - db["books"].insert_all( + db.table("books").insert_all( [ {"title": "Hedgehogs of the world", "author_id": 1}, {"title": "How to train your wolf", "author_id": 2}, @@ -431,7 +431,7 @@ def test_add_foreign_key(db_path, args, assert_message): ForeignKey( table="books", column="author_id", other_table="authors", other_column="id" ) - ] == db["books"].foreign_keys + ] == db.table("books").foreign_keys # Error if we try to add it twice: result = CliRunner().invoke( @@ -460,14 +460,14 @@ def test_add_foreign_key(db_path, args, assert_message): def test_add_column_foreign_key(db_path): db = Database(db_path) - db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - db["books"].insert({"title": "Hedgehogs of the world"}) + db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + db.table("books").insert({"title": "Hedgehogs of the world"}) # Add an author_id foreign key column to the books table result = CliRunner().invoke( cli.cli, ["add-column", db_path, "books", "author_id", "--fk", "authors"] ) assert result.exit_code == 0, result.output - assert db["books"].schema == ( + assert db.table("books").schema == ( 'CREATE TABLE "books" (\n' ' "title" TEXT,\n' ' "author_id" INTEGER REFERENCES "authors"("id")\n' @@ -488,7 +488,7 @@ def test_add_column_foreign_key(db_path): ], ) assert result.exit_code == 0, result.output - assert db["books"].schema == ( + assert db.table("books").schema == ( 'CREATE TABLE "books" (\n' ' "title" TEXT,\n' ' "author_id" INTEGER REFERENCES "authors"("id"),\n' @@ -505,7 +505,7 @@ def test_add_column_foreign_key(db_path): def test_suggest_alter_if_column_missing(db_path): db = Database(db_path) - db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") result = CliRunner().invoke( cli.cli, ["insert", db_path, "authors", "-"], @@ -521,27 +521,27 @@ def test_suggest_alter_if_column_missing(db_path): def test_index_foreign_keys(db_path): test_add_column_foreign_key(db_path) db = Database(db_path) - assert [] == db["books"].indexes + assert [] == db.table("books").indexes result = CliRunner().invoke(cli.cli, ["index-foreign-keys", db_path]) assert result.exit_code == 0 assert [["author_id"], ["author_name_ref"]] == [ - i.columns for i in db["books"].indexes + i.columns for i in db.table("books").indexes ] def test_enable_fts(db_path): db = Database(db_path) - assert db["Gosh"].detect_fts() is None + assert db.table("Gosh").detect_fts() is None result = CliRunner().invoke( cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] ) assert result.exit_code == 0 - assert "Gosh_fts" == db["Gosh"].detect_fts() + assert "Gosh_fts" == db.table("Gosh").detect_fts() # Table names with restricted chars are handled correctly. # colons and dots are restricted characters for table names. - db["http://example.com"].create({"c1": str, "c2": str, "c3": str}) - assert db["http://example.com"].detect_fts() is None + db.table("http://example.com").create({"c1": str, "c2": str, "c3": str}) + assert db.table("http://example.com").detect_fts() is None result = CliRunner().invoke( cli.cli, [ @@ -555,7 +555,7 @@ def test_enable_fts(db_path): ], ) assert result.exit_code == 0 - assert "http://example.com_fts" == db["http://example.com"].detect_fts() + assert "http://example.com_fts" == db.table("http://example.com").detect_fts() # Check tokenize was set to porter assert ( 'CREATE VIRTUAL TABLE "http://example.com_fts" USING FTS4 (\n' @@ -563,19 +563,19 @@ def test_enable_fts(db_path): " tokenize='porter',\n" ' content="http://example.com"' "\n)" - ) == db["http://example.com_fts"].schema - db["http://example.com"].drop() + ) == db.table("http://example.com_fts").schema + db.table("http://example.com").drop() def test_enable_fts_replace(db_path): db = Database(db_path) - assert db["Gosh"].detect_fts() is None + assert db.table("Gosh").detect_fts() is None result = CliRunner().invoke( cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] ) assert result.exit_code == 0 - assert "Gosh_fts" == db["Gosh"].detect_fts() - assert db["Gosh_fts"].columns_dict == {"c1": str} + assert "Gosh_fts" == db.table("Gosh").detect_fts() + assert db.table("Gosh_fts").columns_dict == {"c1": str} # This should throw an error result2 = CliRunner().invoke( @@ -589,11 +589,11 @@ def test_enable_fts_replace(db_path): cli.cli, ["enable-fts", db_path, "Gosh", "c2", "--fts4", "--replace"] ) assert result3.exit_code == 0 - assert db["Gosh_fts"].columns_dict == {"c2": str} + assert db.table("Gosh_fts").columns_dict == {"c2": str} def test_enable_fts_with_triggers(db_path): - Database(db_path)["Gosh"].insert_all([{"c1": "baz"}]) + Database(db_path).table("Gosh").insert_all([{"c1": "baz"}]) exit_code = ( CliRunner() .invoke( @@ -612,12 +612,12 @@ def test_enable_fts_with_triggers(db_path): ) assert [("baz",)] == search("baz") - Database(db_path)["Gosh"].insert_all([{"c1": "martha"}]) + Database(db_path).table("Gosh").insert_all([{"c1": "martha"}]) assert [("martha",)] == search("martha") def test_populate_fts(db_path): - Database(db_path)["Gosh"].insert_all([{"c1": "baz"}]) + Database(db_path).table("Gosh").insert_all([{"c1": "baz"}]) exit_code = ( CliRunner() .invoke(cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"]) @@ -633,7 +633,7 @@ def test_populate_fts(db_path): ) assert [("baz",)] == search("baz") - Database(db_path)["Gosh"].insert_all([{"c1": "martha"}]) + Database(db_path).table("Gosh").insert_all([{"c1": "martha"}]) assert [] == search("martha") exit_code = ( CliRunner().invoke(cli.cli, ["populate-fts", db_path, "Gosh", "c1"]).exit_code @@ -645,7 +645,7 @@ def test_populate_fts(db_path): def test_disable_fts(db_path): db = Database(db_path) assert {"Gosh", "Gosh2"} == set(db.table_names()) - db["Gosh"].enable_fts(["c1"], create_triggers=True) + db.table("Gosh").enable_fts(["c1"], create_triggers=True) assert { "Gosh_fts", "Gosh_fts_idx", @@ -677,7 +677,7 @@ def test_optimize(db_path, tables): db = Database(db_path) with db.conn: for table in ("Gosh", "Gosh2"): - db[table].insert_all( + db.table(table).insert_all( [ { "c1": f"verb{i}", @@ -687,8 +687,8 @@ def test_optimize(db_path, tables): for i in range(10000) ] ) - db["Gosh"].enable_fts(["c1", "c2", "c3"], fts_version="FTS4") - db["Gosh2"].enable_fts(["c1", "c2", "c3"], fts_version="FTS5") + db.table("Gosh").enable_fts(["c1", "c2", "c3"], fts_version="FTS4") + db.table("Gosh2").enable_fts(["c1", "c2", "c3"], fts_version="FTS5") size_before_optimize = os.stat(db_path).st_size result = CliRunner().invoke(cli.cli, ["optimize", db_path] + tables) assert result.exit_code == 0 @@ -713,22 +713,22 @@ def test_rebuild_fts_fixes_docsize_error(db_path): for i in range(10000) ] with db.conn: - db["fts5_table"].insert_all(records, pk="c1") - db["fts5_table"].enable_fts( + db.table("fts5_table").insert_all(records, pk="c1") + db.table("fts5_table").enable_fts( ["c1", "c2", "c3"], fts_version="FTS5", create_triggers=True ) # Search should work - assert list(db["fts5_table"].search("verb1")) + assert list(db.table("fts5_table").search("verb1")) # Replicate docsize error from this issue for FTS5 # https://github.com/simonw/sqlite-utils/issues/149 - assert db["fts5_table_fts_docsize"].count == 10000 - db["fts5_table"].insert_all(records, replace=True) - assert db["fts5_table"].count == 10000 - assert db["fts5_table_fts_docsize"].count == 20000 + assert db.table("fts5_table_fts_docsize").count == 10000 + db.table("fts5_table").insert_all(records, replace=True) + assert db.table("fts5_table").count == 10000 + assert db.table("fts5_table_fts_docsize").count == 20000 # Running rebuild-fts should fix this result = CliRunner().invoke(cli.cli, ["rebuild-fts", db_path, "fts5_table"]) assert result.exit_code == 0 - assert db["fts5_table_fts_docsize"].count == 10000 + assert db.table("fts5_table_fts_docsize").count == 10000 @pytest.mark.parametrize( @@ -741,7 +741,7 @@ def test_rebuild_fts_fixes_docsize_error(db_path): def test_query_csv(db_path, format, expected): db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}, @@ -793,7 +793,7 @@ _one_query = "select id, name, age from dogs where id = 1" def test_query_json(db_path, sql, args, expected): db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}, @@ -807,7 +807,7 @@ def test_query_sql_from_stdin(db_path): # https://github.com/simonw/sqlite-utils/issues/765 db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}, @@ -1004,7 +1004,7 @@ LOREM_IPSUM_COMPRESSED = ( def test_query_json_binary(db_path): db = Database(db_path) with db.conn: - db["files"].insert( + db.table("files").insert( { "name": "lorem.txt", "sz": 16984, @@ -1059,7 +1059,7 @@ def test_query_params(db_path, sql, params, expected): def test_query_json_with_json_cols(db_path): db = Database(db_path) with db.conn: - db["dogs"].insert( + db.table("dogs").insert( { "id": 1, "name": "Cleo", @@ -1088,7 +1088,7 @@ def test_query_json_with_json_cols(db_path): def test_query_json_unicode_not_escaped_by_default(db_path): db = Database(db_path) with db.conn: - db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + db.table("text").insert({"id": 1, "text": "Japanese 日本語"}, pk="id") result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text"]) assert result.exit_code == 0 assert result.output.strip() == '[{"id": 1, "text": "Japanese 日本語"}]' @@ -1102,7 +1102,7 @@ def test_query_json_unicode_not_escaped_by_default(db_path): def test_query_json_ascii_option(db_path, command): db = Database(db_path) with db.conn: - db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + db.table("text").insert({"id": 1, "text": "Japanese 日本語"}, pk="id") if command == "query": args = [db_path, "select id, text from text", "--ascii"] else: @@ -1118,7 +1118,7 @@ def test_query_json_ascii_option(db_path, command): [(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)], ) def test_query_raw(db_path, content, is_binary): - Database(db_path)["files"].insert({"content": content}) + Database(db_path).table("files").insert({"content": content}) result = CliRunner().invoke( cli.cli, [db_path, "select content from files", "--raw"] ) @@ -1133,7 +1133,7 @@ def test_query_raw(db_path, content, is_binary): [(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)], ) def test_query_raw_lines(db_path, content, is_binary): - Database(db_path)["files"].insert_all({"content": content} for _ in range(3)) + Database(db_path).table("files").insert_all({"content": content} for _ in range(3)) result = CliRunner().invoke( cli.cli, [db_path, "select content from files", "--raw-lines"] ) @@ -1215,7 +1215,7 @@ def test_query_memory_does_not_create_file(tmpdir): def test_rows(db_path, args, expected): db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}, @@ -1240,7 +1240,7 @@ def test_upsert(db_path, tmpdir): catch_exceptions=False, ) assert result.exit_code == 0, result.output - assert 2 == db["dogs"].count + assert 2 == db.table("dogs").count # Now run the upsert to update just their ages upsert_dogs = [ {"id": 1, "age": 5}, @@ -1295,8 +1295,8 @@ def test_upsert_pk_inferred_from_existing_table(db_path, tmpdir): def test_upsert_analyze(db_path, tmpdir): db = Database(db_path) - db["rows"].insert({"id": 1, "foo": "x", "n": 3}, pk="id") - db["rows"].create_index(["n"]) + db.table("rows").insert({"id": 1, "foo": "x", "n": 3}, pk="id") + db.table("rows").create_index(["n"]) assert "sqlite_stat1" not in db.table_names() result = CliRunner().invoke( cli.cli, @@ -1310,7 +1310,7 @@ def test_upsert_analyze(db_path, tmpdir): def test_upsert_flatten(tmpdir): db_path = str(tmpdir / "flat.db") db = Database(db_path) - db["upsert_me"].insert({"id": 1, "name": "Example"}, pk="id") + db.table("upsert_me").insert({"id": 1, "name": "Example"}, pk="id") result = CliRunner().invoke( cli.cli, ["upsert", db_path, "upsert_me", "-", "--flatten", "--pk", "id", "--alter"], @@ -1424,7 +1424,7 @@ def test_create_table(args, schema): ) assert result.exit_code == 0 db = Database("test.db") - assert schema == db["t"].schema + assert schema == db.table("t").schema def test_create_table_foreign_key(): @@ -1459,21 +1459,21 @@ def test_create_table_foreign_key(): ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT\n' ")" - ) == db["authors"].schema + ) == db.table("authors").schema assert ( 'CREATE TABLE "books" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "title" TEXT,\n' ' "author_id" INTEGER REFERENCES "authors"("id")\n' ")" - ) == db["books"].schema + ) == db.table("books").schema def test_create_table_error_if_table_exists(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) result = runner.invoke( cli.cli, ["create-table", "test.db", "dogs", "id", "integer"] ) @@ -1488,24 +1488,24 @@ def test_create_table_ignore(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) result = runner.invoke( cli.cli, ["create-table", "test.db", "dogs", "id", "integer", "--ignore"] ) assert result.exit_code == 0 - assert 'CREATE TABLE "dogs" (\n "name" TEXT\n)' == db["dogs"].schema + assert 'CREATE TABLE "dogs" (\n "name" TEXT\n)' == db.table("dogs").schema def test_create_table_replace(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) result = runner.invoke( cli.cli, ["create-table", "test.db", "dogs", "id", "integer", "--replace"] ) assert result.exit_code == 0 - assert 'CREATE TABLE "dogs" (\n "id" INTEGER\n)' == db["dogs"].schema + assert 'CREATE TABLE "dogs" (\n "id" INTEGER\n)' == db.table("dogs").schema def test_create_view(): @@ -1517,7 +1517,8 @@ def test_create_view(): ) assert result.exit_code == 0 assert ( - 'CREATE VIEW "version" AS select sqlite_version()' == db["version"].schema + 'CREATE VIEW "version" AS select sqlite_version()' + == db.view("version").schema ) @@ -1554,7 +1555,7 @@ def test_create_view_ignore(): assert result.exit_code == 0 assert ( 'CREATE VIEW "version" AS select sqlite_version() + 1' - == db["version"].schema + == db.view("version").schema ) @@ -1575,7 +1576,8 @@ def test_create_view_replace(): ) assert result.exit_code == 0 assert ( - 'CREATE VIEW "version" AS select sqlite_version()' == db["version"].schema + 'CREATE VIEW "version" AS select sqlite_version()' + == db.view("version").schema ) @@ -1583,7 +1585,7 @@ def test_drop_table(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["t"].create({"pk": int}, pk="pk") + db.table("t").create({"pk": int}, pk="pk") assert "t" in db.table_names() result = runner.invoke( cli.cli, @@ -1601,7 +1603,7 @@ def test_drop_table_error(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["t"].create({"pk": int}, pk="pk") + db.table("t").create({"pk": int}, pk="pk") result = runner.invoke( cli.cli, [ @@ -1624,7 +1626,7 @@ def test_drop_table_on_view_errors(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["t"].insert({"id": 1}) + db.table("t").insert({"id": 1}) db.create_view("v", "select * from t") result = runner.invoke(cli.cli, ["drop-table", "test.db", "v"]) assert result.exit_code == 1 @@ -1660,7 +1662,7 @@ def test_drop_view_on_table_errors(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["t"].insert({"id": 1}) + db.table("t").insert({"id": 1}) result = runner.invoke(cli.cli, ["drop-view", "test.db", "t"]) assert result.exit_code == 1 assert 'Error: "t" is a table, not a view - use drop-table to drop it' == ( @@ -1677,7 +1679,7 @@ def test_drop_view_error(): runner = CliRunner() with runner.isolated_filesystem(): db = Database("test.db") - db["t"].create({"pk": int}, pk="pk") + db.table("t").create({"pk": int}, pk="pk") result = runner.invoke( cli.cli, [ @@ -1702,7 +1704,7 @@ def test_enable_wal(): with runner.isolated_filesystem(): for dbname in dbs: db = Database(dbname) - db["t"].create({"pk": int}, pk="pk") + db.table("t").create({"pk": int}, pk="pk") assert db.journal_mode == "delete" result = runner.invoke(cli.cli, ["enable-wal"] + dbs, catch_exceptions=False) assert result.exit_code == 0 @@ -1717,7 +1719,7 @@ def test_disable_wal(): with runner.isolated_filesystem(): for dbname in dbs: db = Database(dbname) - db["t"].create({"pk": int}, pk="pk") + db.table("t").create({"pk": int}, pk="pk") db.enable_wal() assert db.journal_mode == "wal" result = runner.invoke(cli.cli, ["disable-wal"] + dbs) @@ -1740,7 +1742,7 @@ def test_disable_wal(): def test_query_update(db_path, args, expected): db = Database(db_path) with db.conn: - db["dogs"].insert_all( + db.table("dogs").insert_all( [ {"id": 1, "age": 4, "name": "Cleo"}, ] @@ -1756,11 +1758,13 @@ def test_query_update(db_path, args, expected): def test_add_foreign_keys(db_path): db = Database(db_path) - db["countries"].insert({"id": 7, "name": "Panama"}, pk="id") - db["authors"].insert({"id": 3, "name": "Matilda", "country_id": 7}, pk="id") - db["books"].insert({"id": 2, "title": "Wolf anatomy", "author_id": 3}, pk="id") - assert db["authors"].foreign_keys == [] - assert db["books"].foreign_keys == [] + db.table("countries").insert({"id": 7, "name": "Panama"}, pk="id") + db.table("authors").insert({"id": 3, "name": "Matilda", "country_id": 7}, pk="id") + db.table("books").insert( + {"id": 2, "title": "Wolf anatomy", "author_id": 3}, pk="id" + ) + assert db.table("authors").foreign_keys == [] + assert db.table("books").foreign_keys == [] result = CliRunner().invoke( cli.cli, [ @@ -1777,7 +1781,7 @@ def test_add_foreign_keys(db_path): ], ) assert result.exit_code == 0 - assert db["authors"].foreign_keys == [ + assert db.table("authors").foreign_keys == [ ForeignKey( table="authors", column="country_id", @@ -1785,7 +1789,7 @@ def test_add_foreign_keys(db_path): other_column="id", ) ] - assert db["books"].foreign_keys == [ + assert db.table("books").foreign_keys == [ ForeignKey( table="books", column="author_id", other_table="authors", other_column="id" ) @@ -1909,7 +1913,7 @@ def test_add_foreign_keys(db_path): def test_transform(db_path, args, expected_schema): db = Database(db_path) with db.conn: - db["dogs"].insert( + db.table("dogs").insert( {"id": 1, "age": 4, "name": "Cleo"}, not_null={"age"}, defaults={"age": 1}, @@ -1918,20 +1922,20 @@ def test_transform(db_path, args, expected_schema): result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs"] + args) print(result.output) assert result.exit_code == 0 - schema = db["dogs"].schema + schema = db.table("dogs").schema assert schema == expected_schema def test_transform_sql(db_path): db = Database(db_path) with db.conn: - db["dogs"].insert( + db.table("dogs").insert( {"id": 1, "age": 4, "name": "Cleo"}, not_null={"age"}, defaults={"age": 1}, pk="id", ) - original_schema = db["dogs"].schema + original_schema = db.table("dogs").schema result = CliRunner().invoke( cli.cli, ["transform", db_path, "dogs", "--drop", "name", "--sql"] @@ -1942,7 +1946,7 @@ def test_transform_sql(db_path): assert '"age" INTEGER NOT NULL DEFAULT' in result.output assert 'DROP TABLE "dogs";' in result.output assert 'ALTER TABLE "dogs_new_' in result.output - assert db["dogs"].schema == original_schema + assert db.table("dogs").schema == original_schema @pytest.mark.parametrize( @@ -1958,12 +1962,12 @@ def test_transform_strict_option(db_path, initial_strict, args, expected_strict) db = Database(db_path) if not db.supports_strict: pytest.skip("SQLite version does not support strict tables") - db["dogs"].create({"id": int}, strict=initial_strict) + db.table("dogs").create({"id": int}, strict=initial_strict) result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs"] + args) assert result.exit_code == 0, result.output - assert db["dogs"].strict is expected_strict + assert db.table("dogs").strict is expected_strict @pytest.mark.parametrize( @@ -1977,20 +1981,20 @@ def test_transform_strict_option_sql(db_path, initial_strict, flag, sql_is_stric db = Database(db_path) if not db.supports_strict: pytest.skip("SQLite version does not support strict tables") - db["dogs"].create({"id": int}, strict=initial_strict) + db.table("dogs").create({"id": int}, strict=initial_strict) result = CliRunner().invoke(cli.cli, ["transform", db_path, "dogs", flag, "--sql"]) assert result.exit_code == 0, result.output assert (") STRICT;" in result.output) is sql_is_strict - assert db["dogs"].strict is initial_strict + assert db.table("dogs").strict is initial_strict def test_transform_strict_option_with_invalid_data(db_path): db = Database(db_path) if not db.supports_strict: pytest.skip("SQLite version does not support strict tables") - dogs = db["dogs"] + dogs = db.table("dogs") dogs.create({"id": int}) dogs.insert({"id": "not-an-integer"}) @@ -2048,10 +2052,10 @@ def test_transform_add_or_drop_foreign_key(db_path, extra_args, expected_schema) db = Database(db_path) with db.conn: # Create table with three foreign keys so we can drop two of them - db["continent"].insert({"id": 1, "name": "Europe"}, pk="id") - db["country"].insert({"id": 1, "name": "France"}, pk="id") - db["city"].insert({"id": 24, "name": "Paris"}, pk="id") - db["places"].insert( + db.table("continent").insert({"id": 1, "name": "Europe"}, pk="id") + db.table("country").insert({"id": 1, "name": "France"}, pk="id") + db.table("city").insert({"id": 24, "name": "Paris"}, pk="id") + db.table("places").insert( { "id": 32, "name": "Caveau de la Huchette", @@ -2072,7 +2076,7 @@ def test_transform_add_or_drop_foreign_key(db_path, extra_args, expected_schema) + extra_args, ) assert result.exit_code == 0 - schema = db["places"].schema + schema = db.table("places").schema assert schema == expected_schema @@ -2133,7 +2137,7 @@ _common_other_schema = ( def test_extract(db_path, args, expected_table_schema, expected_other_schema): db = Database(db_path) with db.conn: - db["trees"].insert( + db.table("trees").insert( {"id": 1, "address": "4 Park Ave", "species": "Palm"}, pk="id", ) @@ -2142,7 +2146,7 @@ def test_extract(db_path, args, expected_table_schema, expected_other_schema): ) print(result.output) assert result.exit_code == 0 - schema = db["trees"].schema + schema = db.table("trees").schema assert schema == expected_table_schema other_schema = next( t for t in db.tables if t.name not in ("trees", "Gosh", "Gosh2") @@ -2190,7 +2194,7 @@ def test_insert_encoding(tmpdir): ) assert good_result.exit_code == 0 db = Database(db_path) - assert list(db["places"].rows) == [ + assert list(db.table("places").rows) == [ { "date": "2020-01-01", "name": "Barra da Lagoa", @@ -2226,7 +2230,7 @@ def test_insert_encoding(tmpdir): def test_search(tmpdir, fts, extra_arg, expected): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["articles"].insert_all( + db.table("articles").insert_all( [ {"id": 1, "title": "Title the first"}, {"id": 2, "title": "Title the second"}, @@ -2234,7 +2238,7 @@ def test_search(tmpdir, fts, extra_arg, expected): ], pk="id", ) - db["articles"].enable_fts(["title"], fts_version=fts) + db.table("articles").enable_fts(["title"], fts_version=fts) result = CliRunner().invoke( cli.cli, ["search", db_path, "articles", "second"] + ([extra_arg] if extra_arg else []), @@ -2247,7 +2251,7 @@ def test_search(tmpdir, fts, extra_arg, expected): def test_search_quote(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["creatures"].insert({"name": "dog."}).enable_fts(["name"]) + db.table("creatures").insert({"name": "dog."}).enable_fts(["name"]) # Without --quote should return an error error_result = CliRunner().invoke(cli.cli, ["search", db_path, "creatures", 'dog"']) assert error_result.exit_code == 1 @@ -2355,11 +2359,11 @@ _TRIGGERS_EXPECTED = ( def test_triggers(tmpdir, extra_args, expected): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["articles"].insert( + db.table("articles").insert( {"id": 1, "title": "Title the first"}, pk="id", ) - db["counter"].insert({"count": 1}) + db.table("counter").insert({"count": 1}) db.conn.execute(textwrap.dedent(""" CREATE TRIGGER blah AFTER INSERT ON articles BEGIN @@ -2420,9 +2424,9 @@ def test_triggers(tmpdir, extra_args, expected): def test_schema(tmpdir, options, expected): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["dogs"].create({"id": int, "name": str}) - db["chickens"].create({"id": int, "name": str, "breed": str}) - db["chickens"].create_index(["breed"]) + db.table("dogs").create({"id": int, "name": str}) + db.table("chickens").create({"id": int, "name": str, "breed": str}) + db.table("chickens").create_index(["breed"]) result = CliRunner().invoke( cli.cli, ["schema", db_path] + options, @@ -2446,7 +2450,7 @@ def test_long_csv_column_value(tmpdir): ) assert result.exit_code == 0 db = Database(db_path) - rows = list(db["bigtable"].rows) + rows = list(db.table("bigtable").rows) assert len(rows) == 1 assert rows[0]["text"] == long_string @@ -2473,7 +2477,7 @@ def test_import_no_headers(tmpdir, args, tsv): ) assert result.exit_code == 0, result.output db = Database(db_path) - schema = db["creatures"].schema + schema = db.table("creatures").schema assert schema == ( 'CREATE TABLE "creatures" (\n' ' "untitled_1" TEXT,\n' @@ -2481,7 +2485,7 @@ def test_import_no_headers(tmpdir, args, tsv): ' "untitled_3" TEXT\n' ")" ) - rows = list(db["creatures"].rows) + rows = list(db.table("creatures").rows) assert rows == [ {"untitled_1": "Cleo", "untitled_2": "Dog", "untitled_3": "5"}, {"untitled_1": "Tracy", "untitled_2": "Spider", "untitled_3": "7"}, @@ -2493,10 +2497,10 @@ def test_attach(tmpdir): bar_path = str(tmpdir / "bar.db") db = Database(foo_path) with db.conn: - db["foo"].insert({"id": 1, "text": "foo"}) + db.table("foo").insert({"id": 1, "text": "foo"}) db2 = Database(bar_path) with db2.conn: - db2["bar"].insert({"id": 1, "text": "bar"}) + db2.table("bar").insert({"id": 1, "text": "bar"}) db.attach("bar", bar_path) sql = "select * from foo union all select * from bar.bar" result = CliRunner().invoke( @@ -2557,7 +2561,7 @@ def test_insert_detect_types(tmpdir): ) assert result.exit_code == 0 db = Database(db_path) - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"name": "Cleo", "age": 6, "weight": 45.5}, {"name": "Dori", "age": 1, "weight": 3.5}, ] @@ -2589,7 +2593,7 @@ def test_upsert_detect_types(tmpdir): ) assert result.exit_code == 0 db = Database(db_path) - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"id": 1, "name": "Cleo", "age": 6, "weight": 45.5}, {"id": 2, "name": "Dori", "age": 1, "weight": 3.5}, ] @@ -2608,7 +2612,7 @@ def test_csv_detect_types_creates_real_columns(tmpdir): assert result.exit_code == 0 db = Database(db_path) # Check that the schema uses REAL for the weight column - assert db["creatures"].schema == ( + assert db.table("creatures").schema == ( 'CREATE TABLE "creatures" (\n' ' "name" TEXT,\n' ' "age" INTEGER,\n' @@ -2630,11 +2634,11 @@ def test_insert_no_detect_types(tmpdir): assert result.exit_code == 0 db = Database(db_path) # All columns should be TEXT when --no-detect-types is used - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"name": "Cleo", "age": "6", "weight": "45.5"}, {"name": "Dori", "age": "1", "weight": "3.5"}, ] - assert db["creatures"].schema == ( + assert db.table("creatures").schema == ( 'CREATE TABLE "creatures" (\n' ' "name" TEXT,\n' ' "age" TEXT,\n' @@ -2665,11 +2669,11 @@ def test_upsert_no_detect_types(tmpdir): assert result.exit_code == 0 db = Database(db_path) # All columns should be TEXT when --no-detect-types is used - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"id": "1", "name": "Cleo", "age": "6", "weight": "45.5"}, {"id": "2", "name": "Dori", "age": "1", "weight": "3.5"}, ] - assert db["creatures"].schema == ( + assert db.table("creatures").schema == ( 'CREATE TABLE "creatures" (\n' ' "id" TEXT PRIMARY KEY,\n' ' "name" TEXT,\n' @@ -2751,20 +2755,20 @@ def test_create_database(tmpdir, enable_wal): def test_analyze(tmpdir, options, expected): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id") - db["one_index"].create_index(["name"]) - db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") - db["two_indexes"].create_index(["name"]) - db["two_indexes"].create_index(["species"]) + db.table("one_index").insert({"id": 1, "name": "Cleo"}, pk="id") + db.table("one_index").create_index(["name"]) + db.table("two_indexes").insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") + db.table("two_indexes").create_index(["name"]) + db.table("two_indexes").create_index(["species"]) result = CliRunner().invoke(cli.cli, ["analyze", db_path] + options) assert result.exit_code == 0 - assert list(db["sqlite_stat1"].rows) == expected + assert list(db.table("sqlite_stat1").rows) == expected def test_rename_table(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["one"].insert({"id": 1, "name": "Cleo"}, pk="id") + db.table("one").insert({"id": 1, "name": "Cleo"}, pk="id") # First try a non-existent table result_error = CliRunner().invoke( cli.cli, @@ -2782,7 +2786,7 @@ def test_rename_table(tmpdir): catch_exceptions=False, ) assert result_error2.exit_code == 0 - previous_columns = db["one"].columns_dict + previous_columns = db.table("one").columns_dict # Now try for a table that exists result = CliRunner().invoke( cli.cli, @@ -2790,13 +2794,13 @@ def test_rename_table(tmpdir): catch_exceptions=False, ) assert result.exit_code == 0 - assert db["two"].columns_dict == previous_columns + assert db.table("two").columns_dict == previous_columns def test_duplicate_table(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["one"].insert({"id": 1, "name": "Cleo"}, pk="id") + db.table("one").insert({"id": 1, "name": "Cleo"}, pk="id") # First try a non-existent table result_error = CliRunner().invoke( cli.cli, @@ -2819,8 +2823,8 @@ def test_duplicate_table(tmpdir): catch_exceptions=False, ) assert result.exit_code == 0 - assert db["one"].columns_dict == db["two"].columns_dict - assert list(db["one"].rows) == list(db["two"].rows) + assert db.table("one").columns_dict == db.table("two").columns_dict + assert list(db.table("one").rows) == list(db.table("two").rows) @pytest.mark.skipif(not _has_compiled_ext(), reason="Requires compiled ext.c") @@ -2863,9 +2867,9 @@ def test_create_table_strict(strict): + (["--strict"] if strict else []), ) assert result.exit_code == 0 - assert db["items"].strict == strict or not db.supports_strict + assert db.table("items").strict == strict or not db.supports_strict # Should have a floating point column - assert db["items"].columns_dict == {"id": int, "w": float} + assert db.table("items").columns_dict == {"id": int, "w": float} @pytest.mark.parametrize("method", ("insert", "upsert")) @@ -2880,12 +2884,12 @@ def test_insert_upsert_strict(tmpdir, method, strict): ) assert result.exit_code == 0 db = Database(db_path) - assert db["items"].strict == strict or not db.supports_strict + assert db.table("items").strict == strict or not db.supports_strict def test_extract_bad_column_clean_error(db_path): db = Database(db_path) - db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") + db.table("trees").insert({"id": 1, "species": "Palm"}, pk="id") result = CliRunner().invoke(cli.cli, ["extract", db_path, "trees", "nope"]) assert result.exit_code == 1 assert result.exception is None or isinstance(result.exception, SystemExit) @@ -2894,7 +2898,7 @@ def test_extract_bad_column_clean_error(db_path): def test_extract_view_clean_error(db_path): db = Database(db_path) - db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") + db.table("trees").insert({"id": 1, "species": "Palm"}, pk="id") db.create_view("v", "select * from trees") result = CliRunner().invoke(cli.cli, ["extract", db_path, "v", "species"]) assert result.exit_code == 1 diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py index 932269b..24889b3 100644 --- a/tests/test_cli_bulk.py +++ b/tests/test_cli_bulk.py @@ -13,7 +13,7 @@ from sqlite_utils import Database, cli def test_db_and_path(tmpdir): db_path = str(pathlib.Path(tmpdir) / "data.db") db = Database(db_path) - db["example"].insert_all( + db.table("example").insert_all( [ {"id": 1, "name": "One"}, {"id": 2, "name": "Two"}, @@ -44,7 +44,7 @@ def test_cli_bulk(test_db_and_path): {"id": 2, "name": "Two"}, {"id": 3, "name": "THREE"}, {"id": 4, "name": "FOUR"}, - ] == list(db["example"].rows) + ] == list(db.table("example").rows) def test_cli_bulk_multiple_functions(test_db_and_path): @@ -70,7 +70,7 @@ def test_cli_bulk_multiple_functions(test_db_and_path): {"id": 2, "name": "Two"}, {"id": 3, "name": "THREE"}, {"id": 4, "name": "FOUR"}, - ] == list(db["example"].rows) + ] == list(db.table("example").rows) def test_cli_bulk_batch_size(test_db_and_path): @@ -95,13 +95,13 @@ def test_cli_bulk_batch_size(test_db_and_path): proc.stdin.write(b'{"id": 3, "name": "Three"}\n\n') proc.stdin.flush() time.sleep(1) - assert db["example"].count == 2 + assert db.table("example").count == 2 # Writing another should trigger a commit: proc.stdin.write(b'{"id": 4, "name": "Four"}\n\n') proc.stdin.flush() time.sleep(1) - assert db["example"].count == 4 + assert db.table("example").count == 4 proc.stdin.close() proc.wait() diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 65543b1..1101f0f 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -12,7 +12,7 @@ from sqlite_utils import cli @pytest.fixture def test_db_and_path(fresh_db_and_path): db, db_path = fresh_db_and_path - db["example"].insert_all( + db.table("example").insert_all( [ {"id": 1, "dt": "5th October 2019 12:04"}, {"id": 2, "dt": "6th October 2019 00:05:06"}, @@ -47,12 +47,12 @@ def fresh_db_and_path(tmpdir): ) def test_convert_code(fresh_db_and_path, code): db, db_path = fresh_db_and_path - db["t"].insert({"text": "October"}) + db.table("t").insert({"text": "October"}) result = CliRunner().invoke( cli.cli, ["convert", db_path, "t", "text", code], catch_exceptions=False ) assert result.exit_code == 0, result.output - value = next(iter(db["t"].rows))["text"] + value = next(iter(db.table("t").rows))["text"] assert value == "Spooktober" @@ -65,7 +65,7 @@ def test_convert_code(fresh_db_and_path, code): ) def test_convert_code_errors(fresh_db_and_path, bad_code): db, db_path = fresh_db_and_path - db["t"].insert({"text": "October"}) + db.table("t").insert({"text": "October"}) result = CliRunner().invoke( cli.cli, ["convert", db_path, "t", "text", bad_code], catch_exceptions=False ) @@ -93,12 +93,12 @@ def test_convert_import(test_db_and_path): {"id": 2, "dt": "6th OXXober 2019 00:05:06"}, {"id": 3, "dt": ""}, {"id": 4, "dt": None}, - ] == list(db["example"].rows) + ] == list(db.table("example").rows) def test_convert_import_nested(fresh_db_and_path): db, db_path = fresh_db_and_path - db["example"].insert({"xml": ''}) + db.table("example").insert({"xml": ''}) result = CliRunner().invoke( cli.cli, [ @@ -114,7 +114,7 @@ def test_convert_import_nested(fresh_db_and_path): assert result.exit_code == 0, result.output assert [ {"xml": "Cleo"}, - ] == list(db["example"].rows) + ] == list(db.table("example").rows) def test_convert_dryrun(test_db_and_path): @@ -152,7 +152,7 @@ def test_convert_dryrun(test_db_and_path): "Would affect 4 rows" ) # But it should not have actually modified the table data - assert list(db["example"].rows) == [ + assert list(db.table("example").rows) == [ {"id": 1, "dt": "5th October 2019 12:04"}, {"id": 2, "dt": "6th October 2019 00:05:06"}, {"id": 3, "dt": ""}, @@ -269,7 +269,7 @@ def test_convert_output_column(test_db_and_path, drop): if drop: for row in expected: del row["dt"] - assert list(db["example"].rows) == expected + assert list(db.table("example").rows) == expected @pytest.mark.parametrize( @@ -352,7 +352,7 @@ def test_convert_output_error(test_db_and_path, options, expected_error): @pytest.mark.parametrize("drop", (True, False)) def test_convert_multi(fresh_db_and_path, drop): db, db_path = fresh_db_and_path - db["creatures"].insert_all( + db.table("creatures").insert_all( [ {"id": 1, "name": "Simon"}, {"id": 2, "name": "Cleo"}, @@ -378,12 +378,12 @@ def test_convert_multi(fresh_db_and_path, drop): if drop: for row in expected: del row["name"] - assert list(db["creatures"].rows) == expected + assert list(db.table("creatures").rows) == expected def test_convert_multi_complex_column_types(fresh_db_and_path): db, db_path = fresh_db_and_path - db["rows"].insert_all( + db.table("rows").insert_all( [ {"id": 1}, {"id": 2}, @@ -412,7 +412,7 @@ def test_convert_multi_complex_column_types(fresh_db_and_path): ], ) assert result.exit_code == 0, result.output - assert list(db["rows"].rows) == [ + assert list(db.table("rows").rows) == [ {"id": 1, "is_str": "", "is_float": 1.2, "is_int": None, "is_bytes": None}, {"id": 2, "is_str": None, "is_float": 1.0, "is_int": 12, "is_bytes": None}, { @@ -424,7 +424,7 @@ def test_convert_multi_complex_column_types(fresh_db_and_path): }, {"id": 4, "is_str": None, "is_float": None, "is_int": None, "is_bytes": None}, ] - assert db["rows"].schema == ( + assert db.table("rows").schema == ( 'CREATE TABLE "rows" (\n' ' "id" INTEGER PRIMARY KEY\n' ', "is_str" TEXT, "is_float" REAL, "is_int" INTEGER, "is_bytes" BLOB)' @@ -435,7 +435,7 @@ def test_convert_multi_complex_column_types(fresh_db_and_path): def test_recipe_jsonsplit(tmpdir, delimiter): db_path = str(pathlib.Path(tmpdir) / "data.db") db = sqlite_utils.Database(db_path) - db["example"].insert_all( + db.table("example").insert_all( [ {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])}, {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])}, @@ -448,7 +448,7 @@ def test_recipe_jsonsplit(tmpdir, delimiter): args = ["convert", db_path, "example", "tags", code] result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0, result.output - assert list(db["example"].rows) == [ + assert list(db.table("example").rows) == [ {"id": 1, "tags": '["foo", "bar"]'}, {"id": 2, "tags": '["bar", "baz"]'}, ] @@ -464,7 +464,7 @@ def test_recipe_jsonsplit(tmpdir, delimiter): ) def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array): db, db_path = fresh_db_and_path - db["example"].insert_all( + db.table("example").insert_all( [ {"id": 1, "records": "1,2,3"}, ], @@ -476,13 +476,13 @@ def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array): args = ["convert", db_path, "example", "records", code] result = CliRunner().invoke(cli.cli, args) assert result.exit_code == 0, result.output - assert json.loads(db["example"].get(1)["records"]) == expected_array + assert json.loads(db.table("example").get(1)["records"]) == expected_array @pytest.mark.parametrize("drop", (True, False)) def test_recipe_jsonsplit_output(fresh_db_and_path, drop): db, db_path = fresh_db_and_path - db["example"].insert_all( + db.table("example").insert_all( [ {"id": 1, "records": "1,2,3"}, ], @@ -501,7 +501,7 @@ def test_recipe_jsonsplit_output(fresh_db_and_path, drop): } if drop: del expected["records"] - assert db["example"].get(1) == expected + assert db.table("example").get(1) == expected def test_cannot_use_drop_without_multi_or_output(fresh_db_and_path): @@ -558,7 +558,7 @@ def test_convert_where(test_db_and_path): ], ) assert result.exit_code == 0, result.output - assert list(db["example"].rows) == [ + assert list(db.table("example").rows) == [ {"id": 1, "dt": "5th October 2019 12:04"}, {"id": 2, "dt": "6TH OCTOBER 2019 00:05:06"}, {"id": 3, "dt": ""}, @@ -568,7 +568,7 @@ def test_convert_where(test_db_and_path): def test_convert_where_multi(fresh_db_and_path): db, db_path = fresh_db_and_path - db["names"].insert_all( + db.table("names").insert_all( [{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Bants"}], pk="id" ) result = CliRunner().invoke( @@ -588,7 +588,7 @@ def test_convert_where_multi(fresh_db_and_path): ], ) assert result.exit_code == 0, result.output - assert list(db["names"].rows) == [ + assert list(db.table("names").rows) == [ {"id": 1, "name": "Cleo", "upper": None}, {"id": 2, "name": "Bants", "upper": "BANTS"}, ] @@ -596,7 +596,7 @@ def test_convert_where_multi(fresh_db_and_path): def test_convert_code_standard_input(fresh_db_and_path): db, db_path = fresh_db_and_path - db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + db.table("names").insert_all([{"id": 1, "name": "Cleo"}], pk="id") result = CliRunner().invoke( cli.cli, [ @@ -609,27 +609,27 @@ def test_convert_code_standard_input(fresh_db_and_path): input="value.upper()", ) assert result.exit_code == 0, result.output - assert list(db["names"].rows) == [ + assert list(db.table("names").rows) == [ {"id": 1, "name": "CLEO"}, ] def test_convert_hyphen_workaround(fresh_db_and_path): db, db_path = fresh_db_and_path - db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + db.table("names").insert_all([{"id": 1, "name": "Cleo"}], pk="id") result = CliRunner().invoke( cli.cli, ["convert", db_path, "names", "name", '"-"'], ) assert result.exit_code == 0, result.output - assert list(db["names"].rows) == [ + assert list(db.table("names").rows) == [ {"id": 1, "name": "-"}, ] def test_convert_initialization_pattern(fresh_db_and_path): db, db_path = fresh_db_and_path - db["names"].insert_all([{"id": 1, "name": "Cleo"}], pk="id") + db.table("names").insert_all([{"id": 1, "name": "Cleo"}], pk="id") result = CliRunner().invoke( cli.cli, [ @@ -642,7 +642,7 @@ def test_convert_initialization_pattern(fresh_db_and_path): input="import random\nrandom.seed(1)\ndef convert(value): return random.randint(0, 100)", ) assert result.exit_code == 0, result.output - assert list(db["names"].rows) == [ + assert list(db.table("names").rows) == [ {"id": 1, "name": "17"}, ] @@ -657,13 +657,13 @@ def test_convert_handles_falsey_values(fresh_db_and_path): "x", "-", ] - db["t"].insert_all([{"x": 0}, {"x": 1}]) - assert db["t"].get(1)["x"] == 0 - assert db["t"].get(2)["x"] == 1 + db.table("t").insert_all([{"x": 0}, {"x": 1}]) + assert db.table("t").get(1)["x"] == 0 + assert db.table("t").get(2)["x"] == 1 result = CliRunner().invoke(cli.cli, args, input="value + 1") assert result.exit_code == 0, result.output - assert db["t"].get(1)["x"] == 1 - assert db["t"].get(2)["x"] == 2 + assert db.table("t").get(1)["x"] == 1 + assert db.table("t").get(2)["x"] == 2 @pytest.mark.parametrize( @@ -684,7 +684,7 @@ def test_convert_callable_reference(test_db_and_path, code): cli.cli, ["convert", db_path, "example", "dt", code], catch_exceptions=False ) assert result.exit_code == 0, result.output - rows = list(db["example"].rows) + rows = list(db.table("example").rows) assert rows[0]["dt"] == "2019-10-05" assert rows[1]["dt"] == "2019-10-06" assert rows[2]["dt"] == "" @@ -694,7 +694,7 @@ def test_convert_callable_reference(test_db_and_path, code): def test_convert_callable_reference_with_import(fresh_db_and_path): """Test callable reference from an imported module""" db, db_path = fresh_db_and_path - db["example"].insert({"id": 1, "data": '{"name": "test"}'}) + db.table("example").insert({"id": 1, "data": '{"name": "test"}'}) result = CliRunner().invoke( cli.cli, [ @@ -710,5 +710,5 @@ def test_convert_callable_reference_with_import(fresh_db_and_path): ) assert result.exit_code == 0, result.output # json.loads returns a dict, which sqlite stores as JSON string - row = db["example"].get(1) + row = db.table("example").get(1) assert row["data"] == '{"name": "test"}' diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index eefb3fa..01e7e94 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -21,7 +21,7 @@ def test_insert_simple(tmpdir): ) db = Database(db_path) assert ["dogs"] == db.table_names() - assert [] == db["dogs"].indexes + assert [] == db.table("dogs").indexes def test_insert_from_stdin(tmpdir): @@ -96,7 +96,7 @@ def test_insert_with_primary_keys(db_path, tmpdir, args, expected_pks): Database(db_path).query("select * from dogs") ) db = Database(db_path) - assert db["dogs"].pks == expected_pks + assert db.table("dogs").pks == expected_pks def test_insert_multiple_with_primary_key(db_path, tmpdir): @@ -110,7 +110,7 @@ def test_insert_multiple_with_primary_key(db_path, tmpdir): assert result.exit_code == 0 db = Database(db_path) assert dogs == list(db.query("select * from dogs order by id")) - assert ["id"] == db["dogs"].pks + assert ["id"] == db.table("dogs").pks def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): @@ -127,7 +127,7 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): assert result.exit_code == 0 db = Database(db_path) assert dogs == list(db.query("select * from dogs order by breed, id")) - assert {"breed", "id"} == set(db["dogs"].pks) + assert {"breed", "id"} == set(db.table("dogs").pks) assert ( 'CREATE TABLE "dogs" (\n' ' "breed" TEXT,\n' @@ -136,7 +136,7 @@ def test_insert_multiple_with_compound_primary_key(db_path, tmpdir): ' "age" INTEGER,\n' ' PRIMARY KEY ("id", "breed")\n' ")" - ) == db["dogs"].schema + ) == db.table("dogs").schema def test_insert_not_null_default(db_path, tmpdir): @@ -160,7 +160,7 @@ def test_insert_not_null_default(db_path, tmpdir): ' "name" TEXT NOT NULL,\n' " \"age\" INTEGER NOT NULL DEFAULT '1',\n" " \"score\" INTEGER DEFAULT '5'\n)" - ) == db["dogs"].schema + ) == db.table("dogs").schema def test_insert_binary_base64(db_path): @@ -191,7 +191,7 @@ def test_insert_newline_delimited(db_path): def test_insert_ignore(db_path, tmpdir): db = Database(db_path) - db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") json_path = str(tmpdir / "dogs.json") with open(json_path, "w") as fp: fp.write(json.dumps([{"id": 1, "name": "Bailey"}])) @@ -232,7 +232,7 @@ def test_insert_csv_tsv(content, options, db_path, tmpdir): catch_exceptions=False, ) assert result.exit_code == 0 - assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows) + assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db.table("data").rows) @pytest.mark.parametrize("empty_null", (True, False)) @@ -248,7 +248,7 @@ def test_insert_csv_empty_null(db_path, empty_null): ) assert result.exit_code == 0 db = Database(db_path) - assert [r for r in db["data"].rows] == [ + assert [r for r in db.table("data").rows] == [ {"foo": "1", "bar": None if empty_null else "", "baz": "cat"} ] @@ -302,7 +302,7 @@ def test_insert_replace(db_path, tmpdir): test_insert_multiple_with_primary_key(db_path, tmpdir) json_path = str(tmpdir / "insert-replace.json") db = Database(db_path) - assert db["dogs"].count == 20 + assert db.table("dogs").count == 20 insert_replace_dogs = [ {"id": 1, "name": "Insert replaced 1", "age": 4}, {"id": 2, "name": "Insert replaced 2", "age": 4}, @@ -314,7 +314,7 @@ def test_insert_replace(db_path, tmpdir): cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"] ) assert result.exit_code == 0, result.output - assert db["dogs"].count == 21 + assert db.table("dogs").count == 21 assert ( list(db.query("select * from dogs where id in (1, 2, 21) order by id")) == insert_replace_dogs @@ -377,7 +377,7 @@ def test_insert_alter(db_path, tmpdir): assert result.exit_code == 0, result.output # Soundness check the database itself db = Database(db_path) - assert {"foo": str, "n": int, "baz": int} == db["from_json_nl"].columns_dict + assert {"foo": str, "n": int, "baz": int} == db.table("from_json_nl").columns_dict assert [ {"foo": "bar", "n": 1, "baz": None}, {"foo": "baz", "n": 2, "baz": None}, @@ -387,8 +387,8 @@ def test_insert_alter(db_path, tmpdir): def test_insert_analyze(db_path): db = Database(db_path) - db["rows"].insert({"foo": "x", "n": 3}) - db["rows"].create_index(["n"]) + db.table("rows").insert({"foo": "x", "n": 3}) + db.table("rows").create_index(["n"]) assert "sqlite_stat1" not in db.table_names() result = CliRunner().invoke( cli.cli, @@ -583,7 +583,7 @@ def test_insert_streaming_batch_size_1(db_path): def try_until(expected): tries = 0 while True: - rows = list(Database(db_path)["rows"].rows) + rows = list(Database(db_path).table("rows").rows) if rows == expected: return tries += 1 @@ -615,13 +615,13 @@ def test_insert_csv_headers_only(tmpdir): assert result.exit_code == 0 # Table should not exist since there were no data rows db = Database(db_path) - assert not db["data"].exists() + assert not db.table("data").exists() def test_insert_into_view_errors(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["t"].insert({"id": 1}) + db.table("t").insert({"id": 1}) db.create_view("v", "select * from t") db.close() result = CliRunner().invoke( @@ -637,7 +637,7 @@ def test_insert_csv_detect_types_leaves_existing_table_alone(db_path): # table would rewrite its column types and corrupt data such as # TEXT zip codes with leading zeros db = Database(db_path) - db["places"].insert({"name": "Boston", "zip": "01234"}) + db.table("places").insert({"name": "Boston", "zip": "01234"}) result = CliRunner().invoke( cli.cli, ["insert", db_path, "places", "-", "--csv"], @@ -645,8 +645,8 @@ def test_insert_csv_detect_types_leaves_existing_table_alone(db_path): input="name,zip\nSF,94107", ) assert result.exit_code == 0, result.output - assert db["places"].columns_dict["zip"] is str - assert list(db["places"].rows) == [ + assert db.table("places").columns_dict["zip"] is str + assert list(db.table("places").rows) == [ {"name": "Boston", "zip": "01234"}, {"name": "SF", "zip": "94107"}, ] @@ -662,7 +662,7 @@ def test_insert_csv_detect_types_new_table(db_path): ) assert result.exit_code == 0, result.output db = Database(db_path) - assert db["data"].columns_dict == {"name": str, "age": int, "weight": float} + assert db.table("data").columns_dict == {"name": str, "age": int, "weight": float} @pytest.mark.parametrize( @@ -708,13 +708,13 @@ def test_insert_upsert_csv_type_overrides_detected_types( expected_columns = {"zipcode": str, "score": float} if command == "upsert": expected_columns = {"id": int, **expected_columns} - assert db["places"].columns_dict == expected_columns - assert list(db["places"].rows) == [expected_row] + assert db.table("places").columns_dict == expected_columns + assert list(db.table("places").rows) == [expected_row] def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): db = Database(db_path) - db["places"].insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id") + db.table("places").insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id") result = CliRunner().invoke( cli.cli, ["upsert", db_path, "places", "-", "--csv", "--pk", "id"], @@ -722,15 +722,15 @@ def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): input="id,name,zip\n2,SF,94107", ) assert result.exit_code == 0, result.output - assert db["places"].columns_dict["zip"] is str - assert db["places"].get(1)["zip"] == "01234" + assert db.table("places").columns_dict["zip"] is str + assert db.table("places").get(1)["zip"] == "01234" def test_insert_invalid_pk_clean_error(db_path): # An invalid --pk against an existing table should be a clean CLI # error, not a raw InvalidColumns traceback db = Database(db_path) - db["t"].insert({"a": 1}) + db.table("t").insert({"a": 1}) result = CliRunner().invoke( cli.cli, ["insert", db_path, "t", "-", "--pk", "badcol"], @@ -765,8 +765,8 @@ def test_insert_code(tmpdir, code): ) assert result.exit_code == 0, result.output db = Database(db_path) - assert db["creatures"].pks == ["id"] - assert list(db["creatures"].rows) == [ + assert db.table("creatures").pks == ["id"] + assert list(db.table("creatures").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 2, "name": "Suna"}, ] @@ -782,7 +782,7 @@ def test_insert_code_from_file(tmpdir): ["insert", db_path, "creatures", "--code", code_path], ) assert result.exit_code == 0, result.output - assert list(Database(db_path)["creatures"].rows) == [ + assert list(Database(db_path).table("creatures").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 2, "name": "Suna"}, ] @@ -791,7 +791,7 @@ def test_insert_code_from_file(tmpdir): def test_upsert_code(tmpdir): db_path = str(tmpdir / "dogs.db") db = Database(db_path) - db["creatures"].insert_all( + db.table("creatures").insert_all( [{"id": 1, "name": "old"}, {"id": 2, "name": "Suna"}], pk="id" ) result = CliRunner().invoke( @@ -799,7 +799,7 @@ def test_upsert_code(tmpdir): ["upsert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--pk", "id"], ) assert result.exit_code == 0, result.output - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 2, "name": "Suna"}, ] @@ -858,7 +858,9 @@ def test_insert_code_single_dict(tmpdir): ], ) assert result.exit_code == 0, result.output - assert list(Database(db_path)["creatures"].rows) == [{"id": 1, "name": "Cleo"}] + assert list(Database(db_path).table("creatures").rows) == [ + {"id": 1, "name": "Cleo"} + ] def test_insert_code_not_iterable(tmpdir): diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index 4fb4fb3..445f50b 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -228,7 +228,7 @@ def test_memory_save(tmpdir, extra_args): ) assert result.exit_code == 0 db = Database(save_to) - assert list(db["stdin"].rows) == [ + assert list(db.table("stdin").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 2, "name": "Bants"}, ] diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index f49ef10..7439887 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -13,11 +13,11 @@ m = Migrations("hello") @m() def foo(db): - db["foo"].insert({"hello": "world"}) + db.table("foo").insert({"hello": "world"}) @m() def bar(db): - db["bar"].insert({"hello": "world"}) + db.table("bar").insert({"hello": "world"}) """ @@ -42,21 +42,21 @@ creatures = Migrations("creatures") @creatures() def create_table(db): - db["creatures"].insert({"name": "Cleo"}) + db.table("creatures").insert({"name": "Cleo"}) @creatures() def add_weight(db): - db["creature_weights"].insert({"weight": 4.2}) + db.table("creature_weights").insert({"weight": 4.2}) sales = Migrations("sales") @sales() def create_table(db): - db["sales"].insert({"id": 1}) + db.table("sales").insert({"id": 1}) @sales() def add_weight(db): - db["sales_weights"].insert({"weight": 10}) + db.table("sales_weights").insert({"weight": 10}) """, "utf-8", ) @@ -99,10 +99,10 @@ def test_basic(two_migrations, arg): assert " Pending:\n (none)" in list_output db = sqlite_utils.Database(db_path) - assert db["foo"].exists() - assert db["bar"].exists() - assert db["_sqlite_migrations"].exists() - rows = list(db["_sqlite_migrations"].rows) + assert db.table("foo").exists() + assert db.table("bar").exists() + assert db.table("_sqlite_migrations").exists() + rows = list(db.table("_sqlite_migrations").rows) assert len(rows) == 2 assert rows[0]["name"] == "foo" assert rows[1]["name"] == "bar" @@ -113,13 +113,13 @@ def test_list_same_migration_names_in_different_sets(capsys): @applied(name="foo") def applied_foo(db): - db["applied"].insert({"hello": "world"}) + db.table("applied").insert({"hello": "world"}) pending = sqlite_utils.Migrations("pending") @pending(name="foo") def pending_foo(db): - db["pending"].insert({"hello": "world"}) + db.table("pending").insert({"hello": "world"}) db = sqlite_utils.Database(memory=True) applied.apply(db) @@ -144,7 +144,7 @@ m = Migrations("hello") @m() def foo(db): - db["dogs"].insert({"id": 1, "name": "Cleo"}) + db.table("dogs").insert({"id": 1, "name": "Cleo"}) """, "utf-8", ) @@ -184,9 +184,9 @@ Schema after: new_migration = """ @m() def bar(db): - db["dogs"].add_column("age", int) - db["dogs"].add_column("weight", float) - db["dogs"].transform() + db.table("dogs").add_column("age", int) + db.table("dogs").add_column("weight", float) + db.table("dogs").transform() """ migrations_py.write_text(migrations_py.read_text("utf-8") + new_migration) @@ -224,8 +224,8 @@ def test_stop_before(two_migrations): ) assert result.exit_code == 0 db = sqlite_utils.Database(db_path) - assert db["foo"].exists() - assert not db["bar"].exists() + assert db.table("foo").exists() + assert not db.table("bar").exists() def test_stop_before_multiple_sets_unqualified(two_migrations): @@ -239,7 +239,7 @@ m = Migrations("hello2") @m() def foo(db): - db["foo"].insert({"hello": "world"}) + db.table("foo").insert({"hello": "world"}) """, "utf-8", ) @@ -257,7 +257,7 @@ def foo(db): assert result.exit_code == 0, result.output db = sqlite_utils.Database(db_path) assert db.table_names() == ["_sqlite_migrations"] - assert list(db["_sqlite_migrations"].rows) == [] + assert list(db.table("_sqlite_migrations").rows) == [] def test_stop_before_qualified_only_affects_named_set(two_sets_same_migration_name): @@ -275,10 +275,10 @@ def test_stop_before_qualified_only_affects_named_set(two_sets_same_migration_na ) assert result.exit_code == 0, result.output db = sqlite_utils.Database(db_path) - assert db["creatures"].exists() - assert not db["creature_weights"].exists() - assert db["sales"].exists() - assert db["sales_weights"].exists() + assert db.table("creatures").exists() + assert not db.table("creature_weights").exists() + assert db.table("sales").exists() + assert db.table("sales_weights").exists() def test_stop_before_multiple_qualified(two_sets_same_migration_name): @@ -298,10 +298,10 @@ def test_stop_before_multiple_qualified(two_sets_same_migration_name): ) assert result.exit_code == 0, result.output db = sqlite_utils.Database(db_path) - assert db["creatures"].exists() - assert not db["creature_weights"].exists() - assert db["sales"].exists() - assert not db["sales_weights"].exists() + assert db.table("creatures").exists() + assert not db.table("creature_weights").exists() + assert db.table("sales").exists() + assert not db.table("sales_weights").exists() LEGACY_MIGRATIONS = """ @@ -331,7 +331,7 @@ class LegacyMigrations: return fn def ensure_migrations_table(self, db): - db[self.migrations_table].create( + db.table(self.migrations_table).create( {"migration_set": str, "name": str, "applied_at": str}, pk=("migration_set", "name"), if_not_exists=True, @@ -341,7 +341,7 @@ class LegacyMigrations: self.ensure_migrations_table(db) return [ _Applied(row["name"], row["applied_at"]) - for row in db[self.migrations_table].rows_where( + for row in db.table(self.migrations_table).rows_where( "migration_set = ?", [self.name] ) ] @@ -355,7 +355,7 @@ class LegacyMigrations: if migration.name == stop_before: return migration.fn(db) - db[self.migrations_table].insert( + db.table(self.migrations_table).insert( { "migration_set": self.name, "name": migration.name, @@ -369,11 +369,11 @@ legacy = LegacyMigrations("legacy_set") @legacy def first(db): - db["first"].insert({"hello": "world"}) + db.table("first").insert({"hello": "world"}) @legacy def second(db): - db["second"].insert({"hello": "world"}) + db.table("second").insert({"hello": "world"}) """ @@ -446,11 +446,11 @@ def test_list_does_not_upgrade_legacy_migrations_table(two_migrations): path, _ = two_migrations db_path = str(path / "test.db") db = sqlite_utils.Database(db_path) - db["_sqlite_migrations"].create( + db.table("_sqlite_migrations").create( {"migration_set": str, "name": str, "applied_at": str}, pk=("migration_set", "name"), ) - db["_sqlite_migrations"].insert( + db.table("_sqlite_migrations").insert( {"migration_set": "hello", "name": "foo", "applied_at": "x"} ) db.close() @@ -462,7 +462,7 @@ def test_list_does_not_upgrade_legacy_migrations_table(two_migrations): assert "foo - x" in result.output # --list must not perform the one-way legacy schema upgrade db2 = sqlite_utils.Database(db_path) - assert db2["_sqlite_migrations"].pks == ["migration_set", "name"] + assert db2.table("_sqlite_migrations").pks == ["migration_set", "name"] db2.close() @@ -485,7 +485,7 @@ def test_stop_before_applied_migration_errors(two_migrations): assert result.exit_code != 0 assert "already been applied" in result.output db = sqlite_utils.Database(db_path) - assert not db["bar"].exists() + assert not db.table("bar").exists() def test_list_with_legacy_class_is_read_only(tmpdir): @@ -496,7 +496,7 @@ def test_list_with_legacy_class_is_read_only(tmpdir): (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8") db_path = str(path / "test.db") db = sqlite_utils.Database(db_path) - db["existing"].insert({"id": 1}) + db.table("existing").insert({"id": 1}) db.close() result = CliRunner().invoke( sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"] diff --git a/tests/test_column_affinity.py b/tests/test_column_affinity.py index fa23345..8c619e1 100644 --- a/tests/test_column_affinity.py +++ b/tests/test_column_affinity.py @@ -43,4 +43,4 @@ def test_column_affinity(column_def, expected_type): @pytest.mark.parametrize("column_def,expected_type", EXAMPLES) def test_columns_dict(fresh_db, column_def, expected_type): fresh_db.execute(f"create table foo (col {column_def})") - assert {"col": expected_type} == fresh_db["foo"].columns_dict + assert {"col": expected_type} == fresh_db.table("foo").columns_dict diff --git a/tests/test_column_casing.py b/tests/test_column_casing.py index ce11345..b3f03c9 100644 --- a/tests/test_column_casing.py +++ b/tests/test_column_casing.py @@ -13,14 +13,14 @@ from sqlite_utils.db import ForeignKey def test_insert_populates_last_pk_case_insensitively(fresh_db): - books = fresh_db["books"] + books = fresh_db.table("books") books.create({"Id": int, "Title": str}, pk="Id") books.insert({"Id": 1, "Title": "One"}, pk="id") assert books.last_pk == 1 def test_insert_populates_last_pk_compound_pk_case_insensitively(fresh_db): - books = fresh_db["books"] + books = fresh_db.table("books") books.create({"Author": str, "Position": int, "Title": str}) books.insert( {"Author": "Sue", "Position": 1, "Title": "One"}, pk=("author", "position") @@ -31,7 +31,7 @@ def test_insert_populates_last_pk_compound_pk_case_insensitively(fresh_db): @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_upsert_pk_case_differs_from_schema(use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) - books = db["books"] + books = db.table("books") books.create({"Id": int, "Title": str}, pk="Id") books.insert({"Id": 1, "Title": "One"}) books.upsert({"id": 1, "title": "Won"}, pk="id") @@ -43,7 +43,7 @@ def test_upsert_pk_case_differs_from_schema(use_old_upsert): def test_upsert_record_key_case_differs_from_pk(use_old_upsert): # all_columns comes from the record keys, pk= from the caller db = Database(memory=True, use_old_upsert=use_old_upsert) - books = db["books"] + books = db.table("books") books.create({"Id": int, "Title": str}, pk="Id") books.upsert({"ID": 1, "Title": "One"}, pk="id") assert list(books.rows) == [{"Id": 1, "Title": "One"}] @@ -52,7 +52,7 @@ def test_upsert_record_key_case_differs_from_pk(use_old_upsert): def test_upsert_inferred_pk_case_differs_from_record_keys(fresh_db): # pk is inferred from the existing schema as "Id", records use "id" - books = fresh_db["books"] + books = fresh_db.table("books") books.create({"Id": int, "Title": str}, pk="Id") books.upsert({"id": 1, "title": "One"}) assert list(books.rows) == [{"Id": 1, "Title": "One"}] @@ -60,7 +60,7 @@ def test_upsert_inferred_pk_case_differs_from_record_keys(fresh_db): def test_upsert_list_mode_pk_case_insensitive(fresh_db): - books = fresh_db["books"] + books = fresh_db.table("books") books.create({"Id": int, "Title": str}, pk="Id") books.upsert_all([["id", "title"], [1, "One"]], pk="Id") assert list(books.rows) == [{"Id": 1, "Title": "One"}] @@ -68,84 +68,84 @@ def test_upsert_list_mode_pk_case_insensitive(fresh_db): def test_lookup_pk_case_insensitive(fresh_db): - fresh_db["species"].create({"ID": int, "Name": str}, pk="ID") - fresh_db["species"].insert({"ID": 5, "Name": "Palm"}) - fresh_db["species"].create_index(["Name"], unique=True) - assert fresh_db["species"].lookup({"Name": "Palm"}, pk="id") == 5 + fresh_db.table("species").create({"ID": int, "Name": str}, pk="ID") + fresh_db.table("species").insert({"ID": 5, "Name": "Palm"}) + fresh_db.table("species").create_index(["Name"], unique=True) + assert fresh_db.table("species").lookup({"Name": "Palm"}, pk="id") == 5 def test_lookup_does_not_create_redundant_index(fresh_db): - fresh_db["species"].create({"id": int, "Name": str}, pk="id") - fresh_db["species"].create_index(["Name"], unique=True) - fresh_db["species"].lookup({"name": "Palm"}) - assert len(fresh_db["species"].indexes) == 1 + fresh_db.table("species").create({"id": int, "Name": str}, pk="id") + fresh_db.table("species").create_index(["Name"], unique=True) + fresh_db.table("species").lookup({"name": "Palm"}) + assert len(fresh_db.table("species").indexes) == 1 def test_create_table_transform_same_columns_different_case(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) - fresh_db["t"].insert({"Name": "Cleo", "Age": 5}) + fresh_db.table("t").create({"Name": str, "Age": int}) + fresh_db.table("t").insert({"Name": "Cleo", "Age": 5}) fresh_db.create_table("t", {"name": str, "age": int}, transform=True) # Schema casing is preserved - SQLite considers these the same columns - assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} - assert list(fresh_db["t"].rows) == [{"Name": "Cleo", "Age": 5}] + assert fresh_db.table("t").columns_dict == {"Name": str, "Age": int} + assert list(fresh_db.table("t").rows) == [{"Name": "Cleo", "Age": 5}] def test_create_table_transform_case_insensitive_with_changes(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db.table("t").create({"Name": str, "Age": int}) fresh_db.create_table("t", {"name": str, "age": str, "size": int}, transform=True) # age changed type, size added, Name untouched - assert fresh_db["t"].columns_dict == {"Name": str, "Age": str, "size": int} + assert fresh_db.table("t").columns_dict == {"Name": str, "Age": str, "size": int} def test_transform_types_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str, "Age": str}) - fresh_db["t"].transform(types={"age": int}) - assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} + fresh_db.table("t").create({"Name": str, "Age": str}) + fresh_db.table("t").transform(types={"age": int}) + assert fresh_db.table("t").columns_dict == {"Name": str, "Age": int} def test_transform_rename_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str}) - fresh_db["t"].transform(rename={"name": "title"}) - assert fresh_db["t"].columns_dict == {"title": str} + fresh_db.table("t").create({"Name": str}) + fresh_db.table("t").transform(rename={"name": "title"}) + assert fresh_db.table("t").columns_dict == {"title": str} def test_transform_drop_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) - fresh_db["t"].transform(drop=["name"]) - assert fresh_db["t"].columns_dict == {"Age": int} + fresh_db.table("t").create({"Name": str, "Age": int}) + fresh_db.table("t").transform(drop=["name"]) + assert fresh_db.table("t").columns_dict == {"Age": int} def test_transform_not_null_and_defaults_case_insensitive(fresh_db): - fresh_db["t"].create({"Name": str, "Age": int}) - fresh_db["t"].transform(not_null={"name"}, defaults={"age": 3}) - columns = {c.name: c for c in fresh_db["t"].columns} + fresh_db.table("t").create({"Name": str, "Age": int}) + fresh_db.table("t").transform(not_null={"name"}, defaults={"age": 3}) + columns = {c.name: c for c in fresh_db.table("t").columns} assert columns["Name"].notnull - assert fresh_db["t"].default_values == {"Age": 3} + assert fresh_db.table("t").default_values == {"Age": 3} def test_transform_pk_case_insensitive(fresh_db): - fresh_db["t"].create({"Id": int, "Name": str}) - fresh_db["t"].transform(pk="id") - assert fresh_db["t"].pks == ["Id"] - assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} + fresh_db.table("t").create({"Id": int, "Name": str}) + fresh_db.table("t").transform(pk="id") + assert fresh_db.table("t").pks == ["Id"] + assert fresh_db.table("t").columns_dict == {"Id": int, "Name": str} def test_transform_drop_foreign_keys_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create( + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create( {"id": int, "Parent_ID": int}, pk="id", foreign_keys=[("Parent_ID", "parent", "Id")], ) - fresh_db["child"].transform(drop_foreign_keys=["parent_id"]) - assert fresh_db["child"].foreign_keys == [] + fresh_db.table("child").transform(drop_foreign_keys=["parent_id"]) + assert fresh_db.table("child").foreign_keys == [] def test_add_foreign_key_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") - fresh_db["child"].add_foreign_key("parent_id", "parent", "id") - fks = fresh_db["child"].foreign_keys + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create({"id": int, "Parent_ID": int}, pk="id") + fresh_db.table("child").add_foreign_key("parent_id", "parent", "id") + fks = fresh_db.table("child").foreign_keys assert len(fks) == 1 # The foreign key should use the schema casing of the columns assert fks[0].column == "Parent_ID" @@ -153,79 +153,83 @@ def test_add_foreign_key_case_insensitive(fresh_db): def test_add_foreign_keys_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create({"id": int, "Parent_ID": int}, pk="id") fresh_db.add_foreign_keys([("child", "parent_id", "parent", "id")]) - fks = fresh_db["child"].foreign_keys + fks = fresh_db.table("child").foreign_keys assert len(fks) == 1 assert fks[0].column == "Parent_ID" assert fks[0].other_column == "Id" def test_add_foreign_key_detects_existing_case_insensitively(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create( + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create( {"id": int, "Parent_ID": int}, pk="id", foreign_keys=[("Parent_ID", "parent", "Id")], ) # ignore=True should treat this as already existing, not add a duplicate - fresh_db["child"].add_foreign_key("parent_id", "parent", "id", ignore=True) - assert len(fresh_db["child"].foreign_keys) == 1 + fresh_db.table("child").add_foreign_key("parent_id", "parent", "id", ignore=True) + assert len(fresh_db.table("child").foreign_keys) == 1 def test_add_column_fk_col_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create({"id": int}, pk="id") - fresh_db["child"].add_column("parent_id", int, fk="parent", fk_col="id") - fks = fresh_db["child"].foreign_keys + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create({"id": int}, pk="id") + fresh_db.table("child").add_column("parent_id", int, fk="parent", fk_col="id") + fks = fresh_db.table("child").foreign_keys assert len(fks) == 1 assert fks[0].other_column == "Id" def test_extract_case_insensitive(fresh_db): - fresh_db["trees"].insert({"id": 1, "Species": "Palm"}, pk="id") - fresh_db["trees"].extract("species") - assert fresh_db["trees"].columns_dict == {"id": int, "Species_id": int} - assert list(fresh_db["Species"].rows) == [{"id": 1, "Species": "Palm"}] + fresh_db.table("trees").insert({"id": 1, "Species": "Palm"}, pk="id") + fresh_db.table("trees").extract("species") + assert fresh_db.table("trees").columns_dict == {"id": int, "Species_id": int} + assert list(fresh_db.table("Species").rows) == [{"id": 1, "Species": "Palm"}] def test_convert_multi_case_insensitive(fresh_db): - fresh_db["t"].insert({"id": 1, "Name": "Cleo"}, pk="id") - fresh_db["t"].convert("name", lambda v: {"upper": v.upper()}, multi=True) - assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "upper": "CLEO"}] + fresh_db.table("t").insert({"id": 1, "Name": "Cleo"}, pk="id") + fresh_db.table("t").convert("name", lambda v: {"upper": v.upper()}, multi=True) + assert list(fresh_db.table("t").rows) == [ + {"id": 1, "Name": "Cleo", "upper": "CLEO"} + ] def test_convert_output_case_insensitive(fresh_db): - fresh_db["t"].insert({"id": 1, "Name": "Cleo", "Upper": None}, pk="id") - fresh_db["t"].convert("name", lambda v: v.upper(), output="upper") - assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "Upper": "CLEO"}] + fresh_db.table("t").insert({"id": 1, "Name": "Cleo", "Upper": None}, pk="id") + fresh_db.table("t").convert("name", lambda v: v.upper(), output="upper") + assert list(fresh_db.table("t").rows) == [ + {"id": 1, "Name": "Cleo", "Upper": "CLEO"} + ] def test_create_table_sql_pk_case_insensitive(fresh_db): - fresh_db["t"].create({"Id": int, "Name": str}, pk="id") + fresh_db.table("t").create({"Id": int, "Name": str}, pk="id") # Should not have created an extra lowercase "id" column - assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} - assert fresh_db["t"].pks == ["Id"] + assert fresh_db.table("t").columns_dict == {"Id": int, "Name": str} + assert fresh_db.table("t").pks == ["Id"] def test_create_table_not_null_and_defaults_case_insensitive(fresh_db): - fresh_db["t"].create( + fresh_db.table("t").create( {"Name": str, "Age": int}, not_null={"name"}, defaults={"age": 1} ) - columns = {c.name: c for c in fresh_db["t"].columns} + columns = {c.name: c for c in fresh_db.table("t").columns} assert columns["Name"].notnull - assert fresh_db["t"].default_values == {"Age": 1} + assert fresh_db.table("t").default_values == {"Age": 1} def test_create_table_foreign_keys_case_insensitive(fresh_db): - fresh_db["parent"].create({"Id": int}, pk="Id") - fresh_db["child"].create( + fresh_db.table("parent").create({"Id": int}, pk="Id") + fresh_db.table("child").create( {"id": int, "Parent_ID": int}, pk="id", foreign_keys=[("parent_id", "parent", "id")], ) - fks = fresh_db["child"].foreign_keys + fks = fresh_db.table("child").foreign_keys assert fks == [ ForeignKey( table="child", column="Parent_ID", other_table="parent", other_column="Id" diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 4282969..2d0a298 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -20,8 +20,8 @@ def test_recursive_triggers_off(): def test_memory_name(): db1 = Database(memory_name="shared") db2 = Database(memory_name="shared") - db1["dogs"].insert({"name": "Cleo"}) - assert list(db2["dogs"].rows) == [{"name": "Cleo"}] + db1.table("dogs").insert({"name": "Cleo"}) + assert list(db2.table("dogs").rows) == [{"name": "Cleo"}] def test_sqlite_version(): @@ -36,7 +36,7 @@ def test_sqlite_version(): def test_database_context_manager(tmpdir): path = str(tmpdir / "test.db") with Database(path) as db: - db["t"].insert({"id": 1}) + db.table("t").insert({"id": 1}) # Raw writes commit automatically too db.execute("insert into t (id) values (2)") # An explicitly opened transaction left uncommitted on purpose: @@ -47,7 +47,7 @@ def test_database_context_manager(tmpdir): db.execute("select 1") # ... and the open explicit transaction was rolled back, not committed db2 = Database(path) - assert [r["id"] for r in db2["t"].rows] == [1, 2] + assert [r["id"] for r in db2.table("t").rows] == [1, 2] db2.close() @@ -86,8 +86,8 @@ def test_legacy_transaction_control_connection_is_accepted(tmpdir): str(tmpdir / "test.db"), autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL ) db = Database(conn) - db["t"].insert({"id": 1}, pk="id") - assert [r["id"] for r in db["t"].rows] == [1] + db.table("t").insert({"id": 1}, pk="id") + assert [r["id"] for r in db.table("t").rows] == [1] db.close() diff --git a/tests/test_conversions.py b/tests/test_conversions.py index d70f5c8..bb58df4 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -1,17 +1,17 @@ def test_insert_conversion(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"foo": "bar"}, conversions={"foo": "upper(?)"}) assert [{"foo": "BAR"}] == list(table.rows) def test_insert_all_conversion(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all([{"foo": "bar"}], conversions={"foo": "upper(?)"}) assert [{"foo": "BAR"}] == list(table.rows) def test_upsert_conversion(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert({"id": 1, "foo": "bar"}, pk="id", conversions={"foo": "upper(?)"}) assert [{"id": 1, "foo": "BAR"}] == list(table.rows) table.upsert( @@ -21,7 +21,7 @@ def test_upsert_conversion(fresh_db): def test_upsert_all_conversion(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert_all( [{"id": 1, "foo": "bar"}], pk="id", conversions={"foo": "upper(?)"} ) @@ -29,7 +29,7 @@ def test_upsert_all_conversion(fresh_db): def test_update_conversion(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"id": 5, "foo": "bar"}, pk="id") table.update(5, {"foo": "baz"}, conversions={"foo": "upper(?)"}) assert [{"id": 5, "foo": "BAZ"}] == list(table.rows) diff --git a/tests/test_convert.py b/tests/test_convert.py index 879267a..1f9e9ed 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -27,7 +27,7 @@ from sqlite_utils.db import BadMultiValues ), ) def test_convert(fresh_db, columns, fn, expected): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"title": "Mixed Case", "abstract": "Abstract"}) table.convert(columns, fn) assert list(table.rows) == [expected] @@ -37,7 +37,7 @@ def test_convert(fresh_db, columns, fn, expected): "where,where_args", (("id > 1", None), ("id > :id", {"id": 1}), ("id > ?", [1])) ) def test_convert_where(fresh_db, where, where_args): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all( [ {"id": 1, "title": "One"}, @@ -53,7 +53,7 @@ def test_convert_where(fresh_db, where, where_args): def test_convert_handles_falsey_values(fresh_db): # Falsey values like 0 should be converted (issue #527) - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all([{"x": 0}, {"x": 1}]) assert table.get(1)["x"] == 0 assert table.get(2)["x"] == 1 @@ -70,14 +70,14 @@ def test_convert_handles_falsey_values(fresh_db): ), ) def test_convert_output(fresh_db, drop, expected): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"title": "Mixed Case"}) table.convert("title", lambda v: v.upper(), output="other", drop=drop) assert list(table.rows) == [expected] def test_convert_output_multiple_column_error(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") with pytest.raises(ValueError) as excinfo: table.convert(["title", "other"], lambda v: v, output="out") assert "output= can only be used with a single column" in str(excinfo.value) @@ -91,14 +91,14 @@ def test_convert_output_multiple_column_error(fresh_db): ), ) def test_convert_output_type(fresh_db, type, expected): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"number": "123"}) table.convert("number", lambda v: v, output="other", output_type=type, drop=True) assert list(table.rows) == [expected] def test_convert_multi(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"title": "Mixed Case"}) table.convert( "title", @@ -123,7 +123,7 @@ def test_convert_multi(fresh_db): def test_convert_multi_where(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all( [ {"id": 1, "title": "One"}, @@ -145,14 +145,14 @@ def test_convert_multi_where(fresh_db): def test_convert_multi_exception(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"title": "Mixed Case"}) with pytest.raises(BadMultiValues): table.convert("title", lambda v: v.upper(), multi=True) def test_convert_repeated(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") col = "num" table.insert({col: 1}) table.convert(col, lambda x: x * 2) diff --git a/tests/test_create.py b/tests/test_create.py index 40746bf..0af68a6 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -80,9 +80,10 @@ def test_create_table_compound_primary_key(fresh_db): @pytest.mark.parametrize("pk", ("id", ["id"])) def test_create_table_with_single_primary_key(fresh_db, pk): - fresh_db["foo"].insert({"id": 1}, pk=pk) + fresh_db.table("foo").insert({"id": 1}, pk=pk) assert ( - fresh_db["foo"].schema == 'CREATE TABLE "foo" (\n "id" INTEGER PRIMARY KEY\n)' + fresh_db.table("foo").schema + == 'CREATE TABLE "foo" (\n "id" INTEGER PRIMARY KEY\n)' ) @@ -159,7 +160,7 @@ def test_create_table_with_not_null(fresh_db): ), ) def test_create_table_from_example(fresh_db, example, expected_columns): - people_table = fresh_db["people"] + people_table = fresh_db.table("people") assert people_table.last_rowid is None assert people_table.last_pk is None people_table.insert(example) @@ -167,13 +168,13 @@ def test_create_table_from_example(fresh_db, example, expected_columns): assert people_table.last_pk == 1 assert ["people"] == fresh_db.table_names() assert expected_columns == [ - {"name": col.name, "type": col.type} for col in fresh_db["people"].columns + {"name": col.name, "type": col.type} for col in fresh_db.table("people").columns ] def test_create_table_from_example_with_compound_primary_keys(fresh_db): record = {"name": "Zhang", "group": "staff", "employee_id": 2} - table = fresh_db["people"].insert(record, pk=("group", "employee_id")) + table = fresh_db.table("people").insert(record, pk=("group", "employee_id")) assert ["group", "employee_id"] == table.pks assert record == table.get(("staff", 2)) @@ -184,7 +185,7 @@ def test_create_table_from_example_with_compound_primary_keys(fresh_db): @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_create_table_with_custom_columns(method_name, use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) - table = db["dogs"] + table = db.table("dogs") method = getattr(table, method_name) record = {"id": 1, "name": "Cleo", "age": "5"} if method_name.endswith("_all"): @@ -218,14 +219,16 @@ def test_create_table_column_order(fresh_db, use_table_factory): if use_table_factory: fresh_db.table("table", column_order=column_order).insert(row) else: - fresh_db["table"].insert(row, column_order=column_order) + fresh_db.table("table").insert(row, column_order=column_order) assert [ {"name": "abc", "type": "TEXT"}, {"name": "ccc", "type": "TEXT"}, {"name": "zzz", "type": "TEXT"}, {"name": "bbb", "type": "TEXT"}, {"name": "aaa", "type": "TEXT"}, - ] == [{"name": col.name, "type": col.type} for col in fresh_db["table"].columns] + ] == [ + {"name": col.name, "type": col.type} for col in fresh_db.table("table").columns + ] @pytest.mark.parametrize( @@ -261,8 +264,8 @@ def test_create_table_works_for_m2m_with_only_foreign_keys( fresh_db.table("one", pk="id").insert({"id": 1}) fresh_db.table("two", pk="id").insert({"id": 1}) else: - fresh_db["one"].insert({"id": 1}, pk="id") - fresh_db["two"].insert({"id": 1}, pk="id") + fresh_db.table("one").insert({"id": 1}, pk="id") + fresh_db.table("two").insert({"id": 1}, pk="id") row = {"one_id": 1, "two_id": 1} @@ -270,7 +273,7 @@ def test_create_table_works_for_m2m_with_only_foreign_keys( if use_table_factory: fresh_db.table("m2m", foreign_keys=foreign_key_specification).insert(row) else: - fresh_db["m2m"].insert(row, foreign_keys=foreign_key_specification) + fresh_db.table("m2m").insert(row, foreign_keys=foreign_key_specification) if expected_exception: with pytest.raises(expected_exception): @@ -281,7 +284,7 @@ def test_create_table_works_for_m2m_with_only_foreign_keys( assert [ {"name": "one_id", "type": "INTEGER"}, {"name": "two_id", "type": "INTEGER"}, - ] == [{"name": col.name, "type": col.type} for col in fresh_db["m2m"].columns] + ] == [{"name": col.name, "type": col.type} for col in fresh_db.table("m2m").columns] assert sorted( [ {"column": "one_id", "other_table": "one", "other_column": "id"}, @@ -295,7 +298,7 @@ def test_create_table_works_for_m2m_with_only_foreign_keys( "other_table": fk.other_table, "other_column": fk.other_column, } - for fk in fresh_db["m2m"].foreign_keys + for fk in fresh_db.table("m2m").foreign_keys ], key=lambda s: repr(s), ) @@ -322,7 +325,7 @@ def test_self_referential_foreign_key(fresh_db): def test_create_error_if_invalid_foreign_keys(fresh_db): with pytest.raises(AlterError): - fresh_db["one"].insert( + fresh_db.table("one").insert( {"id": 1, "ref_id": 3}, pk="id", foreign_keys=(("ref_id", "bad_table", "bad_column"),), @@ -331,7 +334,7 @@ def test_create_error_if_invalid_foreign_keys(fresh_db): def test_create_error_if_invalid_self_referential_foreign_keys(fresh_db): with pytest.raises(AlterError) as ex: - fresh_db["one"].insert( + fresh_db.table("one").insert( {"id": 1, "ref_id": 3}, pk="id", foreign_keys=(("ref_id", "one", "bad_column"),), @@ -397,41 +400,43 @@ def test_create_error_if_invalid_self_referential_foreign_keys(fresh_db): ) def test_add_column(fresh_db, col_name, col_type, not_null_default, expected_schema): fresh_db.create_table("dogs", {"name": str}) - assert fresh_db["dogs"].schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' - fresh_db["dogs"].add_column(col_name, col_type, not_null_default=not_null_default) - assert fresh_db["dogs"].schema == expected_schema + assert fresh_db.table("dogs").schema == 'CREATE TABLE "dogs" (\n "name" TEXT\n)' + fresh_db.table("dogs").add_column( + col_name, col_type, not_null_default=not_null_default + ) + assert fresh_db.table("dogs").schema == expected_schema def test_add_foreign_key(fresh_db): - fresh_db["authors"].insert_all( + fresh_db.table("authors").insert_all( [{"id": 1, "name": "Sally"}, {"id": 2, "name": "Asheesh"}], pk="id" ) - fresh_db["books"].insert_all( + fresh_db.table("books").insert_all( [ {"title": "Hedgehogs of the world", "author_id": 1}, {"title": "How to train your wolf", "author_id": 2}, ] ) - assert [] == fresh_db["books"].foreign_keys - t = fresh_db["books"].add_foreign_key("author_id", "authors", "id") + assert [] == fresh_db.table("books").foreign_keys + t = fresh_db.table("books").add_foreign_key("author_id", "authors", "id") # Ensure it returned self: assert isinstance(t, Table) and t.name == "books" assert [ ForeignKey( table="books", column="author_id", other_table="authors", other_column="id" ) - ] == fresh_db["books"].foreign_keys + ] == fresh_db.table("books").foreign_keys def test_add_foreign_key_if_column_contains_space(fresh_db): - fresh_db["authors"].insert_all([{"id": 1, "name": "Sally"}], pk="id") - fresh_db["books"].insert_all( + fresh_db.table("authors").insert_all([{"id": 1, "name": "Sally"}], pk="id") + fresh_db.table("books").insert_all( [ {"title": "Hedgehogs of the world", "author id": 1}, ] ) - fresh_db["books"].add_foreign_key("author id", "authors", "id") - assert fresh_db["books"].foreign_keys == [ + fresh_db.table("books").add_foreign_key("author id", "authors", "id") + assert fresh_db.table("books").foreign_keys == [ ForeignKey( table="books", column="author id", other_table="authors", other_column="id" ) @@ -439,44 +444,44 @@ def test_add_foreign_key_if_column_contains_space(fresh_db): def test_add_foreign_key_error_if_column_does_not_exist(fresh_db): - fresh_db["books"].insert( + fresh_db.table("books").insert( {"id": 1, "title": "Hedgehogs of the world", "author_id": 1} ) with pytest.raises(AlterError): - fresh_db["books"].add_foreign_key("author2_id", "books", "id") + fresh_db.table("books").add_foreign_key("author2_id", "books", "id") def test_add_foreign_key_error_if_other_table_does_not_exist(fresh_db): - fresh_db["books"].insert({"title": "Hedgehogs of the world", "author_id": 1}) + fresh_db.table("books").insert({"title": "Hedgehogs of the world", "author_id": 1}) with pytest.raises(AlterError): - fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") def test_add_foreign_key_error_if_already_exists(fresh_db): - fresh_db["books"].insert({"title": "Hedgehogs of the world", "author_id": 1}) - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fresh_db.table("books").insert({"title": "Hedgehogs of the world", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") with pytest.raises(AlterError) as ex: - fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") assert "Foreign key already exists for author_id => authors.id" == ex.value.args[0] def test_add_foreign_key_no_error_if_exists_and_ignore_true(fresh_db): - fresh_db["books"].insert({"title": "Hedgehogs of the world", "author_id": 1}) - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id", ignore=True) + fresh_db.table("books").insert({"title": "Hedgehogs of the world", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id", ignore=True) def test_add_foreign_keys(fresh_db): - fresh_db["authors"].insert_all( + fresh_db.table("authors").insert_all( [{"id": 1, "name": "Sally"}, {"id": 2, "name": "Asheesh"}], pk="id" ) - fresh_db["categories"].insert_all([{"id": 1, "name": "Wildlife"}], pk="id") - fresh_db["books"].insert_all( + fresh_db.table("categories").insert_all([{"id": 1, "name": "Wildlife"}], pk="id") + fresh_db.table("books").insert_all( [{"title": "Hedgehogs of the world", "author_id": 1, "category_id": 1}] ) - assert [] == fresh_db["books"].foreign_keys + assert [] == fresh_db.table("books").foreign_keys fresh_db.add_foreign_keys( [ ("books", "author_id", "authors", "id"), @@ -493,14 +498,14 @@ def test_add_foreign_keys(fresh_db): other_table="categories", other_column="id", ), - ] == sorted(fresh_db["books"].foreign_keys) + ] == sorted(fresh_db.table("books").foreign_keys) def test_add_column_foreign_key(fresh_db): fresh_db.create_table("dogs", {"name": str}) fresh_db.create_table("breeds", {"name": str}) - fresh_db["dogs"].add_column("breed_id", fk="breeds") - assert fresh_db["dogs"].schema == ( + fresh_db.table("dogs").add_column("breed_id", fk="breeds") + assert fresh_db.table("dogs").schema == ( 'CREATE TABLE "dogs" (\n' ' "name" TEXT,\n' ' "breed_id" INTEGER REFERENCES "breeds"("rowid")\n' @@ -508,8 +513,8 @@ def test_add_column_foreign_key(fresh_db): ) # And again with an explicit primary key column fresh_db.create_table("subbreeds", {"name": str, "primkey": str}, pk="primkey") - fresh_db["dogs"].add_column("subbreed_id", fk="subbreeds") - assert fresh_db["dogs"].schema == ( + fresh_db.table("dogs").add_column("subbreed_id", fk="subbreeds") + assert fresh_db.table("dogs").schema == ( 'CREATE TABLE "dogs" (\n' ' "name" TEXT,\n' ' "breed_id" INTEGER REFERENCES "breeds"("rowid"),\n' @@ -521,9 +526,9 @@ def test_add_column_foreign_key(fresh_db): def test_add_foreign_key_guess_table(fresh_db): fresh_db.create_table("dogs", {"name": str}) fresh_db.create_table("breeds", {"name": str, "id": int}, pk="id") - fresh_db["dogs"].add_column("breed_id", int) - fresh_db["dogs"].add_foreign_key("breed_id") - assert fresh_db["dogs"].schema == ( + fresh_db.table("dogs").add_column("breed_id", int) + fresh_db.table("dogs").add_foreign_key("breed_id") + assert fresh_db.table("dogs").schema == ( 'CREATE TABLE "dogs" (\n' ' "name" TEXT,\n' ' "breed_id" INTEGER REFERENCES "breeds"("id")\n' @@ -533,21 +538,23 @@ def test_add_foreign_key_guess_table(fresh_db): def test_index_foreign_keys(fresh_db): test_add_foreign_key_guess_table(fresh_db) - assert [] == fresh_db["dogs"].indexes + assert [] == fresh_db.table("dogs").indexes fresh_db.index_foreign_keys() - assert [["breed_id"]] == [i.columns for i in fresh_db["dogs"].indexes] + assert [["breed_id"]] == [i.columns for i in fresh_db.table("dogs").indexes] # Calling it a second time should do nothing fresh_db.index_foreign_keys() - assert [["breed_id"]] == [i.columns for i in fresh_db["dogs"].indexes] + assert [["breed_id"]] == [i.columns for i in fresh_db.table("dogs").indexes] def test_index_foreign_keys_if_index_name_is_already_used(fresh_db): # https://github.com/simonw/sqlite-utils/issues/335 test_add_foreign_key_guess_table(fresh_db) # Add index with a name that will conflict with index_foreign_keys() - fresh_db["dogs"].create_index(["name"], index_name="idx_dogs_breed_id") + fresh_db.table("dogs").create_index(["name"], index_name="idx_dogs_breed_id") fresh_db.index_foreign_keys() - assert {(idx.name, tuple(idx.columns)) for idx in fresh_db["dogs"].indexes} == { + assert { + (idx.name, tuple(idx.columns)) for idx in fresh_db.table("dogs").indexes + } == { ("idx_dogs_breed_id_2", ("breed_id",)), ("idx_dogs_breed_id", ("name",)), } @@ -571,7 +578,7 @@ def test_index_foreign_keys_if_index_name_is_already_used(fresh_db): def test_insert_row_alter_table( fresh_db, extra_data, expected_new_columns, use_table_factory ): - table = fresh_db["books"] + table = fresh_db.table("books") table.insert({"title": "Hedgehogs of the world", "author_id": 1}) assert [ {"name": "title", "type": "TEXT"}, @@ -582,7 +589,7 @@ def test_insert_row_alter_table( if use_table_factory: fresh_db.table("books", alter=True).insert(record) else: - fresh_db["books"].insert(record, alter=True) + fresh_db.table("books").insert(record, alter=True) assert [ {"name": "title", "type": "TEXT"}, {"name": "author_id", "type": "INTEGER"}, @@ -592,7 +599,7 @@ def test_insert_row_alter_table( def test_add_missing_columns_case_insensitive(fresh_db): - table = fresh_db["foo"] + table = fresh_db.table("foo") table.insert({"id": 1, "name": "Cleo"}, pk="id") table.add_missing_columns([{"Name": ".", "age": 4}]) assert ( @@ -618,7 +625,7 @@ def test_insert_replace_rows_alter_table(fresh_db, use_table_factory): table.insert(first_row) table.insert_all(next_rows, replace=True) else: - table = fresh_db["books"] + table = fresh_db.table("books") table.insert(first_row, pk="id") table.insert_all(next_rows, alter=True, replace=True) assert { @@ -664,8 +671,8 @@ def test_insert_all_with_extra_columns_in_later_chunks(fresh_db): {"record": "Record 3"}, {"record": "Record 4", "extra": 1}, ] - fresh_db["t"].insert_all(chunk, batch_size=2, alter=True) - assert list(fresh_db["t"].rows) == [ + fresh_db.table("t").insert_all(chunk, batch_size=2, alter=True) + assert list(fresh_db.table("t").rows) == [ {"record": "Record 1", "extra": None}, {"record": "Record 2", "extra": None}, {"record": "Record 3", "extra": None}, @@ -675,7 +682,7 @@ def test_insert_all_with_extra_columns_in_later_chunks(fresh_db): def test_bulk_insert_more_than_999_values(fresh_db): "Inserting 100 items with 11 columns should work" - fresh_db["big"].insert_all( + fresh_db.table("big").insert_all( ( { "id": i + 1, @@ -694,7 +701,7 @@ def test_bulk_insert_more_than_999_values(fresh_db): ), pk="id", ) - assert fresh_db["big"].count == 100 + assert fresh_db.table("big").count == 100 @pytest.mark.parametrize( @@ -704,9 +711,9 @@ def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): record = {f"c{i}": i for i in range(num_columns)} if should_error: with pytest.raises(ValueError): - fresh_db["big"].insert(record) + fresh_db.table("big").insert(record) else: - fresh_db["big"].insert(record) + fresh_db.table("big").insert(record) def test_columns_not_in_first_record_should_not_cause_batch_to_be_too_large(fresh_db): @@ -722,7 +729,9 @@ def test_columns_not_in_first_record_should_not_cause_batch_to_be_too_large(fres # fill out the batch with 99 records with enough columns to exceed THRESHOLD *[{f"c{i}": j for i in range(extra_columns)} for j in range(batch_size - 1)], ] - fresh_db["too_many_columns"].insert_all(records, alter=True, batch_size=batch_size) + fresh_db.table("too_many_columns").insert_all( + records, alter=True, batch_size=batch_size + ) @pytest.mark.parametrize( @@ -767,7 +776,7 @@ def test_columns_not_in_first_record_should_not_cause_batch_to_be_too_large(fres ), ) def test_create_index(fresh_db, columns, index_name, expected_index): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True}) assert [] == dogs.indexes dogs.create_index(columns, index_name) @@ -775,7 +784,7 @@ def test_create_index(fresh_db, columns, index_name, expected_index): def test_create_index_unique(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is_good_dog": True}) assert [] == dogs.indexes dogs.create_index(["name"], unique=True) @@ -793,7 +802,7 @@ def test_create_index_unique(fresh_db): def test_create_index_if_not_exists(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is_good_dog": True}) assert [] == dogs.indexes dogs.create_index(["name"]) @@ -804,7 +813,7 @@ def test_create_index_if_not_exists(fresh_db): def test_drop_index(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is_good_dog": True}) dogs.create_index(["name"]) assert [index.name for index in dogs.indexes] == ["idx_dogs_name"] @@ -813,7 +822,7 @@ def test_drop_index(fresh_db): def test_drop_index_ignore(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo"}) with pytest.raises(OperationalError, match="No index named idx_dogs_name"): dogs.drop_index("idx_dogs_name") @@ -821,8 +830,8 @@ def test_drop_index_ignore(fresh_db): def test_drop_index_wrong_table(fresh_db): - dogs = fresh_db["dogs"] - cats = fresh_db["cats"] + dogs = fresh_db.table("dogs") + cats = fresh_db.table("cats") dogs.insert({"name": "Cleo"}) cats.insert({"name": "Misty"}) dogs.create_index(["name"]) @@ -832,7 +841,7 @@ def test_drop_index_wrong_table(fresh_db): def test_create_index_desc(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"name": "Cleo", "twitter": "cleopaws", "age": 3, "is good dog": True}) assert [] == dogs.indexes dogs.create_index([DescIndex("age"), "name"]) @@ -845,7 +854,7 @@ def test_create_index_desc(fresh_db): def test_create_index_find_unique_name(fresh_db): - table = fresh_db["t"] + table = fresh_db.table("t") table.insert({"id": 1}) table.create_index(["id"]) # Without find_unique_name should error @@ -860,12 +869,12 @@ def test_create_index_find_unique_name(fresh_db): def test_create_index_analyze(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") assert "sqlite_stat1" not in fresh_db.table_names() dogs.insert({"name": "Cleo", "twitter": "cleopaws"}) dogs.create_index(["name"], analyze=True) assert "sqlite_stat1" in fresh_db.table_names() - assert list(fresh_db["sqlite_stat1"].rows) == [ + assert list(fresh_db.table("sqlite_stat1").rows) == [ {"tbl": "dogs", "idx": "idx_dogs_name", "stat": "1 1"} ] @@ -887,14 +896,14 @@ def test_create_index_analyze(fresh_db): ), ) def test_insert_dictionaries_and_lists_as_json(fresh_db, data_structure): - fresh_db["test"].insert({"id": 1, "data": data_structure}, pk="id") + fresh_db.table("test").insert({"id": 1, "data": data_structure}, pk="id") row = fresh_db.execute("select id, data from test").fetchone() assert row[0] == 1 assert data_structure == json.loads(row[1]) def test_insert_list_nested_unicode(fresh_db): - fresh_db["test"].insert( + fresh_db.table("test").insert( {"id": 1, "data": {"key1": {"nested": ["cømplex"]}}}, pk="id" ) row = fresh_db.execute("select id, data from test").fetchone() @@ -903,33 +912,35 @@ def test_insert_list_nested_unicode(fresh_db): def test_insert_uuid(fresh_db): uuid4 = uuid.uuid4() - fresh_db["test"].insert({"uuid": uuid4}) - row = next(iter(fresh_db["test"].rows)) + fresh_db.table("test").insert({"uuid": uuid4}) + row = next(iter(fresh_db.table("test").rows)) assert {"uuid"} == row.keys() assert isinstance(row["uuid"], str) assert row["uuid"] == str(uuid4) def test_insert_memoryview(fresh_db): - fresh_db["test"].insert({"data": memoryview(b"hello")}) - row = next(iter(fresh_db["test"].rows)) + fresh_db.table("test").insert({"data": memoryview(b"hello")}) + row = next(iter(fresh_db.table("test").rows)) assert {"data"} == row.keys() assert isinstance(row["data"], bytes) assert row["data"] == b"hello" def test_insert_thousands_using_generator(fresh_db): - fresh_db["test"].insert_all({"i": i, "word": f"word_{i}"} for i in range(10000)) + fresh_db.table("test").insert_all( + {"i": i, "word": f"word_{i}"} for i in range(10000) + ) assert [{"name": "i", "type": "INTEGER"}, {"name": "word", "type": "TEXT"}] == [ - {"name": col.name, "type": col.type} for col in fresh_db["test"].columns + {"name": col.name, "type": col.type} for col in fresh_db.table("test").columns ] - assert fresh_db["test"].count == 10000 + assert fresh_db.table("test").count == 10000 def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fresh_db): # https://github.com/simonw/sqlite-utils/issues/139 with pytest.raises(Exception, match="table test has no column named extra"): - fresh_db["test"].insert_all( + fresh_db.table("test").insert_all( [{"i": i, "word": f"word_{i}"} for i in range(100)] + [{"i": 101, "extra": "This extra column should cause an exception"}], ) @@ -937,7 +948,7 @@ def test_insert_thousands_raises_exception_with_extra_columns_after_first_100(fr def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db): # https://github.com/simonw/sqlite-utils/issues/139 - fresh_db["test"].insert_all( + fresh_db.table("test").insert_all( [{"i": i, "word": f"word_{i}"} for i in range(100)] + [{"i": 101, "extra": "Should trigger ALTER"}], alter=True, @@ -953,12 +964,12 @@ def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): rows = [{"a": f"x{i}", "b": i} for i in range(num_rows)] with pytest.raises(InvalidColumns) as ex: - fresh_db["t"].insert_all(rows, pk="not_a_column") + fresh_db.table("t").insert_all(rows, pk="not_a_column") assert ex.value.args == ( "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", ) - assert fresh_db["t"].count == 0 + assert fresh_db.table("t").count == 0 @pytest.mark.parametrize("num_rows", (1, 2, 3, 10)) @@ -970,20 +981,20 @@ def test_insert_all_pk_not_in_records_alter_raises(fresh_db, num_rows): rows = [{"a": f"x{i}", "b": i} for i in range(num_rows)] with pytest.raises(InvalidColumns) as ex: - fresh_db["t"].insert_all(rows, pk="not_a_column", alter=True) + fresh_db.table("t").insert_all(rows, pk="not_a_column", alter=True) assert ex.value.args == ( "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", ) - assert fresh_db["t"].count == 0 + assert fresh_db.table("t").count == 0 def test_insert_pk_in_records_with_alter_adds_column(fresh_db): # 3.x allowed insert(pk=..., alter=True) to add the pk column from the # records - the InvalidColumns check must not fire in that case - fresh_db["t"].insert({"a": 1}) - fresh_db["t"].insert({"id": 5, "a": 2}, pk="id", alter=True) - assert fresh_db["t"].columns_dict.keys() == {"a", "id"} + fresh_db.table("t").insert({"a": 1}) + fresh_db.table("t").insert({"id": 5, "a": 2}, pk="id", alter=True) + assert fresh_db.table("t").columns_dict.keys() == {"a", "id"} assert list(fresh_db.query("select * from t order by a")) == [ {"a": 1, "id": None}, {"a": 2, "id": 5}, @@ -994,17 +1005,17 @@ def test_insert_all_invalid_pk_alter_empty_records_is_noop(fresh_db): # With alter=True the pk check needs record keys, so an empty iterator # returns without error - matching the 3.x no-op for empty inserts fresh_db.conn.execute("CREATE TABLE t (a TEXT)") - fresh_db["t"].insert_all([], pk="not_a_column", alter=True) - assert fresh_db["t"].count == 0 + fresh_db.table("t").insert_all([], pk="not_a_column", alter=True) + assert fresh_db.table("t").count == 0 def test_insert_ignore(fresh_db): - fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") + fresh_db.table("test").insert({"id": 1, "bar": 2}, pk="id") # Should raise an error if we try this again with pytest.raises(Exception, match="UNIQUE constraint failed"): - fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") + fresh_db.table("test").insert({"id": 1, "bar": 2}, pk="id") # Using ignore=True should cause our insert to be silently ignored - fresh_db["test"].insert({"id": 1, "bar": 3}, pk="id", ignore=True) + fresh_db.table("test").insert({"id": 1, "bar": 3}, pk="id", ignore=True) # Only one row, and it should be bar=2, not bar=3 rows = list(fresh_db.query("select * from test")) assert rows == [{"id": 1, "bar": 2}] @@ -1013,12 +1024,12 @@ def test_insert_ignore(fresh_db): def test_insert_ignore_reports_existing_row(fresh_db): # An ignored insert (row already exists) should point last_rowid and # last_pk at the existing conflicting row - see the Datasette insert API - fresh_db["docs"].insert({"id": 1, "title": "Exists"}, pk="id") + fresh_db.table("docs").insert({"id": 1, "title": "Exists"}, pk="id") # Insert a conflicting row with ignore=True and no explicit pk= - table = fresh_db["docs"].insert({"id": 1, "title": "One"}, ignore=True) + table = fresh_db.table("docs").insert({"id": 1, "title": "One"}, ignore=True) assert table.last_rowid == 1 assert table.last_pk == 1 - assert list(fresh_db["docs"].rows_where("rowid = ?", [table.last_rowid])) == [ + assert list(fresh_db.table("docs").rows_where("rowid = ?", [table.last_rowid])) == [ {"id": 1, "title": "Exists"} ] @@ -1029,51 +1040,51 @@ def test_pk_rowid_alias_on_rowid_table(fresh_db, rowid_alias, method): # rowid and its aliases are valid primary keys for a rowid table even # though they are not listed among the table's columns - see the Datasette # upsert API against tables without an explicit primary key - fresh_db["t"].insert({"title": "Hello"}) - assert fresh_db["t"].pks == ["rowid"] + fresh_db.table("t").insert({"title": "Hello"}) + assert fresh_db.table("t").pks == ["rowid"] record = {rowid_alias: 1, "title": "Updated"} if method == "upsert": - table = fresh_db["t"].upsert(record, pk=rowid_alias) + table = fresh_db.table("t").upsert(record, pk=rowid_alias) elif method == "insert_replace": - table = fresh_db["t"].insert(record, pk=rowid_alias, replace=True) + table = fresh_db.table("t").insert(record, pk=rowid_alias, replace=True) else: - table = fresh_db["t"].insert(record, pk=rowid_alias, ignore=True) + table = fresh_db.table("t").insert(record, pk=rowid_alias, ignore=True) assert table.last_pk == 1 expected_title = "Hello" if method == "insert_ignore" else "Updated" - assert list(fresh_db["t"].rows) == [{"title": expected_title}] + assert list(fresh_db.table("t").rows) == [{"title": expected_title}] def test_insert_ignore_reports_existing_row_compound_pk(fresh_db): # Compound primary key variant of the ignored-insert lookup - fresh_db["t"].insert_all([{"a": 1, "b": 2, "note": "first"}], pk=("a", "b")) - table = fresh_db["t"].insert( + fresh_db.table("t").insert_all([{"a": 1, "b": 2, "note": "first"}], pk=("a", "b")) + table = fresh_db.table("t").insert( {"a": 1, "b": 2, "note": "second"}, pk=("a", "b"), ignore=True ) assert table.last_pk == (1, 2) - assert list(fresh_db["t"].rows_where("rowid = ?", [table.last_rowid])) == [ + assert list(fresh_db.table("t").rows_where("rowid = ?", [table.last_rowid])) == [ {"a": 1, "b": 2, "note": "first"} ] def test_insert_ignore_reports_existing_row_list_mode(fresh_db): # List-based iteration variant of the ignored-insert lookup - fresh_db["t"].insert_all([["id", "title"], [1, "first"]], pk="id") - table = fresh_db["t"].insert_all( + fresh_db.table("t").insert_all([["id", "title"], [1, "first"]], pk="id") + table = fresh_db.table("t").insert_all( [["id", "title"], [1, "second"]], pk="id", ignore=True ) assert table.last_pk == 1 assert table.last_rowid == 1 - assert list(fresh_db["t"].rows) == [{"id": 1, "title": "first"}] + assert list(fresh_db.table("t").rows) == [{"id": 1, "title": "first"}] def test_insert_ignore_hash_id_reports_pk(fresh_db): # With hash_id the pk is the computed hash; the original record has no id # column to look up so last_rowid is left unset - first = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id") - table = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id", ignore=True) + first = fresh_db.table("dogs").insert({"name": "Cleo"}, hash_id="id") + table = fresh_db.table("dogs").insert({"name": "Cleo"}, hash_id="id", ignore=True) assert table.last_pk == first.last_pk assert table.last_rowid is None - assert fresh_db["dogs"].count == 1 + assert fresh_db.table("dogs").count == 1 def test_insert_ignore_unresolvable_conflict_leaves_pk_unset(fresh_db): @@ -1081,45 +1092,45 @@ def test_insert_ignore_unresolvable_conflict_leaves_pk_unset(fresh_db): # last_rowid are left unset rather than reporting a misleading value # rowid table with a UNIQUE column and no primary key: no pk to look up - fresh_db["u"].db.execute("create table u (title text unique)") - fresh_db["u"].insert({"title": "x"}) - table = fresh_db["u"].insert({"title": "x"}, ignore=True) + fresh_db.table("u").db.execute("create table u (title text unique)") + fresh_db.table("u").insert({"title": "x"}) + table = fresh_db.table("u").insert({"title": "x"}, ignore=True) assert table.last_pk is None assert table.last_rowid is None - assert fresh_db["u"].count == 1 + assert fresh_db.table("u").count == 1 # Conflict on a UNIQUE column other than the primary key: the pk value from # the record does not match the existing row, so the lookup finds nothing - fresh_db["docs"].db.execute( + fresh_db.table("docs").db.execute( "create table docs (id integer primary key, email text unique)" ) - fresh_db["docs"].insert({"id": 1, "email": "a"}, pk="id") - table = fresh_db["docs"].insert({"id": 2, "email": "a"}, ignore=True) + fresh_db.table("docs").insert({"id": 1, "email": "a"}, pk="id") + table = fresh_db.table("docs").insert({"id": 2, "email": "a"}, ignore=True) assert table.last_pk is None assert table.last_rowid is None - assert fresh_db["docs"].count == 1 + assert fresh_db.table("docs").count == 1 def test_insert_ignore_with_pk_after_other_table_insert(fresh_db): # https://github.com/simonw/sqlite-utils/issues/554 user = {"id": "abc", "name": "david"} - fresh_db["users"].insert(user, pk="id") - fresh_db["comments"].insert_all( + fresh_db.table("users").insert(user, pk="id") + fresh_db.table("comments").insert_all( [ {"id": "def", "text": "ok"}, {"id": "ghi", "text": "great"}, ], ) - table = fresh_db["users"].insert(user, pk="id", ignore=True) + table = fresh_db.table("users").insert(user, pk="id", ignore=True) assert table.last_pk == "abc" - assert list(fresh_db["users"].rows) == [user] + assert list(fresh_db.table("users").rows) == [user] def test_insert_hash_id(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") id = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id assert dogs.count == 1 @@ -1137,7 +1148,7 @@ def test_insert_hash_id_columns(fresh_db, use_table_factory): dogs = fresh_db.table("dogs", hash_id_columns=("name", "twitter")) insert_kwargs = {} else: - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") insert_kwargs = {"hash_id_columns": ("name", "twitter")} id = dogs.insert( @@ -1158,26 +1169,26 @@ def test_insert_hash_id_columns(fresh_db, use_table_factory): def test_vacuum(fresh_db): - fresh_db["data"].insert({"foo": "foo", "bar": "bar"}) + fresh_db.table("data").insert({"foo": "foo", "bar": "bar"}) fresh_db.vacuum() def test_works_with_pathlib_path(tmpdir): path = pathlib.Path(tmpdir / "test.db") db = Database(path) - db["demo"].insert_all([{"foo": 1}]) - assert db["demo"].count == 1 + db.table("demo").insert_all([{"foo": 1}]) + assert db.table("demo").count == 1 @pytest.mark.skipif(pd is None, reason="pandas and numpy are not installed") def test_create_table_numpy(fresh_db): df = pd.DataFrame({"col 1": range(3), "col 2": range(3)}) - fresh_db["pandas"].insert_all(df.to_dict(orient="records")) + fresh_db.table("pandas").insert_all(df.to_dict(orient="records")) assert [ {"col 1": 0, "col 2": 0}, {"col 1": 1, "col 2": 1}, {"col 1": 2, "col 2": 2}, - ] == list(fresh_db["pandas"].rows) + ] == list(fresh_db.table("pandas").rows) # Now try all the different types df = pd.DataFrame( { @@ -1222,7 +1233,7 @@ def test_create_table_numpy(fresh_db): "float32", "float64", ] == [str(t) for t in df.dtypes] - fresh_db["types"].insert_all(df.to_dict(orient="records")) + fresh_db.table("types").insert_all(df.to_dict(orient="records")) assert [ { "np.float16": 16.5, @@ -1237,7 +1248,7 @@ def test_create_table_numpy(fresh_db): "np.uint64": 64, "np.uint8": 8, } - ] == list(fresh_db["types"].rows) + ] == list(fresh_db.table("types").rows) def test_cannot_provide_both_filename_and_memory(): @@ -1249,31 +1260,31 @@ def test_cannot_provide_both_filename_and_memory(): def test_creates_id_column(fresh_db): last_pk = fresh_db.table("cats", pk="id").insert({"name": "barry"}).last_pk - assert [{"name": "barry", "id": last_pk}] == list(fresh_db["cats"].rows) + assert [{"name": "barry", "id": last_pk}] == list(fresh_db.table("cats").rows) def test_drop(fresh_db): - fresh_db["t"].insert({"foo": 1}) + fresh_db.table("t").insert({"foo": 1}) assert ["t"] == fresh_db.table_names() - assert None is fresh_db["t"].drop() + assert None is fresh_db.table("t").drop() assert [] == fresh_db.table_names() def test_drop_view(fresh_db): fresh_db.create_view("foo_view", "select 1") assert ["foo_view"] == fresh_db.view_names() - assert None is fresh_db["foo_view"].drop() + assert None is fresh_db.view("foo_view").drop() assert [] == fresh_db.view_names() def test_drop_ignore(fresh_db): with pytest.raises(sqlite3.OperationalError): - fresh_db["does_not_exist"].drop() - fresh_db["does_not_exist"].drop(ignore=True) + fresh_db.table("does_not_exist").drop() + fresh_db.table("does_not_exist").drop(ignore=True) # Testing view is harder, we need to create it in order # to get a View object, then drop it twice fresh_db.create_view("foo_view", "select 1") - view = fresh_db["foo_view"] + view = fresh_db.view("foo_view") assert isinstance(view, View) view.drop() with pytest.raises(sqlite3.OperationalError): @@ -1282,16 +1293,16 @@ def test_drop_ignore(fresh_db): def test_insert_all_empty_list(fresh_db): - fresh_db["t"].insert({"foo": 1}) - assert fresh_db["t"].count == 1 - fresh_db["t"].insert_all([]) - assert fresh_db["t"].count == 1 - fresh_db["t"].insert_all([], replace=True) - assert fresh_db["t"].count == 1 + fresh_db.table("t").insert({"foo": 1}) + assert fresh_db.table("t").count == 1 + fresh_db.table("t").insert_all([]) + assert fresh_db.table("t").count == 1 + fresh_db.table("t").insert_all([], replace=True) + assert fresh_db.table("t").count == 1 def test_insert_all_single_column(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all([{"name": "Cleo"}], pk="name") assert [{"name": "Cleo"}] == list(table.rows) assert table.pks == ["name"] @@ -1299,31 +1310,33 @@ def test_insert_all_single_column(fresh_db): @pytest.mark.parametrize("method_name", ("insert_all", "upsert_all")) def test_insert_all_analyze(fresh_db, method_name): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all([{"id": 1, "name": "Cleo"}], pk="id") assert "sqlite_stat1" not in fresh_db.table_names() table.create_index(["name"], analyze=True) - assert list(fresh_db["sqlite_stat1"].rows) == [ + assert list(fresh_db.table("sqlite_stat1").rows) == [ {"tbl": "table", "idx": "idx_table_name", "stat": "1 1"} ] method = getattr(table, method_name) method([{"id": 2, "name": "Suna"}], pk="id", analyze=True) assert "sqlite_stat1" in fresh_db.table_names() - assert list(fresh_db["sqlite_stat1"].rows) == [ + assert list(fresh_db.table("sqlite_stat1").rows) == [ {"tbl": "table", "idx": "idx_table_name", "stat": "2 1"} ] def test_create_with_a_null_column(fresh_db): record = {"name": "Name", "description": None} - fresh_db["t"].insert(record) - assert [record] == list(fresh_db["t"].rows) + fresh_db.table("t").insert(record) + assert [record] == list(fresh_db.table("t").rows) def test_create_with_nested_bytes(fresh_db): record = {"id": 1, "data": {"foo": b"bytes"}} - fresh_db["t"].insert(record) - assert [{"id": 1, "data": '{"foo": "b\'bytes\'"}'}] == list(fresh_db["t"].rows) + fresh_db.table("t").insert(record) + assert [{"id": 1, "data": '{"foo": "b\'bytes\'"}'}] == list( + fresh_db.table("t").rows + ) @pytest.mark.parametrize( @@ -1361,7 +1374,7 @@ def test_create_table_sql(fresh_db, columns, expected_sql_middle): def test_create(fresh_db): - fresh_db["t"].create( + fresh_db.table("t").create( { "id": int, "text": str, @@ -1374,7 +1387,7 @@ def test_create(fresh_db): not_null=("float", "integer"), defaults={"integer": 0}, ) - assert fresh_db["t"].schema == ( + assert fresh_db.table("t").schema == ( 'CREATE TABLE "t" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "float" REAL NOT NULL,\n' @@ -1386,37 +1399,37 @@ def test_create(fresh_db): def test_create_if_not_exists(fresh_db): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should error with pytest.raises(sqlite3.OperationalError): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should not - fresh_db["t"].create({"id": int}, if_not_exists=True) + fresh_db.table("t").create({"id": int}, if_not_exists=True) def test_create_if_no_columns(fresh_db): with pytest.raises(ValueError) as error: - fresh_db["t"].create({}) + fresh_db.table("t").create({}) assert error.value.args[0] == "Tables must have at least one column" def test_create_ignore(fresh_db): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should error with pytest.raises(sqlite3.OperationalError): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should not - fresh_db["t"].create({"id": int}, ignore=True) + fresh_db.table("t").create({"id": int}, ignore=True) def test_create_replace(fresh_db): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should error with pytest.raises(sqlite3.OperationalError): - fresh_db["t"].create({"id": int}) + fresh_db.table("t").create({"id": int}) # This should not - fresh_db["t"].create({"name": str}, replace=True) - assert fresh_db["t"].schema == ('CREATE TABLE "t" (\n' ' "name" TEXT\n' ")") + fresh_db.table("t").create({"name": str}, replace=True) + assert fresh_db.table("t").schema == ('CREATE TABLE "t" (\n' ' "name" TEXT\n' ")") @pytest.mark.parametrize( @@ -1484,23 +1497,23 @@ def test_create_replace(fresh_db): ) def test_create_transform(fresh_db, cols, kwargs, expected_schema, should_transform): fresh_db.create_table("demo", {"id": int, "name": str}, pk="id") - fresh_db["demo"].insert({"id": 1, "name": "Cleo"}) + fresh_db.table("demo").insert({"id": 1, "name": "Cleo"}) traces = [] with fresh_db.tracer(lambda sql, parameters: traces.append((sql, parameters))): - fresh_db["demo"].create(cols, **kwargs, transform=True) + fresh_db.table("demo").create(cols, **kwargs, transform=True) at_least_one_create_table = any(sql.startswith("CREATE TABLE") for sql, _ in traces) assert should_transform == at_least_one_create_table - new_schema = fresh_db["demo"].schema + new_schema = fresh_db.table("demo").schema assert new_schema == expected_schema, repr(new_schema) - assert fresh_db["demo"].count == 1 + assert fresh_db.table("demo").count == 1 def test_rename_table(fresh_db): - fresh_db["t"].insert({"foo": "bar"}) + fresh_db.table("t").insert({"foo": "bar"}) assert ["t"] == fresh_db.table_names() fresh_db.rename_table("t", "renamed") assert ["renamed"] == fresh_db.table_names() - assert [{"foo": "bar"}] == list(fresh_db["renamed"].rows) + assert [{"foo": "bar"}] == list(fresh_db.table("renamed").rows) # Should error if table does not exist: with pytest.raises(sqlite3.OperationalError): fresh_db.rename_table("does_not_exist", "renamed") @@ -1527,7 +1540,7 @@ def test_database_strict_override(strict): ) @pytest.mark.parametrize("strict", (False, True)) def test_insert_upsert_strict(fresh_db, method_name, strict): - table = fresh_db["t"] + table = fresh_db.table("t") method = getattr(table, method_name) record = {"id": 1} if method_name.endswith("_all"): @@ -1550,7 +1563,7 @@ def test_create_table_strict(fresh_db, strict): @pytest.mark.parametrize("strict", (False, True)) def test_create_strict(fresh_db, strict): - table = fresh_db["t"] + table = fresh_db.table("t") table.create({"id": int}, strict=strict) assert table.strict == strict or not fresh_db.supports_strict @@ -1575,7 +1588,7 @@ def test_bad_table_and_view_exceptions(fresh_db): def test_pk_persists_after_insert_655(fresh_db): """When pk is passed to insert(), subsequent inserts should use it.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.insert({"id": 1, "name": "Alice"}, pk="id") # Second insert should use pk="id" from _defaults table.insert({"id": 2, "name": "Bob"}) @@ -1586,7 +1599,7 @@ def test_pk_persists_after_insert_655(fresh_db): def test_pk_persists_after_insert_all_655(fresh_db): """When pk is passed to insert_all(), subsequent inserts should use it.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.insert_all([{"id": 1, "name": "Alice"}], pk="id") # Second insert_all should use pk="id" from _defaults table.insert_all([{"id": 2, "name": "Bob"}]) @@ -1596,7 +1609,7 @@ def test_pk_persists_after_insert_all_655(fresh_db): def test_pk_persists_after_create_655(fresh_db): """When pk is passed to create(), it should be stored in _defaults.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.create({"id": int, "name": str}, pk="id") assert table._defaults["pk"] == "id" # Subsequent insert should use the pk @@ -1607,8 +1620,8 @@ def test_pk_persists_after_create_655(fresh_db): def test_foreign_keys_persist_after_create_655(fresh_db): """When foreign_keys is passed to create(), it should be stored in _defaults.""" - fresh_db["authors"].insert({"id": 1, "name": "Alice"}, pk="id") - table = fresh_db["books"] + fresh_db.table("authors").insert({"id": 1, "name": "Alice"}, pk="id") + table = fresh_db.table("books") table.create( {"id": int, "title": str, "author_id": int}, pk="id", @@ -1620,28 +1633,28 @@ def test_foreign_keys_persist_after_create_655(fresh_db): def test_not_null_persists_after_create_655(fresh_db): """When not_null is passed to create(), it should be stored in _defaults.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.create({"id": int, "name": str}, pk="id", not_null=["name"]) assert table._defaults["not_null"] == ["name"] def test_defaults_persist_after_create_655(fresh_db): """When defaults is passed to create(), it should be stored in _defaults.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.create({"id": int, "score": int}, pk="id", defaults={"score": 0}) assert table._defaults["defaults"] == {"score": 0} def test_strict_persists_after_create_655(fresh_db): """When strict is passed to create(), it should be stored in _defaults.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.create({"id": int, "name": str}, pk="id", strict=True) assert table._defaults["strict"] is True def test_upsert_uses_pk_from_prior_insert_655(fresh_db): """After insert with pk, upsert should use the same pk.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.insert({"id": 1, "name": "Alice"}, pk="id") # Upsert should work without specifying pk again table.upsert({"id": 1, "name": "Alice Updated"}) @@ -1651,7 +1664,7 @@ def test_upsert_uses_pk_from_prior_insert_655(fresh_db): def test_upsert_all_uses_pk_from_prior_insert_655(fresh_db): """After insert with pk, upsert_all should use the same pk.""" - table = fresh_db["users"] + table = fresh_db.table("users") table.insert({"id": 1, "name": "Alice"}, pk="id") # Upsert_all should work without specifying pk again table.upsert_all([{"id": 1, "name": "Alice Updated"}, {"id": 2, "name": "Bob"}]) diff --git a/tests/test_default_value.py b/tests/test_default_value.py index 2815180..02b28c3 100644 --- a/tests/test_default_value.py +++ b/tests/test_default_value.py @@ -32,9 +32,9 @@ EXAMPLES = [ @pytest.mark.parametrize("column_def,initial_value,expected_value", EXAMPLES) def test_quote_default_value(fresh_db, column_def, initial_value, expected_value): fresh_db.execute(f"create table foo (col {column_def})") - assert initial_value == fresh_db["foo"].columns[0].default_value + assert initial_value == fresh_db.table("foo").columns[0].default_value assert expected_value == fresh_db.quote_default_value( - fresh_db["foo"].columns[0].default_value + fresh_db.table("foo").columns[0].default_value ) @@ -48,7 +48,7 @@ def test_insert_empty_record_uses_default_values(fresh_db): ) """) - table = fresh_db["has_defaults"] + table = fresh_db.table("has_defaults") table.insert({}) rows = list(table.rows) diff --git a/tests/test_delete.py b/tests/test_delete.py index dffb6bb..a9341b8 100644 --- a/tests/test_delete.py +++ b/tests/test_delete.py @@ -2,7 +2,7 @@ import sqlite_utils def test_delete_rowid_table(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"foo": 1}) rowid = table.insert({"foo": 2}).last_pk table.delete(rowid) @@ -10,7 +10,7 @@ def test_delete_rowid_table(fresh_db): def test_delete_pk_table(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"id": 1}, pk="id") table.insert({"id": 2}, pk="id") table.delete(1) @@ -18,7 +18,7 @@ def test_delete_pk_table(fresh_db): def test_delete_where(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") for i in range(1, 11): table.insert({"id": i}, pk="id") assert table.count == 10 @@ -27,7 +27,7 @@ def test_delete_where(fresh_db): def test_delete_where_all(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") for i in range(1, 11): table.insert({"id": i}, pk="id") assert table.count == 10 @@ -38,27 +38,27 @@ def test_delete_where_all(fresh_db): def test_delete_where_commits(tmpdir): path = str(tmpdir / "test.db") db = sqlite_utils.Database(path) - db["table"].insert_all([{"id": i} for i in range(5)], pk="id") - db["table"].delete_where("id > ?", [2]) + db.table("table").insert_all([{"id": i} for i in range(5)], pk="id") + db.table("table").delete_where("id > ?", [2]) # The connection must not be left inside an open transaction, # otherwise subsequent atomic() blocks never commit either assert not db.conn.in_transaction - db["table"].insert({"id": 100}) + db.table("table").insert({"id": 100}) db.close() db2 = sqlite_utils.Database(path) - assert [r["id"] for r in db2["table"].rows] == [0, 1, 2, 100] + assert [r["id"] for r in db2.table("table").rows] == [0, 1, 2, 100] db2.close() def test_delete_where_analyze(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert_all(({"id": i, "i": i} for i in range(10)), pk="id") table.create_index(["i"], analyze=True) assert "sqlite_stat1" in fresh_db.table_names() - assert list(fresh_db["sqlite_stat1"].rows) == [ + assert list(fresh_db.table("sqlite_stat1").rows) == [ {"tbl": "table", "idx": "idx_table_i", "stat": "10 1"} ] table.delete_where("id > ?", [5], analyze=True) - assert list(fresh_db["sqlite_stat1"].rows) == [ + assert list(fresh_db.table("sqlite_stat1").rows) == [ {"tbl": "table", "idx": "idx_table_i", "stat": "6 1"} ] diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py index ad853a5..c7a5612 100644 --- a/tests/test_duplicate.py +++ b/tests/test_duplicate.py @@ -22,7 +22,7 @@ def test_duplicate(fresh_db): "bool_col": True, "datetime_col": str(dt), } - table1 = fresh_db["table1"] + table1 = fresh_db.table("table1") row_id = table1.insert(data).last_rowid # Duplicate table: table2 = table1.duplicate("table2") @@ -40,4 +40,4 @@ def test_duplicate(fresh_db): def test_duplicate_fails_if_table_does_not_exist(fresh_db): with pytest.raises(NoTable): - fresh_db["not_a_table"].duplicate("duplicated") + fresh_db.table("not_a_table").duplicate("duplicated") diff --git a/tests/test_enable_counts.py b/tests/test_enable_counts.py index 71a8936..1230b6c 100644 --- a/tests/test_enable_counts.py +++ b/tests/test_enable_counts.py @@ -5,7 +5,7 @@ from sqlite_utils import Database, cli def test_enable_counts_specific_table(fresh_db): - foo = fresh_db["foo"] + foo = fresh_db.table("foo") assert fresh_db.table_names() == [] for i in range(10): foo.insert({"name": f"item {i}"}) @@ -41,24 +41,24 @@ def test_enable_counts_specific_table(fresh_db): ), } assert fresh_db.table_names() == ["foo", "_counts"] - assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}] + assert list(fresh_db.table("_counts").rows) == [{"count": 10, "table": "foo"}] # Add some items to test the triggers for i in range(5): foo.insert({"name": f"item {10 + i}"}) assert foo.count == 15 - assert list(fresh_db["_counts"].rows) == [{"count": 15, "table": "foo"}] + assert list(fresh_db.table("_counts").rows) == [{"count": 15, "table": "foo"}] # Delete some items foo.delete_where("rowid < 7") assert foo.count == 9 - assert list(fresh_db["_counts"].rows) == [{"count": 9, "table": "foo"}] + assert list(fresh_db.table("_counts").rows) == [{"count": 9, "table": "foo"}] foo.delete_where() assert foo.count == 0 - assert list(fresh_db["_counts"].rows) == [{"count": 0, "table": "foo"}] + assert list(fresh_db.table("_counts").rows) == [{"count": 0, "table": "foo"}] def test_enable_counts_all_tables(fresh_db): - foo = fresh_db["foo"] - bar = fresh_db["bar"] + foo = fresh_db.table("foo") + bar = fresh_db.table("bar") foo.insert({"name": "Cleo"}) bar.insert({"name": "Cleo"}) foo.enable_fts(["name"]) @@ -73,7 +73,7 @@ def test_enable_counts_all_tables(fresh_db): "foo_fts_config", "_counts", } - assert list(fresh_db["_counts"].rows) == [ + assert list(fresh_db.table("_counts").rows) == [ {"count": 1, "table": "foo"}, {"count": 1, "table": "bar"}, {"count": 3, "table": "foo_fts_data"}, @@ -87,10 +87,10 @@ def test_enable_counts_all_tables(fresh_db): def counts_db_path(tmpdir): path = str(tmpdir / "test.db") db = Database(path) - db["foo"].insert({"name": "bar"}) - db["bar"].insert({"name": "bar"}) - db["bar"].insert({"name": "bar"}) - db["baz"].insert({"name": "bar"}) + db.table("foo").insert({"name": "bar"}) + db.table("bar").insert({"name": "bar"}) + db.table("bar").insert({"name": "bar"}) + db.table("baz").insert({"name": "bar"}) return path @@ -163,25 +163,25 @@ def test_uses_counts_after_enable_counts(counts_db_path): def test_reset_counts(counts_db_path): db = Database(counts_db_path) - db["foo"].enable_counts() - db["bar"].enable_counts() + db.table("foo").enable_counts() + db.table("bar").enable_counts() assert db.cached_counts() == {"foo": 1, "bar": 2} # Corrupt the value - db["_counts"].update("foo", {"count": 3}) + db.table("_counts").update("foo", {"count": 3}) assert db.cached_counts() == {"foo": 3, "bar": 2} - assert db["foo"].count == 3 + assert db.table("foo").count == 3 # Reset them db.reset_counts() assert db.cached_counts() == {"foo": 1, "bar": 2} - assert db["foo"].count == 1 + assert db.table("foo").count == 1 def test_reset_counts_cli(counts_db_path): db = Database(counts_db_path) - db["foo"].enable_counts() - db["bar"].enable_counts() + db.table("foo").enable_counts() + db.table("bar").enable_counts() assert db.cached_counts() == {"foo": 1, "bar": 2} - db["_counts"].update("foo", {"count": 3}) + db.table("_counts").update("foo", {"count": 3}) result = CliRunner().invoke(cli.cli, ["reset-counts", counts_db_path]) assert result.exit_code == 0 assert db.cached_counts() == {"foo": 1, "bar": 2} diff --git a/tests/test_extract.py b/tests/test_extract.py index 915e6e1..72579c4 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -11,7 +11,7 @@ def test_extract_single_column(fresh_db, table, fk_column): expected_table = table or "species" expected_fk = fk_column or f"{expected_table}_id" iter_species = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) - fresh_db["tree"].insert_all( + fresh_db.table("tree").insert_all( ( { "id": i, @@ -23,8 +23,8 @@ def test_extract_single_column(fresh_db, table, fk_column): ), pk="id", ) - fresh_db["tree"].extract("species", table=table, fk_column=fk_column) - assert fresh_db["tree"].schema == ( + fresh_db.table("tree").extract("species", table=table, fk_column=fk_column) + assert fresh_db.table("tree").schema == ( 'CREATE TABLE "tree" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT,\n' @@ -32,18 +32,18 @@ def test_extract_single_column(fresh_db, table, fk_column): + ' "end" INTEGER\n' + ")" ) - assert fresh_db[expected_table].schema == ( + assert fresh_db.table(expected_table).schema == ( f'CREATE TABLE "{expected_table}" (\n' + ' "id" INTEGER PRIMARY KEY,\n' ' "species" TEXT\n' ")" ) - assert list(fresh_db[expected_table].rows) == [ + assert list(fresh_db.table(expected_table).rows) == [ {"id": 1, "species": "Palm"}, {"id": 2, "species": "Spruce"}, {"id": 3, "species": "Mangrove"}, {"id": 4, "species": "Oak"}, ] - assert list(itertools.islice(fresh_db["tree"].rows, 0, 4)) == [ + assert list(itertools.islice(fresh_db.table("tree").rows, 0, 4)) == [ {"id": 1, "name": "Tree 1", expected_fk: 1, "end": 1}, {"id": 2, "name": "Tree 2", expected_fk: 2, "end": 1}, {"id": 3, "name": "Tree 3", expected_fk: 3, "end": 1}, @@ -54,7 +54,7 @@ def test_extract_single_column(fresh_db, table, fk_column): def test_extract_multiple_columns_with_rename(fresh_db): iter_common = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) iter_latin = itertools.cycle(["Arecaceae", "Picea", "Rhizophora", "Quercus"]) - fresh_db["tree"].insert_all( + fresh_db.table("tree").insert_all( ( { "id": i, @@ -67,30 +67,30 @@ def test_extract_multiple_columns_with_rename(fresh_db): pk="id", ) - fresh_db["tree"].extract( + fresh_db.table("tree").extract( ["common_name", "latin_name"], rename={"common_name": "name"} ) - assert fresh_db["tree"].schema == ( + assert fresh_db.table("tree").schema == ( 'CREATE TABLE "tree" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT,\n' ' "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n' ")" ) - assert fresh_db["common_name_latin_name"].schema == ( + assert fresh_db.table("common_name_latin_name").schema == ( 'CREATE TABLE "common_name_latin_name" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT,\n' ' "latin_name" TEXT\n' ")" ) - assert list(fresh_db["common_name_latin_name"].rows) == [ + assert list(fresh_db.table("common_name_latin_name").rows) == [ {"name": "Palm", "id": 1, "latin_name": "Arecaceae"}, {"name": "Spruce", "id": 2, "latin_name": "Picea"}, {"name": "Mangrove", "id": 3, "latin_name": "Rhizophora"}, {"name": "Oak", "id": 4, "latin_name": "Quercus"}, ] - assert list(itertools.islice(fresh_db["tree"].rows, 0, 4)) == [ + assert list(itertools.islice(fresh_db.table("tree").rows, 0, 4)) == [ {"id": 1, "name": "Tree 1", "common_name_latin_name_id": 1}, {"id": 2, "name": "Tree 2", "common_name_latin_name_id": 2}, {"id": 3, "name": "Tree 3", "common_name_latin_name_id": 3}, @@ -99,7 +99,7 @@ def test_extract_multiple_columns_with_rename(fresh_db): def test_extract_invalid_columns(fresh_db): - fresh_db["tree"].insert( + fresh_db.table("tree").insert( { "id": 1, "name": "Tree 1", @@ -109,19 +109,19 @@ def test_extract_invalid_columns(fresh_db): pk="id", ) with pytest.raises(InvalidColumns): - fresh_db["tree"].extract(["bad_column"]) + fresh_db.table("tree").extract(["bad_column"]) def test_extract_rowid_table(fresh_db): - fresh_db["tree"].insert( + fresh_db.table("tree").insert( { "name": "Tree 1", "common_name": "Palm", "latin_name": "Arecaceae", } ) - fresh_db["tree"].extract(["common_name", "latin_name"]) - assert fresh_db["tree"].schema == ( + fresh_db.table("tree").extract(["common_name", "latin_name"]) + assert fresh_db.table("tree").schema == ( 'CREATE TABLE "tree" (\n' ' "name" TEXT,\n' ' "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n' @@ -139,68 +139,68 @@ def test_extract_rowid_table(fresh_db): def test_reuse_lookup_table(fresh_db): - fresh_db["species"].insert({"id": 1, "name": "Wolf"}, pk="id") - fresh_db["sightings"].insert({"id": 10, "species": "Wolf"}, pk="id") - fresh_db["individuals"].insert( + fresh_db.table("species").insert({"id": 1, "name": "Wolf"}, pk="id") + fresh_db.table("sightings").insert({"id": 10, "species": "Wolf"}, pk="id") + fresh_db.table("individuals").insert( {"id": 10, "name": "Terriana", "species": "Fox"}, pk="id" ) - fresh_db["sightings"].extract("species", rename={"species": "name"}) - fresh_db["individuals"].extract("species", rename={"species": "name"}) - assert fresh_db["sightings"].schema == ( + fresh_db.table("sightings").extract("species", rename={"species": "name"}) + fresh_db.table("individuals").extract("species", rename={"species": "name"}) + assert fresh_db.table("sightings").schema == ( 'CREATE TABLE "sightings" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "species_id" INTEGER REFERENCES "species"("id")\n' ")" ) - assert fresh_db["individuals"].schema == ( + assert fresh_db.table("individuals").schema == ( 'CREATE TABLE "individuals" (\n' ' "id" INTEGER PRIMARY KEY,\n' ' "name" TEXT,\n' ' "species_id" INTEGER REFERENCES "species"("id")\n' ")" ) - assert list(fresh_db["species"].rows) == [ + assert list(fresh_db.table("species").rows) == [ {"id": 1, "name": "Wolf"}, {"id": 2, "name": "Fox"}, ] def test_extract_error_on_incompatible_existing_lookup_table(fresh_db): - fresh_db["species"].insert({"id": 1}) - fresh_db["tree"].insert({"name": "Tree 1", "common_name": "Palm"}) + fresh_db.table("species").insert({"id": 1}) + fresh_db.table("tree").insert({"name": "Tree 1", "common_name": "Palm"}) with pytest.raises(InvalidColumns): - fresh_db["tree"].extract("common_name", table="species") + fresh_db.table("tree").extract("common_name", table="species") # Try again with incompatible existing column type - fresh_db["species2"].insert({"id": 1, "common_name": 3.5}) + fresh_db.table("species2").insert({"id": 1, "common_name": 3.5}) with pytest.raises(InvalidColumns): - fresh_db["tree"].extract("common_name", table="species2") + fresh_db.table("tree").extract("common_name", table="species2") def test_extract_works_with_null_values(fresh_db): - fresh_db["listens"].insert_all( + fresh_db.table("listens").insert_all( [ {"id": 1, "track_title": "foo", "album_title": "bar"}, {"id": 2, "track_title": "baz", "album_title": None}, ], pk="id", ) - fresh_db["listens"].extract( + fresh_db.table("listens").extract( columns=["album_title"], table="albums", fk_column="album_id" ) - assert list(fresh_db["listens"].rows) == [ + assert list(fresh_db.table("listens").rows) == [ {"id": 1, "track_title": "foo", "album_id": 1}, {"id": 2, "track_title": "baz", "album_id": None}, ] - assert list(fresh_db["albums"].rows) == [ + assert list(fresh_db.table("albums").rows) == [ {"id": 1, "album_title": "bar"}, ] def test_extract_null_values_single_column(fresh_db): # https://github.com/simonw/sqlite-utils/issues/186 - fresh_db["species"].insert({"id": 1, "species": "Wolf"}, pk="id") - fresh_db["individuals"].insert_all( + fresh_db.table("species").insert({"id": 1, "species": "Wolf"}, pk="id") + fresh_db.table("individuals").insert_all( [ {"id": 10, "name": "Terriana", "species": "Fox"}, {"id": 11, "name": "Spenidorm", "species": None}, @@ -210,13 +210,13 @@ def test_extract_null_values_single_column(fresh_db): ], pk="id", ) - fresh_db["individuals"].extract("species") + fresh_db.table("individuals").extract("species") # No null row should have been added to species - assert list(fresh_db["species"].rows) == [ + assert list(fresh_db.table("species").rows) == [ {"id": 1, "species": "Wolf"}, {"id": 2, "species": "Fox"}, ] - assert list(fresh_db["individuals"].rows) == [ + assert list(fresh_db.table("individuals").rows) == [ {"id": 10, "name": "Terriana", "species_id": 2}, {"id": 11, "name": "Spenidorm", "species_id": None}, {"id": 12, "name": "Grantheim", "species_id": 1}, @@ -228,7 +228,7 @@ def test_extract_null_values_single_column(fresh_db): def test_extract_null_values_multiple_columns(fresh_db): # A row should be extracted if at least one column is not null - # only rows where ALL extracted columns are null are left alone - fresh_db["circulation"].insert_all( + fresh_db.table("circulation").insert_all( [ {"id": 1, "title": "title one", "creator": "creator one", "year": 2018}, {"id": 2, "title": "title two", "creator": None, "year": 2019}, @@ -237,14 +237,14 @@ def test_extract_null_values_multiple_columns(fresh_db): ], pk="id", ) - fresh_db["circulation"].extract( + fresh_db.table("circulation").extract( ["title", "creator"], table="books", fk_column="book_id" ) - assert list(fresh_db["books"].rows) == [ + assert list(fresh_db.table("books").rows) == [ {"id": 1, "title": "title one", "creator": "creator one"}, {"id": 2, "title": "title two", "creator": None}, ] - assert list(fresh_db["circulation"].rows) == [ + assert list(fresh_db.table("circulation").rows) == [ {"id": 1, "book_id": 1, "year": 2018}, {"id": 2, "book_id": 2, "year": 2019}, {"id": 3, "book_id": None, "year": 2020}, @@ -255,20 +255,20 @@ def test_extract_null_values_multiple_columns(fresh_db): def test_extract_null_values_existing_lookup_table_with_null_row(fresh_db): # Even if the lookup table already contains an all-null row, rows where # every extracted column is null should keep a null foreign key - fresh_db["species"].insert({"id": 1, "species": None}, pk="id") - fresh_db["individuals"].insert_all( + fresh_db.table("species").insert({"id": 1, "species": None}, pk="id") + fresh_db.table("individuals").insert_all( [ {"id": 10, "name": "Terriana", "species": "Fox"}, {"id": 11, "name": "Spenidorm", "species": None}, ], pk="id", ) - fresh_db["individuals"].extract("species") - assert list(fresh_db["species"].rows) == [ + fresh_db.table("individuals").extract("species") + assert list(fresh_db.table("species").rows) == [ {"id": 1, "species": None}, {"id": 2, "species": "Fox"}, ] - assert list(fresh_db["individuals"].rows) == [ + assert list(fresh_db.table("individuals").rows) == [ {"id": 10, "name": "Terriana", "species_id": 2}, {"id": 11, "name": "Spenidorm", "species_id": None}, ] @@ -279,17 +279,19 @@ def test_extract_repeated_into_shared_lookup_with_nulls(fresh_db): # cannot dedupe NULL-containing rows against the existing lookup # table - extracting a second table into the same lookup previously # inserted duplicate rows that nothing pointed to - fresh_db["t1"].insert_all( + fresh_db.table("t1").insert_all( [ {"id": 1, "species": None, "common": "X"}, {"id": 2, "species": "Oak", "common": "Oak"}, ], pk="id", ) - fresh_db["t2"].insert_all([{"id": 1, "species": None, "common": "X"}], pk="id") - fresh_db["t1"].extract(["species", "common"], table="lk") - fresh_db["t2"].extract(["species", "common"], table="lk") - assert fresh_db["lk"].count == 2 + fresh_db.table("t2").insert_all( + [{"id": 1, "species": None, "common": "X"}], pk="id" + ) + fresh_db.table("t1").extract(["species", "common"], table="lk") + fresh_db.table("t2").extract(["species", "common"], table="lk") + assert fresh_db.table("lk").count == 2 # Both tables point at the same lookup row t1_fk = fresh_db.execute("select lk_id from t1 where id = 1").fetchone()[0] t2_fk = fresh_db.execute("select lk_id from t2 where id = 1").fetchone()[0] @@ -298,8 +300,8 @@ def test_extract_repeated_into_shared_lookup_with_nulls(fresh_db): def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db): # Non-NULL rows were already deduped by the unique index - keep it so - fresh_db["t1"].insert_all([{"id": 1, "species": "Oak"}], pk="id") - fresh_db["t2"].insert_all([{"id": 1, "species": "Oak"}], pk="id") - fresh_db["t1"].extract(["species"], table="lk") - fresh_db["t2"].extract(["species"], table="lk") - assert fresh_db["lk"].count == 1 + fresh_db.table("t1").insert_all([{"id": 1, "species": "Oak"}], pk="id") + fresh_db.table("t2").insert_all([{"id": 1, "species": "Oak"}], pk="id") + fresh_db.table("t1").extract(["species"], table="lk") + fresh_db.table("t2").extract(["species"], table="lk") + assert fresh_db.table("lk").count == 1 diff --git a/tests/test_extracts.py b/tests/test_extracts.py index 9519b91..4e7cf39 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -32,15 +32,15 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): assert {expected_table, "Trees"} == set(fresh_db.table_names()) assert ( f'CREATE TABLE "{expected_table}" (\n "id" INTEGER PRIMARY KEY,\n "value" TEXT\n)' - == fresh_db[expected_table].schema + == fresh_db.table(expected_table).schema ) assert ( f'CREATE TABLE "Trees" (\n "id" INTEGER,\n "species_id" INTEGER REFERENCES "{expected_table}"("id")\n)' - == fresh_db["Trees"].schema + == fresh_db.table("Trees").schema ) # Should have a foreign key reference - assert len(fresh_db["Trees"].foreign_keys) == 1 - fk = fresh_db["Trees"].foreign_keys[0] + assert len(fresh_db.table("Trees").foreign_keys) == 1 + fk = fresh_db.table("Trees").foreign_keys[0] assert fk.table == "Trees" assert fk.column == "species_id" @@ -54,22 +54,22 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): partial=0, columns=["value"], ) - ] == fresh_db[expected_table].indexes + ] == fresh_db.table(expected_table).indexes # Finally, check the rows assert [{"id": 1, "value": "Oak"}, {"id": 2, "value": "Palm"}] == list( - fresh_db[expected_table].rows + fresh_db.table(expected_table).rows ) assert [ {"id": 1, "species_id": 1}, {"id": 2, "species_id": 1}, {"id": 3, "species_id": 2}, - ] == list(fresh_db["Trees"].rows) + ] == list(fresh_db.table("Trees").rows) def test_extracts_null_values(fresh_db): # https://github.com/simonw/sqlite-utils/issues/186 # Null values should stay null, not be extracted into the lookup table - fresh_db["Trees"].insert_all( + fresh_db.table("Trees").insert_all( [ {"id": 1, "species_id": "Oak"}, {"id": 2, "species_id": None}, @@ -78,11 +78,11 @@ def test_extracts_null_values(fresh_db): ], extracts={"species_id": "Species"}, ) - assert list(fresh_db["Species"].rows) == [ + assert list(fresh_db.table("Species").rows) == [ {"id": 1, "value": "Oak"}, {"id": 2, "value": "Palm"}, ] - assert list(fresh_db["Trees"].rows) == [ + assert list(fresh_db.table("Trees").rows) == [ {"id": 1, "species_id": 1}, {"id": 2, "species_id": None}, {"id": 3, "species_id": 2}, @@ -92,7 +92,7 @@ def test_extracts_null_values(fresh_db): def test_extracts_null_values_list_mode(fresh_db): # Same as test_extracts_null_values but for list-based records - fresh_db["Trees"].insert_all( + fresh_db.table("Trees").insert_all( [ ["id", "species_id"], [1, "Oak"], @@ -102,11 +102,11 @@ def test_extracts_null_values_list_mode(fresh_db): ], extracts={"species_id": "Species"}, ) - assert list(fresh_db["Species"].rows) == [ + assert list(fresh_db.table("Species").rows) == [ {"id": 1, "value": "Oak"}, {"id": 2, "value": "Palm"}, ] - assert list(fresh_db["Trees"].rows) == [ + assert list(fresh_db.table("Trees").rows) == [ {"id": 1, "species_id": 1}, {"id": 2, "species_id": None}, {"id": 3, "species_id": 2}, diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 45f4f35..271125c 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -32,7 +32,7 @@ def compound_db(): def test_compound_foreign_key(compound_db): - fks = compound_db["courses"].foreign_keys + fks = compound_db.table("courses").foreign_keys assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True @@ -46,10 +46,10 @@ def test_compound_foreign_key(compound_db): def test_single_foreign_key_gets_columns_fields(fresh_db): - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) - fresh_db["books"].add_foreign_key("author_id", "authors", "id") - fk = fresh_db["books"].foreign_keys[0] + fresh_db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db.table("books").insert({"title": "Hedgehogs", "author_id": 1}) + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") + fk = fresh_db.table("books").foreign_keys[0] assert fk.is_compound is False assert fk.column == "author_id" assert fk.other_column == "id" @@ -60,10 +60,10 @@ def test_single_foreign_key_gets_columns_fields(fresh_db): def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): # Clean break in 4.0: ForeignKey is a dataclass, not a namedtuple, so the # old tuple unpacking and indexing patterns now fail hard. - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) - fresh_db["books"].add_foreign_key("author_id", "authors", "id") - fk = fresh_db["books"].foreign_keys[0] + fresh_db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db.table("books").insert({"title": "Hedgehogs", "author_id": 1}) + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") + fk = fresh_db.table("books").foreign_keys[0] with pytest.raises(TypeError): _table, _column, _other_table, _other_column = fk with pytest.raises(TypeError): @@ -71,16 +71,18 @@ def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): def test_foreign_keys_are_sortable(fresh_db): - fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") - fresh_db["categories"].insert({"id": 1, "name": "Wildlife"}, pk="id") - fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1, "category_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db.table("categories").insert({"id": 1, "name": "Wildlife"}, pk="id") + fresh_db.table("books").insert( + {"title": "Hedgehogs", "author_id": 1, "category_id": 1} + ) fresh_db.add_foreign_keys( [ ("books", "author_id", "authors", "id"), ("books", "category_id", "categories", "id"), ] ) - fks = sorted(fresh_db["books"].foreign_keys) + fks = sorted(fresh_db.table("books").foreign_keys) assert fks[0].column == "author_id" assert fks[1].column == "category_id" @@ -105,7 +107,7 @@ def test_mixed_compound_and_single_foreign_keys_are_sortable(): REFERENCES departments(campus_name, dept_code) ); """) - fks = db["courses"].foreign_keys + fks = db.table("courses").foreign_keys assert len(fks) == 2 assert {fk.is_compound for fk in fks} == {True, False} fks_sorted = sorted(fks) @@ -163,8 +165,8 @@ def test_create_table_with_compound_foreign_key(departments_db, foreign_keys): pk="course_code", foreign_keys=foreign_keys, ) - assert departments_db["courses"].schema == EXPECTED_COURSES_SCHEMA - fks = departments_db["courses"].foreign_keys + assert departments_db.table("courses").schema == EXPECTED_COURSES_SCHEMA + fks = departments_db.table("courses").foreign_keys assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True @@ -181,10 +183,10 @@ def test_create_table_compound_foreign_key_enforced(departments_db): pk="course_code", foreign_keys=[(("campus_name", "dept_code"), "departments")], ) - departments_db["departments"].insert( + departments_db.table("departments").insert( {"campus_name": "Berkeley", "dept_code": "CS", "dept_name": "Computer Science"} ) - departments_db["courses"].insert( + departments_db.table("courses").insert( {"course_code": "CS101", "campus_name": "Berkeley", "dept_code": "CS"} ) with pytest.raises(sqlite3.IntegrityError): @@ -207,8 +209,8 @@ def test_create_table_compound_foreign_key_missing_other_column(departments_db): def test_transform_preserves_compound_foreign_key(compound_db): - compound_db["courses"].transform(rename={"course_name": "title"}) - fks = compound_db["courses"].foreign_keys + compound_db.table("courses").transform(rename={"course_name": "title"}) + fks = compound_db.table("courses").foreign_keys assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True @@ -218,8 +220,8 @@ def test_transform_preserves_compound_foreign_key(compound_db): def test_transform_rename_member_column_updates_compound_foreign_key(compound_db): - compound_db["courses"].transform(rename={"campus_name": "campus"}) - fks = compound_db["courses"].foreign_keys + compound_db.table("courses").transform(rename={"campus_name": "campus"}) + fks = compound_db.table("courses").foreign_keys assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True @@ -231,9 +233,9 @@ def test_transform_rename_member_column_updates_compound_foreign_key(compound_db def test_transform_drop_member_column_drops_compound_foreign_key(compound_db): # Matches single-column behavior: dropping the column silently # drops the foreign key that used it - compound_db["courses"].transform(drop={"dept_code"}) - assert compound_db["courses"].foreign_keys == [] - assert "FOREIGN KEY" not in compound_db["courses"].schema + compound_db.table("courses").transform(drop={"dept_code"}) + assert compound_db.table("courses").foreign_keys == [] + assert "FOREIGN KEY" not in compound_db.table("courses").schema @pytest.mark.parametrize( @@ -246,11 +248,11 @@ def test_transform_drop_member_column_drops_compound_foreign_key(compound_db): ), ) def test_transform_drop_compound_foreign_key(compound_db, drop_foreign_keys): - compound_db["courses"].transform(drop_foreign_keys=drop_foreign_keys) - assert compound_db["courses"].foreign_keys == [] + compound_db.table("courses").transform(drop_foreign_keys=drop_foreign_keys) + assert compound_db.table("courses").foreign_keys == [] # The columns themselves survive assert {"campus_name", "dept_code"} <= set( - compound_db["courses"].columns_dict.keys() + compound_db.table("courses").columns_dict.keys() ) @@ -265,12 +267,12 @@ def courses_db(departments_db): def test_add_compound_foreign_key(courses_db): - t = courses_db["courses"].add_foreign_key( + t = courses_db.table("courses").add_foreign_key( ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code") ) # Returns self assert t.name == "courses" - fks = courses_db["courses"].foreign_keys + fks = courses_db.table("courses").foreign_keys assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True @@ -281,27 +283,33 @@ def test_add_compound_foreign_key(courses_db): def test_add_compound_foreign_key_guesses_other_columns(courses_db): # Lists work here too, though tuples are the documented form - courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") - fk = courses_db["courses"].foreign_keys[0] + courses_db.table("courses").add_foreign_key( + ["campus_name", "dept_code"], "departments" + ) + fk = courses_db.table("courses").foreign_keys[0] assert fk.other_columns == ("campus_name", "dept_code") def test_add_compound_foreign_key_error_if_already_exists(courses_db): - courses_db["courses"].add_foreign_key(("campus_name", "dept_code"), "departments") + courses_db.table("courses").add_foreign_key( + ("campus_name", "dept_code"), "departments" + ) with pytest.raises(AlterError) as ex: - courses_db["courses"].add_foreign_key( + courses_db.table("courses").add_foreign_key( ("campus_name", "dept_code"), "departments" ) assert "already exists" in ex.value.args[0] # ignore=True should not raise - courses_db["courses"].add_foreign_key( + courses_db.table("courses").add_foreign_key( ("campus_name", "dept_code"), "departments", ignore=True ) def test_add_compound_foreign_key_error_if_column_missing(courses_db): with pytest.raises(AlterError): - courses_db["courses"].add_foreign_key(("campus_name", "nope"), "departments") + courses_db.table("courses").add_foreign_key( + ("campus_name", "nope"), "departments" + ) def test_db_add_foreign_keys_compound(courses_db): @@ -315,14 +323,14 @@ def test_db_add_foreign_keys_compound(courses_db): ) ] ) - fk = courses_db["courses"].foreign_keys[0] + fk = courses_db.table("courses").foreign_keys[0] assert fk.is_compound is True assert fk.columns == ("campus_name", "dept_code") def test_index_foreign_keys_compound_creates_composite_index(compound_db): compound_db.index_foreign_keys() - index_columns = [i.columns for i in compound_db["courses"].indexes] + index_columns = [i.columns for i in compound_db.table("courses").indexes] assert ["campus_name", "dept_code"] in index_columns # No separate single-column indexes for the members assert ["campus_name"] not in index_columns @@ -339,22 +347,22 @@ def test_foreign_key_captures_on_delete_and_on_update(): ON DELETE CASCADE ON UPDATE RESTRICT ); """) - fk = db["books"].foreign_keys[0] + fk = db.table("books").foreign_keys[0] assert fk.on_delete == "CASCADE" assert fk.on_update == "RESTRICT" def test_foreign_key_on_delete_defaults_to_no_action(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id") - fk = fresh_db["books"].foreign_keys[0] + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("books").insert({"id": 1, "author_id": 1}, pk="id") + fresh_db.table("books").add_foreign_key("author_id", "authors", "id") + fk = fresh_db.table("books").foreign_keys[0] assert fk.on_delete == "NO ACTION" assert fk.on_update == "NO ACTION" def test_create_table_foreign_key_with_on_delete(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db.table("authors").insert({"id": 1}, pk="id") fresh_db.create_table( "books", {"id": int, "author_id": int}, @@ -369,8 +377,8 @@ def test_create_table_foreign_key_with_on_delete(fresh_db): ) ], ) - assert "ON DELETE CASCADE" in fresh_db["books"].schema - assert fresh_db["books"].foreign_keys[0].on_delete == "CASCADE" + assert "ON DELETE CASCADE" in fresh_db.table("books").schema + assert fresh_db.table("books").foreign_keys[0].on_delete == "CASCADE" def test_transform_preserves_on_delete_cascade(): @@ -383,11 +391,11 @@ def test_transform_preserves_on_delete_cascade(): author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE ); """) - db["books"].transform(rename={"title": "book_title"}) - fk = db["books"].foreign_keys[0] + db.table("books").transform(rename={"title": "book_title"}) + fk = db.table("books").foreign_keys[0] assert fk.on_delete == "CASCADE" assert fk.on_update == "NO ACTION" - assert "ON DELETE CASCADE" in db["books"].schema + assert "ON DELETE CASCADE" in db.table("books").schema def test_transform_preserves_compound_foreign_key_on_delete(): @@ -406,11 +414,11 @@ def test_transform_preserves_compound_foreign_key_on_delete(): REFERENCES departments(campus_name, dept_code) ON DELETE CASCADE ); """) - db["courses"].transform(rename={"course_code": "code"}) - fk = db["courses"].foreign_keys[0] + db.table("courses").transform(rename={"course_code": "code"}) + fk = db.table("courses").foreign_keys[0] assert fk.is_compound is True assert fk.on_delete == "CASCADE" - assert "ON DELETE CASCADE" in db["courses"].schema + assert "ON DELETE CASCADE" in db.table("courses").schema def test_implicit_primary_key_reference_is_resolved(): @@ -424,7 +432,7 @@ def test_implicit_primary_key_reference_is_resolved(): author_id INTEGER REFERENCES authors ); """) - fk = db["books"].foreign_keys[0] + fk = db.table("books").foreign_keys[0] assert fk.is_compound is False assert fk.other_column == "author_id" assert fk.other_columns == ("author_id",) @@ -445,7 +453,7 @@ def test_implicit_compound_primary_key_reference_is_resolved(): FOREIGN KEY (campus_name, dept_code) REFERENCES departments ); """) - fk = db["courses"].foreign_keys[0] + fk = db.table("courses").foreign_keys[0] assert fk.is_compound is True assert fk.other_columns == ("campus_name", "dept_code") @@ -470,14 +478,14 @@ def test_add_foreign_keys_preserves_actions(fresh_db): # https://github.com/simonw/sqlite-utils/issues/594 review finding: # ForeignKey objects passed to db.add_foreign_keys() were flattened # to plain tuples, losing on_delete/on_update - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("books").insert({"id": 1, "author_id": 1}, pk="id") fresh_db.add_foreign_keys( [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] ) - fk = fresh_db["books"].foreign_keys[0] + fk = fresh_db.table("books").foreign_keys[0] assert fk.on_delete == "CASCADE" - assert "ON DELETE CASCADE" in fresh_db["books"].schema + assert "ON DELETE CASCADE" in fresh_db.table("books").schema def test_add_foreign_keys_preserves_actions_compound(courses_db): @@ -495,36 +503,36 @@ def test_add_foreign_keys_preserves_actions_compound(courses_db): ) ] ) - fk = courses_db["courses"].foreign_keys[0] + fk = courses_db.table("courses").foreign_keys[0] assert fk.is_compound is True assert fk.on_delete == "CASCADE" - assert "ON DELETE CASCADE" in courses_db["courses"].schema + assert "ON DELETE CASCADE" in courses_db.table("courses").schema def test_add_foreign_key_on_delete_on_update(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") - fresh_db["books"].add_foreign_key( + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("books").insert({"id": 1, "author_id": 1}, pk="id") + fresh_db.table("books").add_foreign_key( "author_id", "authors", "id", on_delete="CASCADE", on_update="RESTRICT" ) - fk = fresh_db["books"].foreign_keys[0] + fk = fresh_db.table("books").foreign_keys[0] assert fk.on_delete == "CASCADE" assert fk.on_update == "RESTRICT" - assert "ON UPDATE RESTRICT ON DELETE CASCADE" in fresh_db["books"].schema + assert "ON UPDATE RESTRICT ON DELETE CASCADE" in fresh_db.table("books").schema # The cascade should actually fire fresh_db.execute("PRAGMA foreign_keys = ON") fresh_db.execute("delete from authors where id = 1") - assert fresh_db["books"].count == 0 + assert fresh_db.table("books").count == 0 def test_add_compound_foreign_key_on_delete(courses_db): - courses_db["courses"].add_foreign_key( + courses_db.table("courses").add_foreign_key( ("campus_name", "dept_code"), "departments", on_delete="SET NULL" ) - fk = courses_db["courses"].foreign_keys[0] + fk = courses_db.table("courses").foreign_keys[0] assert fk.is_compound is True assert fk.on_delete == "SET NULL" - assert "ON DELETE SET NULL" in courses_db["courses"].schema + assert "ON DELETE SET NULL" in courses_db.table("courses").schema def test_implicit_compound_foreign_key_resolves_pk_declaration_order(fresh_db): @@ -536,7 +544,7 @@ def test_implicit_compound_foreign_key_resolves_pk_declaration_order(fresh_db): fresh_db.execute( "create table child (x text, y text, foreign key (x, y) references other)" ) - fk = fresh_db["child"].foreign_keys[0] + fk = fresh_db.table("child").foreign_keys[0] assert fk.other_columns == ("a", "b") @@ -549,46 +557,46 @@ def test_transform_implicit_compound_foreign_key_stays_valid(fresh_db): "create table child (x text, y text, foreign key (x, y) references other)" ) fresh_db.execute("PRAGMA foreign_keys = ON") - fresh_db["other"].insert({"a": "A", "b": "B"}) - fresh_db["child"].insert({"x": "A", "y": "B"}) - fresh_db["child"].transform(types={"x": str}) - assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + fresh_db.table("other").insert({"a": "A", "b": "B"}) + fresh_db.table("child").insert({"x": "A", "y": "B"}) + fresh_db.table("child").transform(types={"x": str}) + assert fresh_db.table("child").foreign_keys[0].other_columns == ("a", "b") # The constraint still points the right way around - fresh_db["child"].insert({"x": "A", "y": "B"}) + fresh_db.table("child").insert({"x": "A", "y": "B"}) with pytest.raises(sqlite3.IntegrityError): - fresh_db["child"].insert({"x": "B", "y": "A"}) + fresh_db.table("child").insert({"x": "B", "y": "A"}) def test_create_compound_foreign_key_guesses_pk_declaration_order(fresh_db): fresh_db.execute("create table other (b text, a text, primary key (a, b))") - fresh_db["other"].insert({"a": "A", "b": "B"}) - fresh_db["child"].create( + fresh_db.table("other").insert({"a": "A", "b": "B"}) + fresh_db.table("child").create( {"id": int, "x": str, "y": str}, pk="id", foreign_keys=[(("x", "y"), "other")], ) - assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + assert fresh_db.table("child").foreign_keys[0].other_columns == ("a", "b") fresh_db.execute("PRAGMA foreign_keys = ON") - fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}) + fresh_db.table("child").insert({"id": 1, "x": "A", "y": "B"}) with pytest.raises(sqlite3.IntegrityError): - fresh_db["child"].insert({"id": 2, "x": "B", "y": "A"}) + fresh_db.table("child").insert({"id": 2, "x": "B", "y": "A"}) def test_add_compound_foreign_key_guesses_pk_declaration_order(fresh_db): fresh_db.execute("create table other (b text, a text, primary key (a, b))") - fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}, pk="id") - fresh_db["child"].add_foreign_key(("x", "y"), "other") - assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + fresh_db.table("child").insert({"id": 1, "x": "A", "y": "B"}, pk="id") + fresh_db.table("child").add_foreign_key(("x", "y"), "other") + assert fresh_db.table("child").foreign_keys[0].other_columns == ("a", "b") def test_foreign_keys_are_hashable(fresh_db): # set() over foreign_keys worked with the 3.x namedtuple and must # keep working with the dataclass - fresh_db["p"].insert({"id": 1}, pk="id") - fresh_db["c"].insert( + fresh_db.table("p").insert({"id": 1}, pk="id") + fresh_db.table("c").insert( {"id": 1, "pid": 1}, pk="id", foreign_keys=[("pid", "p", "id")] ) - fks = set(fresh_db["c"].foreign_keys) + fks = set(fresh_db.table("c").foreign_keys) assert len(fks) == 1 assert ForeignKey("c", "pid", "p", "id") in fks # Usable as dict keys too @@ -617,9 +625,9 @@ def test_create_table_mixed_foreign_keys_list(fresh_db): # 3.x accepted a mix of ForeignKey objects, tuples and bare column # strings in foreign_keys= (ForeignKey was a namedtuple, so it passed # the tuple check) - keep accepting the mix - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["publishers"].insert({"id": 1}, pk="id") - fresh_db["books"].create( + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("publishers").insert({"id": 1}, pk="id") + fresh_db.table("books").create( {"id": int, "author_id": int, "publisher_id": int}, pk="id", foreign_keys=[ @@ -627,14 +635,14 @@ def test_create_table_mixed_foreign_keys_list(fresh_db): ("publisher_id", "publishers", "id"), ], ) - fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} + fks = {fk.column: fk.other_table for fk in fresh_db.table("books").foreign_keys} assert fks == {"author_id": "authors", "publisher_id": "publishers"} def test_create_table_mixed_foreign_keys_with_string(fresh_db): - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["publishers"].insert({"id": 1}, pk="id") - fresh_db["books"].create( + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("publishers").insert({"id": 1}, pk="id") + fresh_db.table("books").create( {"id": int, "author_id": int, "publisher_id": int}, pk="id", foreign_keys=[ @@ -642,15 +650,15 @@ def test_create_table_mixed_foreign_keys_with_string(fresh_db): ("publisher_id", "publishers", "id"), ], ) - fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} + fks = {fk.column: fk.other_table for fk in fresh_db.table("books").foreign_keys} assert fks == {"author_id": "authors", "publisher_id": "publishers"} def test_add_foreign_keys_existing_with_different_actions_errors(fresh_db): # Requesting an existing foreign key with different ON DELETE/ON UPDATE # actions was silently skipped, dropping the requested change - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert( + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("books").insert( {"id": 1, "author_id": 1}, pk="id", foreign_keys=[("author_id", "authors", "id")], @@ -660,19 +668,21 @@ def test_add_foreign_keys_existing_with_different_actions_errors(fresh_db): [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] ) assert "ON DELETE" in str(ex.value) - assert fresh_db["books"].foreign_keys[0].on_delete == "NO ACTION" + assert fresh_db.table("books").foreign_keys[0].on_delete == "NO ACTION" def test_add_foreign_keys_identical_existing_is_noop(fresh_db): # An exact match, including actions, is silently skipped so repeated # calls stay idempotent - fresh_db["authors"].insert({"id": 1}, pk="id") - fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") - fresh_db["books"].add_foreign_key("author_id", "authors", "id", on_delete="CASCADE") + fresh_db.table("authors").insert({"id": 1}, pk="id") + fresh_db.table("books").insert({"id": 1, "author_id": 1}, pk="id") + fresh_db.table("books").add_foreign_key( + "author_id", "authors", "id", on_delete="CASCADE" + ) fresh_db.add_foreign_keys( [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] ) - fks = fresh_db["books"].foreign_keys + fks = fresh_db.table("books").foreign_keys assert len(fks) == 1 assert fks[0].on_delete == "CASCADE" @@ -680,13 +690,13 @@ def test_add_foreign_keys_identical_existing_is_noop(fresh_db): def test_add_foreign_keys_compound_column_count_mismatch_errors(fresh_db): # Previously the extra other-column was silently discarded, creating # a single-column foreign key to just ("id") - fresh_db["departments"].insert( + fresh_db.table("departments").insert( {"campus": "north", "code": "cs"}, pk=("campus", "code") ) - fresh_db["courses"].insert({"id": 1, "campus": "north"}, pk="id") + fresh_db.table("courses").insert({"id": 1, "campus": "north"}, pk="id") with pytest.raises(ValueError) as ex: fresh_db.add_foreign_keys( [("courses", ("campus",), "departments", ("campus", "code"))] ) assert "same number of columns" in str(ex.value) - assert fresh_db["courses"].foreign_keys == [] + assert fresh_db.table("courses").foreign_keys == [] diff --git a/tests/test_fts.py b/tests/test_fts.py index 79af042..312b032 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -20,7 +20,7 @@ search_records = [ def test_enable_fts(fresh_db): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert_all(search_records) assert ["searchable"] == fresh_db.table_names() table.enable_fts(["text", "country"], fts_version="FTS4") @@ -54,7 +54,7 @@ def test_enable_fts(fresh_db): def test_enable_fts_escape_table_names(fresh_db): # Table names with restricted chars are handled correctly. # colons and dots are restricted characters for table names. - table = fresh_db["http://example.com"] + table = fresh_db.table("http://example.com") table.insert_all(search_records) assert ["http://example.com"] == fresh_db.table_names() table.enable_fts(["text", "country"], fts_version="FTS4") @@ -87,7 +87,7 @@ def test_enable_fts_escape_table_names(fresh_db): def test_search_duplicate_columns_are_deduped(fresh_db): # https://github.com/simonw/sqlite-utils/issues/624 - table = fresh_db["t"] + table = fresh_db.table("t") table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version="FTS4") rows = list(table.search("tanuki", columns=["text", "text"])) @@ -100,7 +100,7 @@ def test_search_duplicate_columns_are_deduped(fresh_db): def test_search_limit_offset(fresh_db): - table = fresh_db["t"] + table = fresh_db.table("t") table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version="FTS4") assert len(list(table.search("are"))) == 2 @@ -113,7 +113,7 @@ def test_search_limit_offset(fresh_db): def test_search_offset_without_limit(fresh_db): - table = fresh_db["t"] + table = fresh_db.table("t") table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version="FTS4") assert [row["rowid"] for row in table.search("are", order_by="rowid")] == [1, 2] @@ -125,7 +125,7 @@ def test_search_offset_without_limit(fresh_db): @pytest.mark.parametrize("fts_version", ("FTS4", "FTS5")) def test_search_where(fresh_db, fts_version): - table = fresh_db["t"] + table = fresh_db.table("t") table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version=fts_version) results = list( @@ -142,7 +142,7 @@ def test_search_where(fresh_db, fts_version): def test_search_where_args_disallows_query(fresh_db): - table = fresh_db["t"] + table = fresh_db.table("t") with pytest.raises(ValueError) as ex: list( table.search( @@ -156,7 +156,7 @@ def test_search_where_args_disallows_query(fresh_db): def test_search_include_rank(fresh_db): - table = fresh_db["t"] + table = fresh_db.table("t") table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version="FTS5") results = list(table.search("are", include_rank=True)) @@ -182,7 +182,7 @@ def test_search_include_rank(fresh_db): def test_enable_fts_table_names_containing_spaces(fresh_db): - table = fresh_db["test"] + table = fresh_db.table("test") table.insert({"column with spaces": "in its name"}) table.enable_fts(["column with spaces"]) assert [ @@ -196,7 +196,7 @@ def test_enable_fts_table_names_containing_spaces(fresh_db): def test_populate_fts(fresh_db): - table = fresh_db["populatable"] + table = fresh_db.table("populatable") table.insert(search_records[0]) table.enable_fts(["text", "country"], fts_version="FTS4") assert [] == list(table.search("trash pandas")) @@ -217,7 +217,7 @@ def test_populate_fts(fresh_db): def test_populate_fts_escape_table_names(fresh_db): # Restricted characters such as colon and dots should be escaped. - table = fresh_db["http://example.com"] + table = fresh_db.table("http://example.com") table.insert(search_records[0]) table.enable_fts(["text", "country"], fts_version="FTS4") assert [] == list(table.search("trash pandas")) @@ -238,7 +238,7 @@ def test_populate_fts_escape_table_names(fresh_db): @pytest.mark.parametrize("fts_version", ("4", "5")) def test_fts_tokenize(fresh_db, fts_version): table_name = f"searchable_{fts_version}" - table = fresh_db[table_name] + table = fresh_db.table(table_name) table.insert_all(search_records) # Test without porter stemming table.enable_fts( @@ -266,7 +266,7 @@ def test_fts_tokenize(fresh_db, fts_version): def test_fts_tokenize_escaped(fresh_db): # A malicious tokenize value must not be able to break out of the # string literal in the CREATE VIRTUAL TABLE statement. - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert_all(search_records) malicious = "porter'); CREATE TABLE injected(x); --" with pytest.raises(Exception): @@ -278,7 +278,7 @@ def test_fts_tokenize_escaped(fresh_db): def test_optimize_fts(fresh_db): for fts_version in ("4", "5"): table_name = f"searchable_{fts_version}" - table = fresh_db[table_name] + table = fresh_db.table(table_name) table.insert_all(search_records) table.enable_fts(["text", "country"], fts_version=f"FTS{fts_version}") # You can call optimize successfully against the tables OR their _fts equivalents: @@ -288,11 +288,11 @@ def test_optimize_fts(fresh_db): "searchable_4_fts", "searchable_5_fts", ): - fresh_db[table_name].optimize() + fresh_db.table(table_name).optimize() def test_enable_fts_with_triggers(fresh_db): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert(search_records[0]) table.enable_fts(["text", "country"], fts_version="FTS4", create_triggers=True) rows1 = list(table.search("tanuki")) @@ -321,7 +321,7 @@ def test_enable_fts_with_triggers(fresh_db): @pytest.mark.parametrize("create_triggers", [True, False]) def test_disable_fts(fresh_db, create_triggers): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert(search_records[0]) table.enable_fts(["text", "country"], create_triggers=create_triggers) assert { @@ -354,7 +354,7 @@ def test_disable_fts(fresh_db, create_triggers): def test_rebuild_fts(fresh_db): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert(search_records[0]) table.enable_fts(["text", "country"]) # Run a search @@ -380,7 +380,7 @@ def test_rebuild_fts(fresh_db): def test_optimize_and_rebuild_fts_commit(tmpdir, method): path = str(tmpdir / "test.db") db = Database(path) - table = db["searchable"] + table = db.table("searchable") table.insert(search_records[0]) table.enable_fts(["text", "country"]) getattr(table, method)() @@ -390,16 +390,16 @@ def test_optimize_and_rebuild_fts_commit(tmpdir, method): table.insert(search_records[1]) db.close() db2 = Database(path) - assert db2["searchable"].count == 2 + assert db2.table("searchable").count == 2 db2.close() @pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"]) def test_rebuild_fts_invalid(fresh_db, invalid_table): - fresh_db["not_searchable"].insert({"foo": "bar"}) + fresh_db.table("not_searchable").insert({"foo": "bar"}) # Raise OperationalError on invalid table with pytest.raises(sqlite3.OperationalError): - fresh_db[invalid_table].rebuild_fts() + fresh_db.table(invalid_table).rebuild_fts() @pytest.mark.parametrize("fts_version", ["FTS4", "FTS5"]) @@ -408,15 +408,17 @@ def test_rebuild_removes_junk_docsize_rows(tmpdir, fts_version): path = tmpdir / "test.db" db = Database(str(path), recursive_triggers=False) licenses = [{"key": "apache2", "name": "Apache 2"}, {"key": "bsd", "name": "BSD"}] - db["licenses"].insert_all(licenses, pk="key", replace=True) - db["licenses"].enable_fts(["name"], create_triggers=True, fts_version=fts_version) - assert db["licenses_fts_docsize"].count == 2 + db.table("licenses").insert_all(licenses, pk="key", replace=True) + db.table("licenses").enable_fts( + ["name"], create_triggers=True, fts_version=fts_version + ) + assert db.table("licenses_fts_docsize").count == 2 # Bug: insert with replace increases the number of rows in _docsize: - db["licenses"].insert_all(licenses, pk="key", replace=True) - assert db["licenses_fts_docsize"].count == 4 + db.table("licenses").insert_all(licenses, pk="key", replace=True) + assert db.table("licenses_fts_docsize").count == 4 # rebuild should fix this: - db["licenses_fts"].rebuild_fts() - assert db["licenses_fts_docsize"].count == 2 + db.table("licenses_fts").rebuild_fts() + assert db.table("licenses_fts_docsize").count == 2 @pytest.mark.parametrize( @@ -430,7 +432,7 @@ def test_rebuild_removes_junk_docsize_rows(tmpdir, fts_version): ) def test_enable_fts_replace(kwargs): db = Database(memory=True) - db["books"].insert( + db.table("books").insert( { "id": 1, "title": "Habits of Australian Marsupials", @@ -438,31 +440,31 @@ def test_enable_fts_replace(kwargs): }, pk="id", ) - db["books"].enable_fts(["title", "author"]) - assert not db["books"].triggers - assert db["books_fts"].columns_dict.keys() == {"title", "author"} - assert "FTS5" in db["books_fts"].schema - assert "porter" not in db["books_fts"].schema + db.table("books").enable_fts(["title", "author"]) + assert not db.table("books").triggers + assert db.table("books_fts").columns_dict.keys() == {"title", "author"} + assert "FTS5" in db.table("books_fts").schema + assert "porter" not in db.table("books_fts").schema # Now modify the FTS configuration should_have_changed_columns = "columns" in kwargs if "columns" not in kwargs: kwargs["columns"] = ["title", "author"] - db["books"].enable_fts(**kwargs, replace=True) + db.table("books").enable_fts(**kwargs, replace=True) # Check that the new configuration is correct if should_have_changed_columns: - assert db["books_fts"].columns_dict.keys() == {"title"} + assert db.table("books_fts").columns_dict.keys() == {"title"} if "create_triggers" in kwargs: - assert db["books"].triggers + assert db.table("books").triggers if "fts_version" in kwargs: - assert "FTS4" in db["books_fts"].schema + assert "FTS4" in db.table("books_fts").schema if "tokenize" in kwargs: - assert "porter" in db["books_fts"].schema + assert "porter" in db.table("books_fts").schema def test_enable_fts_replace_does_nothing_if_args_the_same(): queries = [] db = Database(memory=True, tracer=lambda sql, params: queries.append((sql, params))) - db["books"].insert( + db.table("books").insert( { "id": 1, "title": "Habits of Australian Marsupials", @@ -470,17 +472,19 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): }, pk="id", ) - db["books"].enable_fts(["title", "author"], create_triggers=True) + db.table("books").enable_fts(["title", "author"], create_triggers=True) queries.clear() # Running that again shouldn't run much SQL: - db["books"].enable_fts(["title", "author"], create_triggers=True, replace=True) + db.table("books").enable_fts( + ["title", "author"], create_triggers=True, replace=True + ) # The only SQL that executed should be select statements assert all(q[0].startswith("select ") for q in queries) def test_enable_fts_replace_handles_legacy_bracket_quoted_content_table(): db = Database(memory=True) - db["books"].insert( + db.table("books").insert( { "id": 1, "title": "Habits of Australian Marsupials", @@ -495,10 +499,10 @@ def test_enable_fts_replace_handles_legacy_bracket_quoted_content_table(): ); """) - db["books"].enable_fts(["title", "author"], replace=True) + db.table("books").enable_fts(["title", "author"], replace=True) - assert db["books_fts"].columns_dict.keys() == {"title", "author"} - assert 'content="books"' in db["books_fts"].schema + assert db.table("books_fts").columns_dict.keys() == {"title", "author"} + assert 'content="books"' in db.table("books_fts").schema def test_view_has_no_enable_fts(): @@ -506,7 +510,7 @@ def test_view_has_no_enable_fts(): db.create_view("hello", "select 1 + 1") # Views deliberately do not have an enable_fts() method with pytest.raises(AttributeError): - db["hello"].enable_fts() # type: ignore[union-attr] + db.view("hello").enable_fts() # type: ignore[union-attr] @pytest.mark.parametrize( @@ -712,14 +716,14 @@ def test_view_has_no_enable_fts(): ) def test_search_sql(kwargs, fts, expected): db = Database(memory=True) - db["books"].insert( + db.table("books").insert( { "title": "Habits of Australian Marsupials", "author": "Marlee Hawkins", } ) - db["books"].enable_fts(["title", "author"], fts_version=fts) - sql = db["books"].search_sql(**kwargs) + db.table("books").enable_fts(["title", "author"], fts_version=fts) + sql = db.table("books").search_sql(**kwargs) assert sql == expected @@ -740,7 +744,7 @@ def test_search_sql(kwargs, fts, expected): ), ) def test_quote_fts_query(fresh_db, input, expected): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert_all(search_records) table.enable_fts(["text", "country"]) quoted = fresh_db.quote_fts(input) @@ -750,7 +754,7 @@ def test_quote_fts_query(fresh_db, input, expected): def test_search_quote(fresh_db): - table = fresh_db["searchable"] + table = fresh_db.table("searchable") table.insert_all(search_records) table.enable_fts(["text", "country"]) query = "cat's" @@ -763,7 +767,7 @@ def test_search_quote(fresh_db): def test_enable_fts_cli_on_view_errors(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db["t"].insert({"text": "hello"}) + db.table("t").insert({"text": "hello"}) db.create_view("v", "select * from t") db.close() from click.testing import CliRunner diff --git a/tests/test_get.py b/tests/test_get.py index 3cdaed8..5e29506 100644 --- a/tests/test_get.py +++ b/tests/test_get.py @@ -4,14 +4,14 @@ from sqlite_utils.db import NotFoundError def test_get_rowid(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") cleo = {"name": "Cleo", "age": 4} row_id = dogs.insert(cleo).last_rowid assert cleo == dogs.get(row_id) def test_get_primary_key(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") cleo = {"name": "Cleo", "age": 4, "id": 5} last_pk = dogs.insert(cleo, pk="id").last_pk assert 5 == last_pk @@ -23,10 +23,10 @@ def test_get_primary_key(fresh_db): [(100, None), (None, None), ((1, 2), "Need 1 primary key value"), ("2", None)], ) def test_get_not_found(argument, expected_msg, fresh_db): - fresh_db["dogs"].insert( + fresh_db.table("dogs").insert( {"id": 1, "name": "Cleo", "age": 4, "is_good": True}, pk="id" ) with pytest.raises(NotFoundError) as excinfo: - fresh_db["dogs"].get(argument) + fresh_db.table("dogs").get(argument) if expected_msg is not None: assert expected_msg == excinfo.value.args[0] diff --git a/tests/test_gis.py b/tests/test_gis.py index 8b41d22..592af4c 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -45,7 +45,7 @@ def test_add_geometry_column(): coord_dimension="XY", ) - assert db["geometry_columns"].get(["locations", "geometry"]) == { + assert db.table("geometry_columns").get(["locations", "geometry"]) == { "f_table_name": "locations", "f_geometry_column": "geometry", "geometry_type": 1, # point @@ -133,7 +133,7 @@ def test_cli_add_geometry_column(tmpdir): db = Database(str(db_path)) db.init_spatialite() - table = db["locations"].create({"name": str}) + table = db.table("locations").create({"name": str}) result = CliRunner().invoke( cli, @@ -149,7 +149,7 @@ def test_cli_add_geometry_column(tmpdir): assert result.exit_code == 0 - assert db["geometry_columns"].get(["locations", "geometry"]) == { + assert db.table("geometry_columns").get(["locations", "geometry"]) == { "f_table_name": "locations", "f_geometry_column": "geometry", "geometry_type": 1, # point @@ -164,7 +164,7 @@ def test_cli_add_geometry_column_options(tmpdir): db_path = tmpdir / "spatial.db" db = Database(str(db_path)) db.init_spatialite() - table = db["locations"].create({"name": str}) + table = db.table("locations").create({"name": str}) result = CliRunner().invoke( cli, @@ -183,7 +183,7 @@ def test_cli_add_geometry_column_options(tmpdir): assert result.exit_code == 0 - assert db["geometry_columns"].get(["locations", "geometry"]) == { + assert db.table("geometry_columns").get(["locations", "geometry"]) == { "f_table_name": "locations", "f_geometry_column": "geometry", "geometry_type": 3, # polygon @@ -202,7 +202,7 @@ def test_cli_add_geometry_column_invalid_type(tmpdir): db = Database(str(db_path)) db.init_spatialite() - table = db["locations"].create({"name": str}) + table = db.table("locations").create({"name": str}) result = CliRunner().invoke( cli, @@ -225,7 +225,7 @@ def test_cli_create_spatial_index(tmpdir): db = Database(str(db_path)) db.init_spatialite() - table = db["locations"].create({"name": str}) + table = db.table("locations").create({"name": str}) table.add_geometry_column("geometry", "POINT") result = CliRunner().invoke( diff --git a/tests/test_hypothesis.py b/tests/test_hypothesis.py index ab652c7..d017f1f 100644 --- a/tests/test_hypothesis.py +++ b/tests/test_hypothesis.py @@ -11,8 +11,8 @@ def test_roundtrip_integers(integer): row = { "integer": integer, } - db["test"].insert(row) - assert list(db["test"].rows) == [row] + db.table("test").insert(row) + assert list(db.table("test").rows) == [row] @given(st.text()) @@ -21,8 +21,8 @@ def test_roundtrip_text(text): row = { "text": text, } - db["test"].insert(row) - assert list(db["test"].rows) == [row] + db.table("test").insert(row) + assert list(db.table("test").rows) == [row] @given(st.binary(max_size=1024 * 1024)) @@ -31,8 +31,8 @@ def test_roundtrip_binary(binary): row = { "binary": binary, } - db["test"].insert(row) - assert list(db["test"].rows) == [row] + db.table("test").insert(row) + assert list(db.table("test").rows) == [row] @given(st.floats(allow_nan=False)) @@ -41,5 +41,5 @@ def test_roundtrip_floats(floats): row = { "floats": floats, } - db["test"].insert(row) - assert list(db["test"].rows) == [row] + db.table("test").insert(row) + assert list(db.table("test").rows) == [row] diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 1724d2d..93c4daf 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -57,7 +57,7 @@ def test_insert_files(silent, pk_args, expected_pks): ) assert result.exit_code == 0, result.stdout db = Database(db_path) - rows_by_path = {r["path"]: r for r in db["files"].rows} + rows_by_path = {r["path"]: r for r in db.table("files").rows} one, two, three = ( rows_by_path["one.txt"], rows_by_path["two.txt"], @@ -114,7 +114,7 @@ def test_insert_files(silent, pk_args, expected_pks): for colname, expected_type in expected_types.items(): for row in (one, two, three): assert isinstance(row[colname], expected_type) - assert set(db["files"].pks) == set(expected_pks) + assert set(db.table("files").pks) == set(expected_pks) @pytest.mark.parametrize( @@ -144,7 +144,7 @@ def test_insert_files_stdin(use_text, encoding, input, expected): ) assert result.exit_code == 0, result.stdout db = Database(db_path) - row = next(iter(db["files"].rows)) + row = next(iter(db.table("files").rows)) key = "content" if use_text: key = "content_text" diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 2a8d579..b0953f1 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -1,6 +1,6 @@ import pytest -from sqlite_utils.db import Check, Database, Index, View, XIndex, XIndexColumn +from sqlite_utils.db import Check, Database, Index, Table, View, XIndex, XIndexColumn def _check_supports_strict(): @@ -21,10 +21,10 @@ def test_view_names(fresh_db): def test_table_names_fts4(existing_db): - existing_db["woo"].insert({"title": "Hello"}).enable_fts( + existing_db.table("woo").insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" ) - existing_db["woo2"].insert({"title": "Hello"}).enable_fts( + existing_db.table("woo2").insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS5" ) assert ["woo_fts"] == existing_db.table_names(fts4=True) @@ -32,17 +32,17 @@ def test_table_names_fts4(existing_db): def test_detect_fts(existing_db): - existing_db["woo"].insert({"title": "Hello"}).enable_fts( + existing_db.table("woo").insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" ) - existing_db["woo2"].insert({"title": "Hello"}).enable_fts( + existing_db.table("woo2").insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS5" ) - assert "woo_fts" == existing_db["woo"].detect_fts() - assert "woo_fts" == existing_db["woo_fts"].detect_fts() - assert "woo2_fts" == existing_db["woo2"].detect_fts() - assert "woo2_fts" == existing_db["woo2_fts"].detect_fts() - assert existing_db["foo"].detect_fts() is None + assert "woo_fts" == existing_db.table("woo").detect_fts() + assert "woo_fts" == existing_db.table("woo_fts").detect_fts() + assert "woo2_fts" == existing_db.table("woo2").detect_fts() + assert "woo2_fts" == existing_db.table("woo2_fts").detect_fts() + assert existing_db.table("foo").detect_fts() is None @pytest.mark.parametrize("reverse_order", (True, False)) @@ -52,14 +52,14 @@ def test_detect_fts_similar_tables(fresh_db, reverse_order): if reverse_order: table1, table2 = table2, table1 - fresh_db[table1].insert({"title": "Hello"}).enable_fts( + fresh_db.table(table1).insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" ) - fresh_db[table2].insert({"title": "Hello"}).enable_fts( + fresh_db.table(table2).insert({"title": "Hello"}).enable_fts( ["title"], fts_version="FTS4" ) - assert fresh_db[table1].detect_fts() == f"{table1}_fts" - assert fresh_db[table2].detect_fts() == f"{table2}_fts" + assert fresh_db.table(table1).detect_fts() == f"{table1}_fts" + assert fresh_db.table(table2).detect_fts() == f"{table2}_fts" def test_tables(existing_db): @@ -77,26 +77,34 @@ def test_views(fresh_db): assert view.columns_dict == {"1": str} +def test_getitem_returns_table_or_view(fresh_db): + fresh_db.table("items").insert({"id": 1}, pk="id") + fresh_db.create_view("item_ids", "select id from items") + + assert isinstance(fresh_db["items"], Table) + assert isinstance(fresh_db["item_ids"], View) + + def test_count(existing_db): - assert existing_db["foo"].count == 3 - assert existing_db["foo"].count_where() == 3 - assert existing_db["foo"].execute_count() == 3 + assert existing_db.table("foo").count == 3 + assert existing_db.table("foo").count_where() == 3 + assert existing_db.table("foo").execute_count() == 3 def test_count_where(existing_db): - assert existing_db["foo"].count_where("text != ?", ["two"]) == 2 - assert existing_db["foo"].count_where("text != :t", {"t": "two"}) == 2 + assert existing_db.table("foo").count_where("text != ?", ["two"]) == 2 + assert existing_db.table("foo").count_where("text != :t", {"t": "two"}) == 2 def test_columns(existing_db): - table = existing_db["foo"] + table = existing_db.table("foo") assert [{"name": "text", "type": "TEXT"}] == [ {"name": col.name, "type": col.type} for col in table.columns ] def test_table_schema(existing_db): - assert existing_db["foo"].schema == "CREATE TABLE foo (text TEXT)" + assert existing_db.table("foo").schema == "CREATE TABLE foo (text TEXT)" def test_database_schema(existing_db): @@ -104,9 +112,9 @@ def test_database_schema(existing_db): def test_table_repr(fresh_db): - table = fresh_db["dogs"].insert({"name": "Cleo", "age": 4}) + table = fresh_db.table("dogs").insert({"name": "Cleo", "age": 4}) assert "
" == repr(table) - assert "
" == repr(fresh_db["cats"]) + assert "
" == repr(fresh_db.table("cats")) def test_indexes(fresh_db): @@ -125,7 +133,7 @@ def test_indexes(fresh_db): columns=["c2", "c3"], ), Index(seq=1, name="Gosh_c1", unique=0, origin="c", partial=0, columns=["c1"]), - ] == fresh_db["Gosh"].indexes + ] == fresh_db.table("Gosh").indexes def test_xindexes(fresh_db): @@ -134,7 +142,7 @@ def test_xindexes(fresh_db): create index Gosh_c1 on Gosh(c1); create index Gosh_c2c3 on Gosh(c2, c3 desc); """) - assert fresh_db["Gosh"].xindexes == [ + assert fresh_db.table("Gosh").xindexes == [ XIndex( name="Gosh_c2c3", columns=[ @@ -166,15 +174,15 @@ def test_xindexes(fresh_db): def test_guess_foreign_table(fresh_db, column, expected_table_guess): fresh_db.create_table("authors", {"name": str}) fresh_db.create_table("genre", {"name": str}) - assert expected_table_guess == fresh_db["books"].guess_foreign_table(column) + assert expected_table_guess == fresh_db.table("books").guess_foreign_table(column) @pytest.mark.parametrize( "pk,expected", ((None, ["rowid"]), ("id", ["id"]), (["id", "id2"], ["id", "id2"])) ) def test_pks(fresh_db, pk, expected): - fresh_db["foo"].insert_all([{"id": 1, "id2": 2}], pk=pk) - assert expected == fresh_db["foo"].pks + fresh_db.table("foo").insert_all([{"id": 1, "id2": 2}], pk=pk) + assert expected == fresh_db.table("foo").pks def test_checks(fresh_db): @@ -185,7 +193,7 @@ def test_checks(fresh_db): CONSTRAINT within_maximum CHECK(score <= maximum) ) """) - scores = fresh_db["scores"] + scores = fresh_db.table("scores") expected_column = Check("score > 0", name="positive", column="score") expected_table = Check("score <= maximum", name="within_maximum") assert scores.checks == [expected_column, expected_table] @@ -195,26 +203,26 @@ def test_checks(fresh_db): def test_checks_nonexistent_and_virtual_tables(fresh_db): - assert fresh_db["does_not_exist"].checks == [] - fresh_db["searchable"].insert({"text": "hello"}).enable_fts( + assert fresh_db.table("does_not_exist").checks == [] + fresh_db.table("searchable").insert({"text": "hello"}).enable_fts( ["text"], fts_version="FTS5" ) - assert fresh_db["searchable_fts"].checks == [] + assert fresh_db.table("searchable_fts").checks == [] def test_triggers_and_triggers_dict(fresh_db): assert [] == fresh_db.triggers - authors = fresh_db["authors"] + authors = fresh_db.table("authors") authors.insert_all( [ {"name": "Frank Herbert", "famous_works": "Dune"}, {"name": "Neal Stephenson", "famous_works": "Cryptonomicon"}, ] ) - fresh_db["other"].insert({"foo": "bar"}) + fresh_db.table("other").insert({"foo": "bar"}) assert authors.triggers == [] assert authors.triggers_dict == {} - assert fresh_db["other"].triggers == [] + assert fresh_db.table("other").triggers == [] assert fresh_db.triggers_dict == {} authors.enable_fts( ["name", "famous_works"], fts_version="FTS4", create_triggers=True @@ -226,7 +234,7 @@ def test_triggers_and_triggers_dict(fresh_db): } assert expected_triggers == {(t.name, t.table) for t in fresh_db.triggers} assert expected_triggers == { - (t.name, t.table) for t in fresh_db["authors"].triggers + (t.name, t.table) for t in fresh_db.table("authors").triggers } expected_triggers = { "authors_ai": ( @@ -246,13 +254,13 @@ def test_triggers_and_triggers_dict(fresh_db): ), } assert authors.triggers_dict == expected_triggers - assert fresh_db["other"].triggers == [] - assert fresh_db["other"].triggers_dict == {} + assert fresh_db.table("other").triggers == [] + assert fresh_db.table("other").triggers_dict == {} assert fresh_db.triggers_dict == expected_triggers def test_has_counts_triggers(fresh_db): - authors = fresh_db["authors"] + authors = fresh_db.table("authors") authors.insert({"name": "Frank Herbert"}) assert not authors.has_counts_triggers authors.enable_counts() @@ -301,14 +309,14 @@ def test_has_counts_triggers(fresh_db): ) def test_virtual_table_using(fresh_db, sql, expected_name, expected_using): fresh_db.execute(sql) - assert fresh_db[expected_name].virtual_table_using == expected_using + assert fresh_db.table(expected_name).virtual_table_using == expected_using def test_use_rowid(fresh_db): - fresh_db["rowid_table"].insert({"name": "Cleo"}) - fresh_db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id") - assert fresh_db["rowid_table"].use_rowid - assert not fresh_db["regular_table"].use_rowid + fresh_db.table("rowid_table").insert({"name": "Cleo"}) + fresh_db.table("regular_table").insert({"id": 1, "name": "Cleo"}, pk="id") + assert fresh_db.table("rowid_table").use_rowid + assert not fresh_db.table("regular_table").use_rowid @pytest.mark.skipif( @@ -327,7 +335,7 @@ def test_use_rowid(fresh_db): ) def test_table_strict(fresh_db, create_table, expected_strict): fresh_db.execute(create_table) - table = fresh_db["t"] + table = fresh_db.table("t") assert table.strict == expected_strict @@ -343,10 +351,10 @@ def test_table_strict(fresh_db, create_table, expected_strict): ), ) def test_table_default_values(fresh_db, value): - fresh_db["default_values"].insert( + fresh_db.table("default_values").insert( {"nodefault": 1, "value": value}, defaults={"value": value} ) - default_values = fresh_db["default_values"].default_values + default_values = fresh_db.table("default_values").default_values assert default_values == {"value": value} @@ -356,8 +364,8 @@ def test_table_default_values_escaped_quotes(fresh_db): fresh_db.execute( "create table t (id integer primary key, name text default 'O''Brien')" ) - assert "default 'O''Brien'" in fresh_db["t"].schema - assert fresh_db["t"].default_values == {"name": "O'Brien"} + assert "default 'O''Brien'" in fresh_db.table("t").schema + assert fresh_db.table("t").default_values == {"name": "O'Brien"} def test_pks_use_primary_key_declaration_order(fresh_db): @@ -365,11 +373,11 @@ def test_pks_use_primary_key_declaration_order(fresh_db): # pks must follow the declaration order, which is what SQLite uses to # resolve implicit foreign key references and compound pk lookups fresh_db.execute("create table t (b text, a text, primary key (a, b))") - assert fresh_db["t"].pks == ["a", "b"] + assert fresh_db.table("t").pks == ["a", "b"] def test_transform_preserves_compound_pk_declaration_order(fresh_db): fresh_db.execute("create table t (a text, b text, c text, primary key (b, a))") - fresh_db["t"].transform(drop={"c"}) - assert fresh_db["t"].pks == ["b", "a"] - assert 'PRIMARY KEY ("b", "a")' in fresh_db["t"].schema + fresh_db.table("t").transform(drop={"c"}) + assert fresh_db.table("t").pks == ["b", "a"] + assert 'PRIMARY KEY ("b", "a")' in fresh_db.table("t").schema diff --git a/tests/test_list_mode.py b/tests/test_list_mode.py index 646098e..b9ab812 100644 --- a/tests/test_list_mode.py +++ b/tests/test_list_mode.py @@ -19,9 +19,9 @@ def test_insert_all_list_mode_basic(): yield [2, "Bob", 25] yield [3, "Charlie", 35] - db["people"].insert_all(data_generator()) + db.table("people").insert_all(data_generator()) - rows = list(db["people"].rows) + rows = list(db.table("people").rows) assert len(rows) == 3 assert rows[0] == {"id": 1, "name": "Alice", "age": 30} assert rows[1] == {"id": 2, "name": "Bob", "age": 25} @@ -37,10 +37,10 @@ def test_insert_all_list_mode_with_pk(): yield [1, "Alice", 95] yield [2, "Bob", 87] - db["scores"].insert_all(data_generator(), pk="id") + db.table("scores").insert_all(data_generator(), pk="id") - assert db["scores"].pks == ["id"] - rows = list(db["scores"].rows) + assert db.table("scores").pks == ["id"] + rows = list(db.table("scores").rows) assert len(rows) == 2 @@ -54,7 +54,7 @@ def test_upsert_all_list_mode(): yield [1, "Alice", 100] yield [2, "Bob", 200] - db["data"].insert_all(initial_data(), pk="id") + db.table("data").insert_all(initial_data(), pk="id") # Upsert with some updates and new records def upsert_data(): @@ -62,9 +62,9 @@ def test_upsert_all_list_mode(): yield [1, "Alice", 150] # Update existing yield [3, "Charlie", 300] # Insert new - db["data"].upsert_all(upsert_data(), pk="id") + db.table("data").upsert_all(upsert_data(), pk="id") - rows = list(db["data"].rows_where(order_by="id")) + rows = list(db.table("data").rows_where(order_by="id")) assert len(rows) == 3 assert rows[0] == {"id": 1, "name": "Alice", "value": 150} assert rows[1] == {"id": 2, "name": "Bob", "value": 200} @@ -81,9 +81,9 @@ def test_list_mode_with_various_types(): yield [2, "Bob", 87.3, False] yield [3, "Charlie", None, True] - db["mixed"].insert_all(data_generator()) + db.table("mixed").insert_all(data_generator()) - rows = list(db["mixed"].rows) + rows = list(db.table("mixed").rows) assert len(rows) == 3 assert rows[0]["score"] == 95.5 assert rows[1]["active"] == 0 # SQLite stores boolean as int @@ -99,7 +99,7 @@ def test_list_mode_error_non_string_columns(): yield ["a", "b", "c"] with pytest.raises(ValueError, match="must be a list of column name strings"): - db["bad"].insert_all(bad_data()) + db.table("bad").insert_all(bad_data()) def test_list_mode_error_mixed_types(): @@ -111,7 +111,7 @@ def test_list_mode_error_mixed_types(): yield {"id": 1, "name": "Alice"} # Should be a list, not dict with pytest.raises(ValueError, match="must also be lists"): - db["bad"].insert_all(bad_data()) + db.table("bad").insert_all(bad_data()) def test_list_mode_empty_after_headers(): @@ -122,9 +122,9 @@ def test_list_mode_empty_after_headers(): yield ["id", "name", "age"] # No data rows - result = db["people"].insert_all(data_generator()) + result = db.table("people").insert_all(data_generator()) assert result is not None - assert not db["people"].exists() + assert not db.table("people").exists() def test_list_mode_batch_processing(): @@ -136,7 +136,7 @@ def test_list_mode_batch_processing(): for i in range(1000): yield [i, f"value_{i}"] - db["large"].insert_all(large_data(), batch_size=100) + db.table("large").insert_all(large_data(), batch_size=100) count = db.execute("SELECT COUNT(*) as c FROM large").fetchone()[0] assert count == 1000 @@ -152,9 +152,9 @@ def test_list_mode_shorter_rows(): yield [2, "Bob"] # Missing age and city yield [3, "Charlie", 35] # Missing city - db["people"].insert_all(data_generator()) + db.table("people").insert_all(data_generator()) - rows = list(db["people"].rows_where(order_by="id")) + rows = list(db.table("people").rows_where(order_by="id")) assert rows[0] == {"id": 1, "name": "Alice", "age": 30, "city": "NYC"} assert rows[1] == {"id": 2, "name": "Bob", "age": None, "city": None} assert rows[2] == {"id": 3, "name": "Charlie", "age": 35, "city": None} @@ -170,9 +170,9 @@ def test_backwards_compatibility_dict_mode(): {"id": 2, "name": "Bob", "age": 25}, ] - db["people"].insert_all(data) + db.table("people").insert_all(data) - rows = list(db["people"].rows) + rows = list(db.table("people").rows) assert len(rows) == 2 assert rows[0] == {"id": 1, "name": "Alice", "age": 30} @@ -189,9 +189,9 @@ def test_insert_all_tuple_mode_basic(): yield (2, "Bob", 25) yield (3, "Charlie", 35) - db["people"].insert_all(data_generator()) + db.table("people").insert_all(data_generator()) - rows = list(db["people"].rows) + rows = list(db.table("people").rows) assert len(rows) == 3 assert rows[0] == {"id": 1, "name": "Alice", "age": 30} assert rows[1] == {"id": 2, "name": "Bob", "age": 25} @@ -211,9 +211,9 @@ def test_insert_all_mixed_list_tuple(): yield [3, "Charlie", 35] yield (4, "Diana", 40) - db["people"].insert_all(data_generator()) + db.table("people").insert_all(data_generator()) - rows = list(db["people"].rows) + rows = list(db.table("people").rows) assert len(rows) == 4 assert rows[0] == {"id": 1, "name": "Alice", "age": 30} assert rows[1] == {"id": 2, "name": "Bob", "age": 25} @@ -231,7 +231,7 @@ def test_upsert_all_tuple_mode(): yield (1, "Alice", 100) yield (2, "Bob", 200) - db["data"].insert_all(initial_data(), pk="id") + db.table("data").insert_all(initial_data(), pk="id") # Upsert with tuples def upsert_data(): @@ -239,9 +239,9 @@ def test_upsert_all_tuple_mode(): yield (1, "Alice", 150) # Update existing yield (3, "Charlie", 300) # Insert new - db["data"].upsert_all(upsert_data(), pk="id") + db.table("data").upsert_all(upsert_data(), pk="id") - rows = list(db["data"].rows_where(order_by="id")) + rows = list(db.table("data").rows_where(order_by="id")) assert len(rows) == 3 assert rows[0] == {"id": 1, "name": "Alice", "value": 150} assert rows[1] == {"id": 2, "name": "Bob", "value": 200} @@ -258,9 +258,9 @@ def test_tuple_mode_shorter_rows(): yield 2, "Bob" # Missing age and city yield 3, "Charlie", 35 # Missing city - db["people"].insert_all(data_generator()) + db.table("people").insert_all(data_generator()) - rows = list(db["people"].rows_where(order_by="id")) + rows = list(db.table("people").rows_where(order_by="id")) assert rows[0] == {"id": 1, "name": "Alice", "age": 30, "city": "NYC"} assert rows[1] == {"id": 2, "name": "Bob", "age": None, "city": None} assert rows[2] == {"id": 3, "name": "Charlie", "age": 35, "city": None} @@ -271,18 +271,18 @@ def test_list_mode_single_record_upsert_last_pk(): db = Database(memory=True) # Create table first - db["data"].insert({"id": 1, "name": "Alice", "value": 100}, pk="id") + db.table("data").insert({"id": 1, "name": "Alice", "value": 100}, pk="id") # Now upsert a single record using list mode def upsert_data(): yield ["id", "name", "value"] yield [1, "Alice", 150] # Update existing - table = db["data"] + table = db.table("data") table.upsert_all(upsert_data(), pk="id") # Verify the data was updated - rows = list(db["data"].rows) + rows = list(db.table("data").rows) assert rows == [{"id": 1, "name": "Alice", "value": 150}] # Verify last_pk is populated correctly diff --git a/tests/test_lookup.py b/tests/test_lookup.py index c93d1ed..f96cfef 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -4,7 +4,7 @@ from sqlite_utils.db import Index def test_lookup_new_table(fresh_db): - species = fresh_db["species"] + species = fresh_db.table("species") palm_id = species.lookup({"name": "Palm"}) oak_id = species.lookup({"name": "Oak"}) cherry_id = species.lookup({"name": "Cherry"}) @@ -26,7 +26,7 @@ def test_lookup_new_table(fresh_db): def test_lookup_new_table_compound_key(fresh_db): - species = fresh_db["species"] + species = fresh_db.table("species") palm_id = species.lookup({"name": "Palm", "type": "Tree"}) oak_id = species.lookup({"name": "Oak", "type": "Tree"}) assert palm_id == species.lookup({"name": "Palm", "type": "Tree"}) @@ -70,7 +70,7 @@ def test_lookup_fails_if_constraint_cannot_be_added(fresh_db): def test_lookup_with_extra_values(fresh_db): - species = fresh_db["species"] + species = fresh_db.table("species") id = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2020-01-01"}) assert species.get(id) == { "id": 1, @@ -90,9 +90,9 @@ def test_lookup_with_extra_values(fresh_db): def test_lookup_with_extra_insert_parameters(fresh_db): - other_table = fresh_db["other_table"] + other_table = fresh_db.table("other_table") other_table.insert({"id": 1, "name": "Name"}, pk="id") - species = fresh_db["species"] + species = fresh_db.table("species") id = species.lookup( {"name": "Palm", "type": "Tree"}, { @@ -156,15 +156,15 @@ def test_lookup_with_extra_insert_parameters(fresh_db): @pytest.mark.parametrize("strict", (False, True)) def test_lookup_new_table_strict(fresh_db, strict): - fresh_db["species"].lookup({"name": "Palm"}, strict=strict) - assert fresh_db["species"].strict == strict or not fresh_db.supports_strict + fresh_db.table("species").lookup({"name": "Palm"}, strict=strict) + assert fresh_db.table("species").strict == strict or not fresh_db.supports_strict def test_lookup_null_value_idempotent(fresh_db): # https://github.com/simonw/sqlite-utils/issues/186 # Repeated lookups of a null value should return the same row, # not insert a duplicate row each time - species = fresh_db["species"] + species = fresh_db.table("species") first_id = species.lookup({"name": None}) second_id = species.lookup({"name": None}) assert first_id == second_id @@ -172,7 +172,7 @@ def test_lookup_null_value_idempotent(fresh_db): def test_lookup_compound_key_with_null_idempotent(fresh_db): - species = fresh_db["species"] + species = fresh_db.table("species") palm_id = species.lookup({"name": "Palm", "type": None}) oak_id = species.lookup({"name": "Oak", "type": "Tree"}) assert palm_id == species.lookup({"name": "Palm", "type": None}) diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 4fca918..4dde7e4 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -4,45 +4,45 @@ from sqlite_utils.db import ForeignKey, NoObviousTable def test_insert_m2m_single(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( "humans", {"id": 1, "name": "Natalie D"}, pk="id" ) assert {"dogs_humans", "humans", "dogs"} == set(fresh_db.table_names()) - humans = fresh_db["humans"] - dogs_humans = fresh_db["dogs_humans"] + humans = fresh_db.table("humans") + dogs_humans = fresh_db.table("dogs_humans") assert [{"id": 1, "name": "Natalie D"}] == list(humans.rows) assert [{"humans_id": 1, "dogs_id": 1}] == list(dogs_humans.rows) def test_insert_m2m_alter(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( "humans", {"id": 1, "name": "Natalie D"}, pk="id" ) dogs.update(1).m2m( "humans", {"id": 2, "name": "Simon W", "nerd": True}, pk="id", alter=True ) - assert list(fresh_db["humans"].rows) == [ + assert list(fresh_db.table("humans").rows) == [ {"id": 1, "name": "Natalie D", "nerd": None}, {"id": 2, "name": "Simon W", "nerd": 1}, ] - assert list(fresh_db["dogs_humans"].rows) == [ + assert list(fresh_db.table("dogs_humans").rows) == [ {"humans_id": 1, "dogs_id": 1}, {"humans_id": 2, "dogs_id": 1}, ] def test_insert_m2m_list(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( "humans", [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}], pk="id", ) assert {"dogs", "humans", "dogs_humans"} == set(fresh_db.table_names()) - humans = fresh_db["humans"] - dogs_humans = fresh_db["dogs_humans"] + humans = fresh_db.table("humans") + dogs_humans = fresh_db.table("dogs_humans") assert [{"humans_id": 1, "dogs_id": 1}, {"humans_id": 2, "dogs_id": 1}] == list( dogs_humans.rows ) @@ -68,7 +68,7 @@ def test_insert_m2m_iterable(fresh_db): def iterable(): yield from iterable_records - platypuses = fresh_db["platypuses"] + platypuses = fresh_db.table("platypuses") platypuses.insert({"id": 1, "name": "Perry"}, pk="id").m2m( "humans", iterable(), @@ -76,8 +76,8 @@ def test_insert_m2m_iterable(fresh_db): ) assert {"platypuses", "humans", "humans_platypuses"} == set(fresh_db.table_names()) - humans = fresh_db["humans"] - humans_platypuses = fresh_db["humans_platypuses"] + humans = fresh_db.table("humans") + humans_platypuses = fresh_db.table("humans_platypuses") assert [ {"humans_id": 1, "platypuses_id": 1}, {"humans_id": 2, "platypuses_id": 1}, @@ -111,14 +111,14 @@ def test_m2m_with_table_objects(fresh_db): assert expected_tables == set(fresh_db.table_names()) assert dogs.count == 1 assert humans.count == 2 - assert fresh_db["dogs_humans"].count == 2 + assert fresh_db.table("dogs_humans").count == 2 def test_m2m_lookup(fresh_db): people = fresh_db.table("people", pk="id") people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) - people_tags = fresh_db["people_tags"] - tags = fresh_db["tags"] + people_tags = fresh_db.table("people_tags") + tags = fresh_db.table("tags") assert people_tags.exists() assert tags.exists() assert [ @@ -150,9 +150,9 @@ def test_m2m_explicit_table_name_argument(fresh_db): people.insert({"name": "Wahyu"}).m2m( "tags", lookup={"tag": "Coworker"}, m2m_table="tagged" ) - assert fresh_db["tags"].exists - assert fresh_db["tagged"].exists - assert not fresh_db["people_tags"].exists() + assert fresh_db.table("tags").exists + assert fresh_db.table("tagged").exists + assert not fresh_db.table("people_tags").exists() def test_m2m_table_candidates(fresh_db): @@ -181,25 +181,25 @@ def test_uses_existing_m2m_table_if_exists(fresh_db): # Code should look for an existing table with fks to both tables # and use that if it exists. people = fresh_db.create_table("people", {"id": int, "name": str}, pk="id") - fresh_db["tags"].lookup({"tag": "Coworker"}) + fresh_db.table("tags").lookup({"tag": "Coworker"}) fresh_db.create_table( "tagged", {"people_id": int, "tags_id": int}, foreign_keys=["people_id", "tags_id"], ) people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) - assert fresh_db["tags"].exists() - assert fresh_db["tagged"].exists() - assert not fresh_db["people_tags"].exists() - assert not fresh_db["tags_people"].exists() - assert [{"people_id": 1, "tags_id": 1}] == list(fresh_db["tagged"].rows) + assert fresh_db.table("tags").exists() + assert fresh_db.table("tagged").exists() + assert not fresh_db.table("people_tags").exists() + assert not fresh_db.table("tags_people").exists() + assert [{"people_id": 1, "tags_id": 1}] == list(fresh_db.table("tagged").rows) def test_requires_explicit_m2m_table_if_multiple_options(fresh_db): # If the code scans for m2m tables and finds more than one candidate # it should require that the m2m_table=x argument is used people = fresh_db.create_table("people", {"id": int, "name": str}, pk="id") - fresh_db["tags"].lookup({"tag": "Coworker"}) + fresh_db.table("tags").lookup({"tag": "Coworker"}) fresh_db.create_table( "tagged", {"people_id": int, "tags_id": int}, diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 3f3dfea..fa419ec 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -10,11 +10,11 @@ def migrations(): @migrations() def m001(db): - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) @migrations() def m002(db): - db["cats"].create({"name": str}) + db.table("cats").create({"name": str}) db.execute("insert into dogs (name) values ('Pancakes')") return migrations @@ -28,11 +28,11 @@ def migrations_not_ordered_alphabetically(): @migrations() def m002(db): - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) @migrations() def m001(db): - db["cats"].create({"name": str}) + db.table("cats").create({"name": str}) db.execute("insert into dogs (name) values ('Pancakes')") return migrations @@ -44,7 +44,7 @@ def migrations2(): @migrations() def m001(db): - db["dogs2"].insert({"name": "Cleo"}) + db.table("dogs2").insert({"name": "Cleo"}) return migrations @@ -96,7 +96,7 @@ def test_applied_at_is_a_string(migrations): def test_failing_migration_rolls_back(migrations): @migrations() def m003(db): - db["birds"].create({"name": str}) + db.table("birds").create({"name": str}) db.execute("insert into dogs (name) values ('Dozer')") raise ValueError("boom") @@ -105,7 +105,7 @@ def test_failing_migration_rolls_back(migrations): migrations.apply(db) # m001 and m002 committed before the failure and stay applied assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"} - assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"] + assert [r["name"] for r in db.table("dogs").rows] == ["Cleo", "Pancakes"] assert [m.name for m in migrations.applied(db)] == ["m001", "m002"] # Everything m003 did was rolled back and it is still pending assert [m.name for m in migrations.pending(db)] == ["m003"] @@ -117,11 +117,11 @@ def test_rerun_after_failure_applies_each_migration_once(): @migrations() def m001(db): - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) @migrations() def m002(db): - db["dogs"].insert({"name": "Pancakes"}) + db.table("dogs").insert({"name": "Pancakes"}) if state["fail"]: raise ValueError("boom") @@ -131,7 +131,7 @@ def test_rerun_after_failure_applies_each_migration_once(): state["fail"] = False migrations.apply(db) # m001 must not have been re-applied, m002 applied exactly once - assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"] + assert [r["name"] for r in db.table("dogs").rows] == ["Cleo", "Pancakes"] assert [m.name for m in migrations.applied(db)] == ["m001", "m002"] @@ -142,7 +142,7 @@ def test_non_transactional_migration_allows_vacuum(tmpdir): @migrations() def m001(db): - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) @migrations(transactional=False) def m002(db): @@ -185,11 +185,13 @@ def test_apply_composes_inside_outer_transaction(migrations): ) def test_upgrades_sqlite_migrations(migrations, create_table, pk): db = sqlite_utils.Database(memory=True) - db["_sqlite_migrations"].create(create_table, pk=pk) + db.table("_sqlite_migrations").create(create_table, pk=pk) assert db.table_names() == ["_sqlite_migrations"] - assert db["_sqlite_migrations"].pks == ([pk] if isinstance(pk, str) else list(pk)) + assert db.table("_sqlite_migrations").pks == ( + [pk] if isinstance(pk, str) else list(pk) + ) migrations.apply(db) - assert db["_sqlite_migrations"].pks == ["id"] + assert db.table("_sqlite_migrations").pks == ["id"] def test_pending_and_applied_are_read_only(migrations): @@ -227,7 +229,7 @@ def test_stop_before_applied_migration_errors(migrations): assert "m001" in str(ex.value) assert "already been applied" in str(ex.value) # Nothing else was applied - assert not db["cats"].exists() + assert not db.table("cats").exists() def test_stop_before_applied_migration_errors_before_any_apply(migrations): @@ -238,9 +240,9 @@ def test_stop_before_applied_migration_errors_before_any_apply(migrations): @only_second() def m002(db): - db["cats"].create({"name": str}) + db.table("cats").create({"name": str}) only_second.apply(db) # m002 applied, m001 still pending with pytest.raises(ValueError): migrations.apply(db, stop_before="m002") - assert not db["dogs"].exists() + assert not db.table("dogs").exists() diff --git a/tests/test_mutator_transactions.py b/tests/test_mutator_transactions.py index 37ae1b6..3f13c6b 100644 --- a/tests/test_mutator_transactions.py +++ b/tests/test_mutator_transactions.py @@ -112,7 +112,7 @@ def test_mutator_commits_by_default(tmp_path, mutate, expected_rows): db = seed_database(path) assert not db.conn.in_transaction - mutate(db["items"]) + mutate(db.table("items")) assert current_rows(db) == expected_rows assert not db.conn.in_transaction @@ -127,7 +127,7 @@ def test_mutator_commits_with_outer_atomic(tmp_path, mutate, expected_rows): with db.atomic(): assert db.conn.in_transaction - mutate(db["items"]) + mutate(db.table("items")) assert current_rows(db) == expected_rows assert db.conn.in_transaction @@ -143,7 +143,7 @@ def test_mutator_rolls_back_outer_atomic(tmp_path, mutate, expected_rows): db = seed_database(path) with pytest.raises(RollbackTest), db.atomic(): - mutate(db["items"]) + mutate(db.table("items")) assert current_rows(db) == expected_rows assert db.conn.in_transaction raise RollbackTest diff --git a/tests/test_query.py b/tests/test_query.py index 9d79755..ac0d924 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -6,7 +6,7 @@ from sqlite_utils.utils import sqlite3 def test_query(fresh_db): - fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) + fresh_db.table("dogs").insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) results = fresh_db.query("select * from dogs order by name desc") assert isinstance(results, types.GeneratorType) assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}] @@ -20,13 +20,13 @@ def test_query_executes_eagerly(fresh_db): def test_query_rejects_statements_that_return_no_rows(fresh_db): - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) with pytest.raises(ValueError) as ex: fresh_db.query("update dogs set name = 'Cleopaws'") assert "execute()" in str(ex.value) # The rejected update was rolled back, and no transaction is left open assert not fresh_db.conn.in_transaction - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + assert [row["name"] for row in fresh_db.table("dogs").rows] == ["Cleo"] def test_query_rejected_ddl_is_rolled_back(fresh_db): @@ -37,7 +37,7 @@ def test_query_rejected_ddl_is_rolled_back(fresh_db): def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db): - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) fresh_db.begin() fresh_db.execute("insert into dogs (name) values ('Pancakes')") with pytest.raises(ValueError): @@ -45,7 +45,7 @@ def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db): # The transaction is still open and the earlier insert is intact assert fresh_db.conn.in_transaction fresh_db.commit() - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo", "Pancakes"] + assert [row["name"] for row in fresh_db.table("dogs").rows] == ["Cleo", "Pancakes"] @pytest.mark.parametrize( @@ -77,7 +77,7 @@ def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db): # A COMMIT hidden behind a leading comment must not slip past the # keyword check - previously it committed the caller's open # transaction before the ValueError was raised - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) fresh_db.begin() fresh_db.execute("insert into dogs (name) values ('Pancakes')") with pytest.raises(ValueError): @@ -85,7 +85,7 @@ def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db): # The explicit transaction is still open and can still be rolled back assert fresh_db.conn.in_transaction fresh_db.rollback() - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + assert [row["name"] for row in fresh_db.table("dogs").rows] == ["Cleo"] @pytest.mark.parametrize("sql", ["; COMMIT", "\ufeffCOMMIT"]) @@ -94,7 +94,7 @@ def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql): # real token, so the keyword scanner must skip them too - previously # '; COMMIT' slipped past the check and committed the caller's open # transaction before raising OperationalError - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) fresh_db.begin() fresh_db.execute("insert into dogs (name) values ('Pancakes')") with pytest.raises(ValueError): @@ -102,7 +102,7 @@ def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql): # The explicit transaction is still open and can still be rolled back assert fresh_db.conn.in_transaction fresh_db.rollback() - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + assert [row["name"] for row in fresh_db.table("dogs").rows] == ["Cleo"] def test_query_error_leaves_no_transaction_open(fresh_db): @@ -190,12 +190,12 @@ def test_first_keyword(sql, expected): reason="RETURNING requires SQLite 3.35.0 or higher", ) def test_query_insert_returning(fresh_db): - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) rows = list( fresh_db.query("insert into dogs (name) values ('Pancakes') returning name") ) assert rows == [{"name": "Pancakes"}] - assert fresh_db["dogs"].count == 2 + assert fresh_db.table("dogs").count == 2 @pytest.mark.skipif( @@ -207,7 +207,7 @@ def test_query_insert_returning_commits_without_iteration(tmpdir): path = str(tmpdir / "test.db") db = Database(path) - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) # Never iterate over the results db.query("insert into dogs (name) values ('Pancakes') returning name") assert not db.conn.in_transaction @@ -227,7 +227,7 @@ def test_query_insert_returning_partial_iteration_still_commits(tmpdir): path = str(tmpdir / "test.db") db = Database(path) - db["dogs"].insert({"name": "Cleo"}) + db.table("dogs").insert({"name": "Cleo"}) row = next( db.query( "insert into dogs (name) values ('Pancakes'), ('Marnie') returning name" @@ -246,7 +246,7 @@ def test_query_insert_returning_partial_iteration_still_commits(tmpdir): reason="RETURNING requires SQLite 3.35.0 or higher", ) def test_query_insert_returning_respects_explicit_transaction(fresh_db): - fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.table("dogs").insert({"name": "Cleo"}) fresh_db.begin() rows = list( fresh_db.query("insert into dogs (name) values ('Pancakes') returning name") @@ -255,13 +255,13 @@ def test_query_insert_returning_respects_explicit_transaction(fresh_db): # Still inside the explicit transaction - not committed assert fresh_db.conn.in_transaction fresh_db.rollback() - assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + assert [row["name"] for row in fresh_db.table("dogs").rows] == ["Cleo"] def test_query_duplicate_column_names_are_deduped(fresh_db): # https://github.com/simonw/sqlite-utils/issues/624 - fresh_db["one"].insert({"id": 1, "value": "left"}) - fresh_db["two"].insert({"id": 2, "value": "right"}) + fresh_db.table("one").insert({"id": 1, "value": "left"}) + fresh_db.table("two").insert({"id": 2, "value": "right"}) rows = list( fresh_db.query("select one.id, two.id, one.value, two.value from one, two") ) @@ -277,7 +277,7 @@ def test_query_deduped_column_avoids_existing_names(fresh_db): def test_execute_returning_dicts(fresh_db): # Like db.query() but returns a list, included for backwards compatibility # see https://github.com/simonw/sqlite-utils/issues/290 - fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") + fresh_db.table("test").insert({"id": 1, "bar": 2}, pk="id") assert fresh_db.execute_returning_dicts("select * from test") == [ {"id": 1, "bar": 2} ] diff --git a/tests/test_recipes.py b/tests/test_recipes.py index c6222a3..c6a548c 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -8,7 +8,7 @@ from sqlite_utils.utils import sqlite3 @pytest.fixture def dates_db(fresh_db): - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "dt": "5th October 2019 12:04"}, {"id": 2, "dt": "6th October 2019 00:05:06"}, @@ -21,8 +21,8 @@ def dates_db(fresh_db): def test_parsedate(dates_db): - dates_db["example"].convert("dt", recipes.parsedate) - assert list(dates_db["example"].rows) == [ + dates_db.table("example").convert("dt", recipes.parsedate) + assert list(dates_db.table("example").rows) == [ {"id": 1, "dt": "2019-10-05"}, {"id": 2, "dt": "2019-10-06"}, {"id": 3, "dt": ""}, @@ -31,8 +31,8 @@ def test_parsedate(dates_db): def test_parsedatetime(dates_db): - dates_db["example"].convert("dt", recipes.parsedatetime) - assert list(dates_db["example"].rows) == [ + dates_db.table("example").convert("dt", recipes.parsedatetime) + assert list(dates_db.table("example").rows) == [ {"id": 1, "dt": "2019-10-05T12:04:00"}, {"id": 2, "dt": "2019-10-06T00:05:06"}, {"id": 3, "dt": ""}, @@ -50,16 +50,16 @@ def test_parsedatetime(dates_db): ), ) def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected): - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "dt": "03/04/05"}, ], pk="id", ) - fresh_db["example"].convert( + fresh_db.table("example").convert( "dt", lambda value: getattr(recipes, recipe)(value, **kwargs) ) - assert list(fresh_db["example"].rows) == [ + assert list(fresh_db.table("example").rows) == [ {"id": 1, "dt": expected}, ] @@ -68,7 +68,7 @@ def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected): @pytest.mark.parametrize("fn", ("parsedate", "parsedatetime")) def test_dateparse_errors_raises(fresh_db, fn): """Test that invalid dates raise errors when errors=None""" - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "dt": "invalid"}, ], @@ -76,30 +76,32 @@ def test_dateparse_errors_raises(fresh_db, fn): ) # Exception in SQLite callback surfaces as OperationalError with pytest.raises(sqlite3.OperationalError): - fresh_db["example"].convert("dt", lambda value: getattr(recipes, fn)(value)) + fresh_db.table("example").convert( + "dt", lambda value: getattr(recipes, fn)(value) + ) @pytest.mark.parametrize("fn", ("parsedate", "parsedatetime")) @pytest.mark.parametrize("errors", (recipes.SET_NULL, recipes.IGNORE)) def test_dateparse_errors_handled(fresh_db, fn, errors): """Test error handling modes for invalid dates""" - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "dt": "invalid"}, ], pk="id", ) - fresh_db["example"].convert( + fresh_db.table("example").convert( "dt", lambda value: getattr(recipes, fn)(value, errors=errors) ) - rows = list(fresh_db["example"].rows) + rows = list(fresh_db.table("example").rows) expected = [{"id": 1, "dt": None if errors is recipes.SET_NULL else "invalid"}] assert rows == expected @pytest.mark.parametrize("delimiter", [None, ";", "-"]) def test_jsonsplit(fresh_db, delimiter): - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])}, {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])}, @@ -114,8 +116,8 @@ def test_jsonsplit(fresh_db, delimiter): else: fn = recipes.jsonsplit - fresh_db["example"].convert("tags", fn) - assert list(fresh_db["example"].rows) == [ + fresh_db.table("example").convert("tags", fn) + assert list(fresh_db.table("example").rows) == [ {"id": 1, "tags": '["foo", "bar"]'}, {"id": 2, "tags": '["bar", "baz"]'}, ] @@ -130,7 +132,7 @@ def test_jsonsplit(fresh_db, delimiter): ), ) def test_jsonsplit_type(fresh_db, type, expected): - fresh_db["example"].insert_all( + fresh_db.table("example").insert_all( [ {"id": 1, "records": "1,2,3"}, ], @@ -144,5 +146,5 @@ def test_jsonsplit_type(fresh_db, type, expected): else: fn = recipes.jsonsplit - fresh_db["example"].convert("records", fn) - assert json.loads(fresh_db["example"].get(1)["records"]) == expected + fresh_db.table("example").convert("records", fn) + assert json.loads(fresh_db.table("example").get(1)["records"]) == expected diff --git a/tests/test_recreate.py b/tests/test_recreate.py index 09e237e..d8b846e 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -33,8 +33,8 @@ def test_recreate(tmp_path, use_path, create_file_first): filepath = pathlib.Path(filepath) if create_file_first: db = Database(filepath) - db["t1"].insert({"foo": "bar"}) + db.table("t1").insert({"foo": "bar"}) assert ["t1"] == db.table_names() db.close() - Database(filepath, recreate=True)["t2"].insert({"foo": "bar"}) + Database(filepath, recreate=True).table("t2").insert({"foo": "bar"}) assert ["t2"] == Database(filepath).table_names() diff --git a/tests/test_rows.py b/tests/test_rows.py index dccb6ad..476569e 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -3,7 +3,7 @@ import pytest def test_rows(existing_db): assert [{"text": "one"}, {"text": "two"}, {"text": "three"}] == list( - existing_db["foo"].rows + existing_db.table("foo").rows ) @@ -18,7 +18,7 @@ def test_rows(existing_db): ], ) def test_rows_where(where, where_args, expected_ids, fresh_db): - table = fresh_db["dogs"] + table = fresh_db.table("dogs") table.insert_all( [ {"id": 1, "name": "Cleo", "age": 4, "is_good": True}, @@ -41,7 +41,7 @@ def test_rows_where(where, where_args, expected_ids, fresh_db): ], ) def test_rows_where_order_by(where, order_by, expected_ids, fresh_db): - table = fresh_db["dogs"] + table = fresh_db.table("dogs") table.insert_all( [ {"id": 1, "name": "Cleo", "age": 4}, @@ -65,7 +65,7 @@ def test_rows_where_order_by(where, order_by, expected_ids, fresh_db): ], ) def test_rows_where_offset_limit(fresh_db, offset, limit, expected): - table = fresh_db["rows"] + table = fresh_db.table("rows") table.insert_all([{"id": id} for id in range(1, 101)], pk="id") assert table.count == 100 assert expected == [ @@ -74,13 +74,13 @@ def test_rows_where_offset_limit(fresh_db, offset, limit, expected): def test_pks_and_rows_where_offset_without_limit(fresh_db): - table = fresh_db["rows"] + table = fresh_db.table("rows") table.insert_all([{"id": id} for id in range(1, 6)], pk="id") assert [pk for pk, _ in table.pks_and_rows_where(offset=3, order_by="id")] == [4, 5] def test_pks_and_rows_where_rowid(fresh_db): - table = fresh_db["rowid_table"] + table = fresh_db.table("rowid_table") table.insert_all({"number": i + 10} for i in range(3)) pks_and_rows = list(table.pks_and_rows_where()) assert pks_and_rows == [ @@ -91,7 +91,7 @@ def test_pks_and_rows_where_rowid(fresh_db): def test_pks_and_rows_where_simple_pk(fresh_db): - table = fresh_db["simple_pk_table"] + table = fresh_db.table("simple_pk_table") table.insert_all(({"id": i + 10} for i in range(3)), pk="id") pks_and_rows = list(table.pks_and_rows_where()) assert pks_and_rows == [ @@ -102,7 +102,7 @@ def test_pks_and_rows_where_simple_pk(fresh_db): def test_pks_and_rows_where_compound_pk(fresh_db): - table = fresh_db["compound_pk_table"] + table = fresh_db.table("compound_pk_table") table.insert_all( ({"type": "number", "number": i, "plusone": i + 1} for i in range(3)), pk=("type", "number"), @@ -117,8 +117,8 @@ def test_pks_and_rows_where_compound_pk(fresh_db): def test_rows_where_duplicate_select_columns_are_deduped(fresh_db): # https://github.com/simonw/sqlite-utils/issues/624 - fresh_db["t"].insert({"id": 1, "name": "Cleo"}) - rows = list(fresh_db["t"].rows_where(select="id, id, name")) + fresh_db.table("t").insert({"id": 1, "name": "Cleo"}) + rows = list(fresh_db.table("t").rows_where(select="id, id, name")) assert rows == [{"id": 1, "id_2": 1, "name": "Cleo"}] @@ -130,10 +130,10 @@ def test_pks_and_rows_where_view(fresh_db): # an AttributeError from View lacking Table-only properties from sqlite_utils.utils import sqlite3 - fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.create_view("dog_names", "select name from dogs") try: - result = list(fresh_db["dog_names"].pks_and_rows_where()) + result = list(fresh_db.view("dog_names").pks_and_rows_where()) except sqlite3.OperationalError: pass # SQLite 3.36+: no such column: rowid else: @@ -144,6 +144,6 @@ def test_pks_and_rows_where_view(fresh_db): def test_pks_and_rows_where_compound_pk_declaration_order(fresh_db): # Compound pks are returned in PRIMARY KEY declaration order fresh_db.execute("create table t (b text, a text, primary key (a, b))") - fresh_db["t"].insert({"a": "A", "b": "B"}) - pks_and_rows = list(fresh_db["t"].pks_and_rows_where()) + fresh_db.table("t").insert({"a": "A", "b": "B"}) + pks_and_rows = list(fresh_db.table("t").pks_and_rows_where()) assert pks_and_rows == [(("A", "B"), {"b": "B", "a": "A"})] diff --git a/tests/test_sniff.py b/tests/test_sniff.py index 7149978..029a7fc 100644 --- a/tests/test_sniff.py +++ b/tests/test_sniff.py @@ -19,7 +19,7 @@ def test_sniff(tmpdir, filepath): ) assert result.exit_code == 0, result.stdout db = Database(db_path) - assert list(db["creatures"].rows) == [ + assert list(db.table("creatures").rows) == [ {"id": "1", "species": "dog", "name": "Cleo", "age": "5"}, {"id": "2", "species": "dog", "name": "Pancakes", "age": "4"}, {"id": "3", "species": "cat", "name": "Mozie", "age": "8"}, diff --git a/tests/test_transform.py b/tests/test_transform.py index 980ee9d..28fa4d7 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -128,7 +128,7 @@ def test_transform_sql_table_with_primary_key( def tracer(sql, params): return captured.append((sql, params)) - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") @@ -209,7 +209,7 @@ def test_transform_sql_table_with_no_primary_key( def tracer(sql, params): return captured.append((sql, params)) - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") if use_pragma_foreign_keys: fresh_db.conn.execute("PRAGMA foreign_keys=ON") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) @@ -229,7 +229,7 @@ def test_transform_sql_table_with_no_primary_key( def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}) assert ( dogs.schema @@ -244,7 +244,7 @@ def test_transform_sql_with_no_primary_key_to_primary_key_of_id(fresh_db): def test_transform_rename_pk(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") dogs.transform(rename={"id": "pk"}) assert ( @@ -265,7 +265,7 @@ def test_transform_preserves_keyword_literal_defaults(fresh_db): " note TEXT DEFAULT NULL" ")" ) - table = fresh_db["t"] + table = fresh_db.table("t") table.insert({"id": 1}) before = fresh_db.execute("SELECT is_active, flag, note FROM t").fetchone() assert before == (1, 0, None) @@ -288,7 +288,7 @@ def test_transform_preserves_keyword_literal_defaults(fresh_db): def test_transform_not_null(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") dogs.transform(not_null={"name"}) assert ( @@ -298,7 +298,7 @@ def test_transform_not_null(fresh_db): def test_transform_remove_a_not_null(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, not_null={"age"}, pk="id") dogs.transform(not_null={"name": True, "age": False}) assert ( @@ -309,7 +309,7 @@ def test_transform_remove_a_not_null(fresh_db): @pytest.mark.parametrize("not_null", [{"age"}, {"age": True}]) def test_transform_add_not_null_with_rename(fresh_db, not_null): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") dogs.transform(not_null=not_null, rename={"age": "dog_age"}) assert ( @@ -319,7 +319,7 @@ def test_transform_add_not_null_with_rename(fresh_db, not_null): def test_transform_defaults(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id") dogs.transform(defaults={"age": 1}) assert ( @@ -329,7 +329,7 @@ def test_transform_defaults(fresh_db): def test_transform_defaults_and_rename_column(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id") dogs.transform(rename={"age": "dog_age"}, defaults={"age": 1}) assert ( @@ -339,7 +339,7 @@ def test_transform_defaults_and_rename_column(fresh_db): def test_remove_defaults(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5}, defaults={"age": 1}, pk="id") dogs.transform(defaults={"age": None}) assert ( @@ -350,8 +350,8 @@ def test_remove_defaults(fresh_db): @pytest.fixture def authors_db(fresh_db): - books = fresh_db["books"] - authors = fresh_db["authors"] + books = fresh_db.table("books") + authors = fresh_db.table("authors") authors.insert({"id": 5, "name": "Jane McGonical"}, pk="id") books.insert( {"id": 2, "title": "Reality is Broken", "author_id": 5}, @@ -362,13 +362,13 @@ def authors_db(fresh_db): def test_transform_foreign_keys_persist(authors_db): - assert authors_db["books"].foreign_keys == [ + assert authors_db.table("books").foreign_keys == [ ForeignKey( table="books", column="author_id", other_table="authors", other_column="id" ) ] - authors_db["books"].transform(rename={"title": "book_title"}) - assert authors_db["books"].foreign_keys == [ + authors_db.table("books").transform(rename={"title": "book_title"}) + assert authors_db.table("books").foreign_keys == [ ForeignKey( table="books", column="author_id", other_table="authors", other_column="id" ) @@ -381,8 +381,8 @@ def test_transform_foreign_keys_survive_renamed_column( ): if use_pragma_foreign_keys: authors_db.conn.execute("PRAGMA foreign_keys=ON") - authors_db["books"].transform(rename={"author_id": "author_id_2"}) - assert authors_db["books"].foreign_keys == [ + authors_db.table("books").transform(rename={"author_id": "author_id_2"}) + assert authors_db.table("books").foreign_keys == [ ForeignKey( table="books", column="author_id_2", @@ -393,9 +393,9 @@ def test_transform_foreign_keys_survive_renamed_column( def _add_country_city_continent(db): - db["country"].insert({"id": 1, "name": "France"}, pk="id") - db["continent"].insert({"id": 2, "name": "Europe"}, pk="id") - db["city"].insert({"id": 24, "name": "Paris"}, pk="id") + db.table("country").insert({"id": 1, "name": "France"}, pk="id") + db.table("continent").insert({"id": 2, "name": "Europe"}, pk="id") + db.table("city").insert({"id": 24, "name": "Paris"}, pk="id") _CAVEAU = { @@ -413,11 +413,11 @@ def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys): fresh_db.conn.execute("PRAGMA foreign_keys=ON") # Create table with three foreign keys so we can drop two of them _add_country_city_continent(fresh_db) - fresh_db["places"].insert( + fresh_db.table("places").insert( _CAVEAU, foreign_keys=("country", "continent", "city"), ) - assert fresh_db["places"].foreign_keys == [ + assert fresh_db.table("places").foreign_keys == [ ForeignKey( table="places", column="city", other_table="city", other_column="id" ), @@ -432,9 +432,9 @@ def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys): ), ] # Drop two of those foreign keys - fresh_db["places"].transform(drop_foreign_keys=("country", "continent")) + fresh_db.table("places").transform(drop_foreign_keys=("country", "continent")) # Should be only one foreign key now - assert fresh_db["places"].foreign_keys == [ + assert fresh_db.table("places").foreign_keys == [ ForeignKey(table="places", column="city", other_table="city", other_column="id") ] if use_pragma_foreign_keys: @@ -443,17 +443,17 @@ def test_transform_drop_foreign_keys(fresh_db, use_pragma_foreign_keys): def test_transform_verify_foreign_keys(fresh_db): fresh_db.conn.execute("PRAGMA foreign_keys=ON") - fresh_db["authors"].insert({"id": 3, "name": "Tina"}, pk="id") - fresh_db["books"].insert( + fresh_db.table("authors").insert({"id": 3, "name": "Tina"}, pk="id") + fresh_db.table("books").insert( {"id": 1, "title": "Book", "author_id": 3}, pk="id", foreign_keys={"author_id"} ) # Renaming the id column on authors should break everything with pytest.raises(OperationalError) as e: - fresh_db["authors"].transform(rename={"id": "id2"}) + fresh_db.table("authors").transform(rename={"id": "id2"}) assert e.value.args[0] == 'foreign key mismatch - "books" referencing "authors"' # This should have rolled us back assert ( - fresh_db["authors"].schema + fresh_db.table("authors").schema == 'CREATE TABLE "authors" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n)' ) assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] @@ -476,20 +476,22 @@ def test_transform_on_delete_cascade_does_not_delete_records( author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE ); """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db.table("books").insert( + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ) # Transform the table on the other end of the cascading foreign key - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["authors"].rows) == [ + fresh_db.table("authors").transform(rename={"name": "author_name"}) + assert list(fresh_db.table("authors").rows) == [ {"id": 1, "author_name": "Ursula K. Le Guin"} ] - assert list(fresh_db["books"].rows) == [ + assert list(fresh_db.table("books").rows) == [ {"id": 1, "title": "The Dispossessed", "author_id": 1} ] # Transforming the table with the cascading foreign key should not # delete its records either - fresh_db["books"].transform(rename={"title": "book_title"}) - assert list(fresh_db["books"].rows) == [ + fresh_db.table("books").transform(rename={"title": "book_title"}) + assert list(fresh_db.table("books").rows) == [ {"id": 1, "book_title": "The Dispossessed", "author_id": 1} ] if use_pragma_foreign_keys: @@ -511,17 +513,19 @@ def test_transform_in_transaction_refuses_destructive_on_delete(fresh_db, on_del author_id INTEGER REFERENCES authors(id) ON DELETE {on_delete} ); """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) - previous_schema = fresh_db["authors"].schema + fresh_db.table("authors").insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db.table("books").insert( + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ) + previous_schema = fresh_db.table("authors").schema with fresh_db.atomic(), pytest.raises(TransactionError) as excinfo: - fresh_db["authors"].transform(rename={"name": "author_name"}) + fresh_db.table("authors").transform(rename={"name": "author_name"}) message = str(excinfo.value) assert "books" in message assert f"ON DELETE {on_delete.upper()}" in message # Nothing should have changed - assert fresh_db["authors"].schema == previous_schema - assert list(fresh_db["books"].rows) == [ + assert fresh_db.table("authors").schema == previous_schema + assert list(fresh_db.table("books").rows) == [ {"id": 1, "title": "The Dispossessed", "author_id": 1} ] assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] @@ -538,16 +542,16 @@ def test_transform_in_transaction_refuses_self_referential_cascade(fresh_db): parent_id INTEGER REFERENCES categories(id) ON DELETE CASCADE ); """) - fresh_db["categories"].insert_all( + fresh_db.table("categories").insert_all( [ {"id": 1, "name": "Fiction", "parent_id": None}, {"id": 2, "name": "Science Fiction", "parent_id": 1}, ] ) with fresh_db.atomic(), pytest.raises(TransactionError) as excinfo: - fresh_db["categories"].transform(rename={"name": "title"}) + fresh_db.table("categories").transform(rename={"name": "title"}) assert "categories" in str(excinfo.value) - assert fresh_db["categories"].count == 2 + assert fresh_db.table("categories").count == 2 def test_transform_in_transaction_allowed_with_no_action_foreign_key(fresh_db): @@ -562,14 +566,16 @@ def test_transform_in_transaction_allowed_with_no_action_foreign_key(fresh_db): author_id INTEGER REFERENCES authors(id) ); """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db.table("books").insert( + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ) with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["authors"].rows) == [ + fresh_db.table("authors").transform(rename={"name": "author_name"}) + assert list(fresh_db.table("authors").rows) == [ {"id": 1, "author_name": "Ursula K. Le Guin"} ] - assert list(fresh_db["books"].rows) == [ + assert list(fresh_db.table("books").rows) == [ {"id": 1, "title": "The Dispossessed", "author_id": 1} ] assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] @@ -587,11 +593,13 @@ def test_transform_in_transaction_allowed_for_child_table(fresh_db): author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE ); """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db.table("books").insert( + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ) with fresh_db.atomic(): - fresh_db["books"].transform(rename={"title": "book_title"}) - assert list(fresh_db["books"].rows) == [ + fresh_db.table("books").transform(rename={"title": "book_title"}) + assert list(fresh_db.table("books").rows) == [ {"id": 1, "book_title": "The Dispossessed", "author_id": 1} ] @@ -607,24 +615,28 @@ def test_transform_in_transaction_allowed_with_foreign_keys_off(fresh_db): author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE ); """) - fresh_db["authors"].insert({"id": 1, "name": "Ursula K. Le Guin"}) - fresh_db["books"].insert({"id": 1, "title": "The Dispossessed", "author_id": 1}) + fresh_db.table("authors").insert({"id": 1, "name": "Ursula K. Le Guin"}) + fresh_db.table("books").insert( + {"id": 1, "title": "The Dispossessed", "author_id": 1} + ) with fresh_db.atomic(): - fresh_db["authors"].transform(rename={"name": "author_name"}) - assert list(fresh_db["books"].rows) == [ + fresh_db.table("authors").transform(rename={"name": "author_name"}) + assert list(fresh_db.table("books").rows) == [ {"id": 1, "title": "The Dispossessed", "author_id": 1} ] def test_transform_add_foreign_keys_from_scratch(fresh_db): _add_country_city_continent(fresh_db) - fresh_db["places"].insert(_CAVEAU) + fresh_db.table("places").insert(_CAVEAU) # Should have no foreign keys - assert fresh_db["places"].foreign_keys == [] + assert fresh_db.table("places").foreign_keys == [] # Now add them using .transform() - fresh_db["places"].transform(add_foreign_keys=("country", "continent", "city")) + fresh_db.table("places").transform( + add_foreign_keys=("country", "continent", "city") + ) # Should now have all three: - assert fresh_db["places"].foreign_keys == [ + assert fresh_db.table("places").foreign_keys == [ ForeignKey( table="places", column="city", other_table="city", other_column="id" ), @@ -638,7 +650,7 @@ def test_transform_add_foreign_keys_from_scratch(fresh_db): table="places", column="country", other_table="country", other_column="id" ), ] - assert fresh_db["places"].schema == ( + assert fresh_db.table("places").schema == ( 'CREATE TABLE "places" (\n' ' "id" INTEGER,\n' ' "name" TEXT,\n' @@ -662,18 +674,18 @@ def test_transform_add_foreign_keys_from_scratch(fresh_db): ) def test_transform_add_foreign_keys_from_partial(fresh_db, add_foreign_keys): _add_country_city_continent(fresh_db) - fresh_db["places"].insert( + fresh_db.table("places").insert( _CAVEAU, foreign_keys=("city",), ) # Should have one foreign keys - assert fresh_db["places"].foreign_keys == [ + assert fresh_db.table("places").foreign_keys == [ ForeignKey(table="places", column="city", other_table="city", other_column="id") ] # Now add three more using .transform() - fresh_db["places"].transform(add_foreign_keys=add_foreign_keys) + fresh_db.table("places").transform(add_foreign_keys=add_foreign_keys) # Should now have all three: - assert fresh_db["places"].foreign_keys == [ + assert fresh_db.table("places").foreign_keys == [ ForeignKey( table="places", column="city", other_table="city", other_column="id" ), @@ -702,14 +714,14 @@ def test_transform_add_foreign_keys_from_partial(fresh_db, add_foreign_keys): ) def test_transform_replace_foreign_keys(fresh_db, foreign_keys): _add_country_city_continent(fresh_db) - fresh_db["places"].insert( + fresh_db.table("places").insert( _CAVEAU, foreign_keys=("city",), ) - assert len(fresh_db["places"].foreign_keys) == 1 + assert len(fresh_db.table("places").foreign_keys) == 1 # Replace with two different ones - fresh_db["places"].transform(foreign_keys=foreign_keys) - assert fresh_db["places"].schema == ( + fresh_db.table("places").transform(foreign_keys=foreign_keys) + assert fresh_db.table("places").schema == ( 'CREATE TABLE "places" (\n' ' "id" INTEGER,\n' ' "name" TEXT,\n' @@ -729,7 +741,7 @@ def test_transform_preserves_rowids(fresh_db, table_type): pk = ("id", "name") elif table_type == "rowid": pk = None - fresh_db["places"].insert_all( + fresh_db.table("places").insert_all( [ {"id": "1", "name": "Paris", "country": "France"}, {"id": "2", "name": "London", "country": "UK"}, @@ -738,13 +750,13 @@ def test_transform_preserves_rowids(fresh_db, table_type): pk=pk, ) # Now delete and insert a row to mix up the `rowid` sequence - fresh_db["places"].delete_where("id = ?", ["2"]) - fresh_db["places"].insert({"id": "4", "name": "London", "country": "UK"}) + fresh_db.table("places").delete_where("id = ?", ["2"]) + fresh_db.table("places").insert({"id": "4", "name": "London", "country": "UK"}) previous_rows = [ tuple(row) for row in fresh_db.execute("select rowid, id, name from places") ] # Transform it - fresh_db["places"].transform(column_order=("country", "name")) + fresh_db.table("places").transform(column_order=("country", "name")) # Should be the same next_rows = [ tuple(row) for row in fresh_db.execute("select rowid, id, name from places") @@ -774,7 +786,7 @@ def test_transform_strict(fresh_db, initial_strict, transform_strict, expected_s def test_transform_to_strict_with_invalid_data(fresh_db): if not fresh_db.supports_strict: pytest.skip("SQLite version does not support strict tables") - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.create({"id": int}) dogs.insert({"id": "not-an-integer"}) @@ -801,7 +813,7 @@ def test_transform_strict_updates_default(fresh_db): @pytest.mark.parametrize("method_name", ("transform", "transform_sql")) def test_transform_to_strict_not_supported(fresh_db, method_name): - table = fresh_db["items"] + table = fresh_db.table("items") table.create({"id": int}) fresh_db._supports_strict = False @@ -823,7 +835,7 @@ def test_transform_to_strict_not_supported(fresh_db, method_name): def test_transform_indexes(fresh_db, indexes, transform_params): # https://github.com/simonw/sqlite-utils/issues/633 # New table should have same indexes as old table after transformation - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5, "breed": "Labrador"}, pk="id") for index in indexes: @@ -849,13 +861,13 @@ def test_transform_indexes(fresh_db, indexes, transform_params): if "keep_table" in transform_params: assert all( index.origin == "pk" - for index in fresh_db[transform_params["keep_table"]].indexes + for index in fresh_db.table(transform_params["keep_table"]).indexes ) def test_transform_retains_indexes_with_foreign_keys(fresh_db): - dogs = fresh_db["dogs"] - owners = fresh_db["owners"] + dogs = fresh_db.table("dogs") + owners = fresh_db.table("owners") dogs.insert({"id": 1, "name": "Cleo", "owner_id": 1}, pk="id") owners.insert({"id": 1, "name": "Alice"}, pk="id") @@ -890,7 +902,7 @@ def test_transform_retains_indexes_with_foreign_keys(fresh_db): ) def test_transform_with_indexes_errors(fresh_db, transform_params): # Should error with a compound (name, age) index if age is renamed or dropped - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id") dogs.create_index(["name", "age"]) @@ -906,7 +918,7 @@ def test_transform_with_indexes_errors(fresh_db, transform_params): def test_transform_with_unique_constraint_implicit_index(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") # Create a table with a UNIQUE constraint on 'name', which creates an implicit index fresh_db.execute(""" CREATE TABLE dogs ( @@ -933,7 +945,7 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db): def test_transform_preserves_view(fresh_db): # https://github.com/simonw/sqlite-utils/issues/831 - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view dogs_view as select id, name from dogs") view_sql_before = fresh_db.execute( @@ -958,8 +970,8 @@ def test_transform_preserves_view(fresh_db): def test_transform_variants_preserve_view(fresh_db, transform_params): # Covers retyping, changing primary key and foreign key modifications, # with a view whose columns are untouched by the transform - fresh_db["other"].insert({"id": 1}, pk="id") - dogs = fresh_db["dogs"] + fresh_db.table("other").insert({"id": 1}, pk="id") + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "other_id": 1}, pk="id") if "drop_foreign_keys" in transform_params: dogs.transform(add_foreign_keys=[("other_id", "other", "id")]) @@ -972,13 +984,13 @@ def test_transform_variants_preserve_view(fresh_db, transform_params): "select sql from sqlite_master where name = 'dogs_view'" ).fetchone()[0] assert view_sql_before == view_sql_after - assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}] + assert list(fresh_db.view("dogs_view").rows) == [{"id": 1, "name": "Cleo"}] def test_transform_view_referencing_renamed_column(fresh_db): # The view survives but querying it raises "no such column" - inherent # to SQLite views, whose SQL is stored as text - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view dogs_view as select id, name from dogs") dogs.transform(rename={"name": "title"}) @@ -987,7 +999,7 @@ def test_transform_view_referencing_renamed_column(fresh_db): def test_transform_view_on_view(fresh_db): - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view v1 as select id, name from dogs") fresh_db.execute("create view v2 as select name from v1") @@ -999,13 +1011,13 @@ def test_transform_view_on_view(fresh_db): "select sql from sqlite_master where type = 'view' order by name" ).fetchall() assert sqls_before == sqls_after - assert list(fresh_db["v2"].rows) == [{"name": "Cleo"}] + assert list(fresh_db.view("v2").rows) == [{"name": "Cleo"}] def test_transform_keep_table_does_not_repoint_view(fresh_db): # Without legacy_alter_table the ALTER TABLE dogs RENAME TO dogs_backup # step would rewrite the view to select from "dogs_backup" - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view dogs_view as select id, name from dogs") dogs.transform(types={"name": str}, keep_table="dogs_backup") @@ -1015,7 +1027,7 @@ def test_transform_keep_table_does_not_repoint_view(fresh_db): assert "dogs_backup" not in view_sql # View reads from the live table, not the frozen backup dogs.insert({"id": 2, "name": "Pancakes"}) - assert list(fresh_db["dogs_view"].rows) == [ + assert list(fresh_db.view("dogs_view").rows) == [ {"id": 1, "name": "Cleo"}, {"id": 2, "name": "Pancakes"}, ] @@ -1024,7 +1036,7 @@ def test_transform_keep_table_does_not_repoint_view(fresh_db): def test_transform_sql_standalone_statements_work_with_view(fresh_db): # The documented "run these statements yourself" workflow should be # standalone-correct, so the pragmas must come from transform_sql() - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view dogs_view as select id, name from dogs") sqls = dogs.transform_sql(types={"name": str}, tmp_suffix="suffix") @@ -1033,12 +1045,12 @@ def test_transform_sql_standalone_statements_work_with_view(fresh_db): assert sqls[-1] == "PRAGMA legacy_alter_table=OFF;" for sql in sqls: fresh_db.execute(sql) - assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}] + assert list(fresh_db.view("dogs_view").rows) == [{"id": 1, "name": "Cleo"}] def test_transform_with_view_in_open_transaction(fresh_db): fresh_db.conn.execute("PRAGMA foreign_keys=ON") - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.execute("create view dogs_view as select id, name from dogs") with fresh_db.conn: @@ -1054,7 +1066,7 @@ def test_transform_with_view_in_open_transaction(fresh_db): def test_transform_restores_legacy_alter_table_setting(fresh_db): if sqlite3.sqlite_version_info < (3, 25, 0): pytest.skip("legacy_alter_table pragma requires SQLite 3.25 or higher") - dogs = fresh_db["dogs"] + dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo"}, pk="id") # Default is OFF, reset to OFF afterwards dogs.transform(types={"name": str}) @@ -1075,7 +1087,7 @@ def test_transform_preserves_check_constraints(fresh_db): CONSTRAINT nonzero_id CHECK(id != 0) ) """) - scores = fresh_db["scores"] + scores = fresh_db.table("scores") scores.insert({"id": 1, "score": 50}) scores.transform() assert scores.checks == [ @@ -1095,7 +1107,7 @@ def test_transform_preserves_check_ending_in_line_comment(fresh_db): ) ) """) - inventory = fresh_db["inventory"] + inventory = fresh_db.table("inventory") inventory.transform(types={"quantity": float}) assert inventory.checks == [Check("quantity >= 0 -- Quantity cannot be negative")] with pytest.raises(sqlite3.IntegrityError, match="CHECK constraint failed"): @@ -1113,7 +1125,7 @@ def test_transform_preserves_comments_owned_by_columns(fresh_db): age INTEGER -- May be NULL ) """) - people = fresh_db["people"] + people = fresh_db.table("people") people.insert({"id": 1, "name": "Cleo", "age": 5}) people.transform( rename={"name": "display_name"}, @@ -1143,8 +1155,8 @@ def test_transform_drops_comments_owned_by_dropped_column(fresh_db): obsolete TEXT /* Drop this too */ ) """) - fresh_db["t"].transform(drop={"obsolete"}) - schema = fresh_db["t"].schema + fresh_db.table("t").transform(drop={"obsolete"}) + schema = fresh_db.table("t").schema assert "Keep this explanation" in schema assert "Drop this explanation" not in schema assert "Drop this too" not in schema @@ -1159,7 +1171,7 @@ def test_transform_renames_columns_inside_check_constraints(fresh_db): CONSTRAINT within_maximum CHECK(quantity <= maximum) ) """) - inventory = fresh_db["inventory"] + inventory = fresh_db.table("inventory") inventory.insert({"quantity": 2, "maximum": 3}) inventory.transform(rename={"quantity": "amount"}) assert inventory.checks == [ @@ -1182,7 +1194,7 @@ def test_transform_check_rewrite_preserves_functions_and_quotes(fresh_db): CHECK(length("old name") > 0 AND length != '') ) """) - items = fresh_db["items"] + items = fresh_db.table("items") items.insert({"length": "label", "old name": "hello"}) items.transform(rename={"length": "description", "old name": "new name"}) assert items.checks == [Check("length(\"new name\") > 0 AND description != ''")] @@ -1190,9 +1202,9 @@ def test_transform_check_rewrite_preserves_functions_and_quotes(fresh_db): def test_transform_check_rewrite_quotes_keyword_column(fresh_db): fresh_db.execute("CREATE TABLE t(old_name TEXT CHECK(old_name != ''))") - fresh_db["t"].insert({"old_name": "value"}) - fresh_db["t"].transform(rename={"old_name": "select"}) - assert fresh_db["t"].checks == [Check("\"select\" != ''", column="select")] + fresh_db.table("t").insert({"old_name": "value"}) + fresh_db.table("t").transform(rename={"old_name": "select"}) + assert fresh_db.table("t").checks == [Check("\"select\" != ''", column="select")] def test_transform_check_rewrite_does_not_rename_collations_or_cast_types(fresh_db): @@ -1209,9 +1221,9 @@ def test_transform_check_rewrite_does_not_rename_collations_or_cast_types(fresh_ ) ) """) - fresh_db["t"].insert({"nocase": "n", "kind": "k", "other": "o"}) - fresh_db["t"].transform(rename={"nocase": "label", "kind": "category"}) - check = fresh_db["t"].checks[0].check + fresh_db.table("t").insert({"nocase": "n", "kind": "k", "other": "o"}) + fresh_db.table("t").transform(rename={"nocase": "label", "kind": "category"}) + check = fresh_db.table("t").checks[0].check assert "COLLATE nocase" in check assert "AS kind" in check assert "AND label != ''" in check @@ -1226,9 +1238,9 @@ def test_transform_drops_check_owned_by_dropped_column(fresh_db): CHECK(id > 0) ) """) - fresh_db["t"].insert({"id": 1, "obsolete": 2}) - fresh_db["t"].transform(drop={"obsolete"}) - assert fresh_db["t"].checks == [Check("id > 0")] + fresh_db.table("t").insert({"id": 1, "obsolete": 2}) + fresh_db.table("t").transform(drop={"obsolete"}) + assert fresh_db.table("t").checks == [Check("id > 0")] def test_transform_refuses_to_drop_column_used_by_remaining_check(fresh_db): @@ -1239,7 +1251,7 @@ def test_transform_refuses_to_drop_column_used_by_remaining_check(fresh_db): CHECK(minimum <= maximum) ) """) - ranges = fresh_db["ranges"] + ranges = fresh_db.table("ranges") ranges.insert({"minimum": 1, "maximum": 2}) schema_before = ranges.schema with pytest.raises( diff --git a/tests/test_update.py b/tests/test_update.py index e6ae7d8..44cc098 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -7,14 +7,14 @@ from sqlite_utils.db import NotFoundError def test_update_rowid_table(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") rowid = table.insert({"foo": "bar"}).last_pk table.update(rowid, {"foo": "baz"}) assert [{"foo": "baz"}] == list(table.rows) def test_update_pk_table(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") pk = table.insert({"foo": "bar", "id": 5}, pk="id").last_pk assert 5 == pk table.update(pk, {"foo": "baz"}) @@ -22,7 +22,7 @@ def test_update_pk_table(fresh_db): def test_update_compound_pk_table(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") pk = table.insert({"id1": 5, "id2": 3, "v": 1}, pk=("id1", "id2")).last_pk assert (5, 3) == pk table.update(pk, {"v": 2}) @@ -42,14 +42,14 @@ def test_update_compound_pk_table(fresh_db): ), ) def test_update_invalid_pk(fresh_db, pk, update_pk): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"id1": 5, "id2": 3, "v": 1}, pk=pk) with pytest.raises(NotFoundError): table.update(update_pk, {"v": 2}) def test_update_alter(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") rowid = table.insert({"foo": "bar"}).last_pk table.update(rowid, {"new_col": 1.2}, alter=True) assert [{"foo": "bar", "new_col": 1.2}] == list(table.rows) @@ -72,7 +72,7 @@ def test_update_alter(fresh_db): def test_update_alter_with_special_column_characters(fresh_db): # With double-quote escaping, columns with special characters are now valid - table = fresh_db["table"] + table = fresh_db.table("table") rowid = table.insert({"foo": "bar"}).last_pk table.update(rowid, {"new_col[abc]": 1.2}, alter=True) assert list(table.rows) == [{"foo": "bar", "new_col[abc]": 1.2}] @@ -106,8 +106,8 @@ def test_update_with_no_values_sets_last_pk(fresh_db): ), ) def test_update_dictionaries_and_lists_as_json(fresh_db, data_structure): - fresh_db["test"].insert({"id": 1, "data": ""}, pk="id") - fresh_db["test"].update(1, {"data": data_structure}) + fresh_db.table("test").insert({"id": 1, "data": ""}, pk="id") + fresh_db.table("test").update(1, {"data": data_structure}) row = fresh_db.execute("select id, data from test").fetchone() assert row[0] == 1 assert data_structure == json.loads(row[1]) diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 0eaae9b..0f44cc7 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -7,7 +7,7 @@ from sqlite_utils.db import PrimaryKeyRequired @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_upsert(use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) - table = db["table"] + table = db.table("table") table.insert({"id": 1, "name": "Cleo"}, pk="id") table.upsert({"id": 1, "age": 5}, pk="id", alter=True) assert list(table.rows) == [{"id": 1, "name": "Cleo", "age": 5}] @@ -15,7 +15,7 @@ def test_upsert(use_old_upsert): def test_upsert_all(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Nixie"}], pk="id") table.upsert_all([{"id": 1, "age": 5}, {"id": 2, "age": 5}], pk="id", alter=True) assert list(table.rows) == [ @@ -26,7 +26,7 @@ def test_upsert_all(fresh_db): def test_upsert_all_single_column(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert_all([{"name": "Cleo"}], pk="name") assert list(table.rows) == [{"name": "Cleo"}] assert table.pks == ["name"] @@ -34,16 +34,16 @@ def test_upsert_all_single_column(fresh_db): def test_upsert_all_not_null(fresh_db): # https://github.com/simonw/sqlite-utils/issues/538 - fresh_db["comments"].upsert_all( + fresh_db.table("comments").upsert_all( [{"id": 1, "name": "Cleo"}], pk="id", not_null=["name"], ) - assert list(fresh_db["comments"].rows) == [{"id": 1, "name": "Cleo"}] + assert list(fresh_db.table("comments").rows) == [{"id": 1, "name": "Cleo"}] def test_upsert_error_if_no_pk(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") with pytest.raises(PrimaryKeyRequired): table.upsert_all([{"id": 1, "name": "Cleo"}]) with pytest.raises(PrimaryKeyRequired): @@ -53,7 +53,7 @@ def test_upsert_error_if_no_pk(fresh_db): @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_upsert_empty_record_errors(use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) - table = db["table"] + table = db.table("table") table.insert({"id": 1, "name": "Cleo"}, pk="id") with pytest.raises(PrimaryKeyRequired): table.upsert({}, pk="id") @@ -66,7 +66,7 @@ def test_upsert_empty_record_errors(use_old_upsert): @pytest.mark.parametrize("use_old_upsert", (False, True)) def test_upsert_missing_pk_value_errors(use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) - table = db["table"] + table = db.table("table") table.insert({"id": 1, "name": "Cleo"}, pk="id") # Records that omit the pk column entirely with pytest.raises(PrimaryKeyRequired): @@ -78,7 +78,7 @@ def test_upsert_missing_pk_value_errors(use_old_upsert): def test_upsert_missing_compound_pk_value_errors(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.insert({"a": "x", "b": "y", "v": 1}, pk=("a", "b")) # Missing one component of the detected compound primary key with pytest.raises(PrimaryKeyRequired): @@ -105,7 +105,7 @@ def test_upsert_uses_compound_pk_from_existing_table(use_old_upsert): primary key (Source, Object, Category) ) """) - table = db["summary"] + table = db.table("summary") table.upsert( { "Source": "Client A", @@ -134,7 +134,7 @@ def test_upsert_uses_compound_pk_from_existing_table(use_old_upsert): def test_upsert_with_hash_id(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert({"foo": "bar"}, hash_id="pk") assert [{"pk": "a5e744d0164540d33b1d7ea616c28f2fa97e754a", "foo": "bar"}] == list( table.rows @@ -144,7 +144,7 @@ def test_upsert_with_hash_id(fresh_db): @pytest.mark.parametrize("hash_id", (None, "custom_id")) def test_upsert_with_hash_id_columns(fresh_db, hash_id): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert({"a": 1, "b": 2, "c": 3}, hash_id=hash_id, hash_id_columns=("a", "b")) assert list(table.rows) == [ { @@ -167,7 +167,7 @@ def test_upsert_with_hash_id_columns(fresh_db, hash_id): def test_upsert_compound_primary_key(fresh_db): - table = fresh_db["table"] + table = fresh_db.table("table") table.upsert_all( [ {"species": "dog", "id": 1, "name": "Cleo", "age": 4}, diff --git a/tests/test_wal.py b/tests/test_wal.py index 35318f8..0e8f332 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -18,7 +18,7 @@ def test_enable_disable_wal(db_path_tmpdir): assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()] db.enable_wal() assert "wal" == db.journal_mode - db["test"].insert({"foo": "bar"}) + db.table("test").insert({"foo": "bar"}) assert "test.db-wal" in [f.basename for f in tmpdir.listdir()] db.disable_wal() assert "delete" == db.journal_mode @@ -27,25 +27,25 @@ def test_enable_disable_wal(db_path_tmpdir): def test_enable_wal_inside_transaction_raises(db_path_tmpdir): db, _path, _tmpdir = db_path_tmpdir - db["test"].insert({"id": 1}, pk="id") + db.table("test").insert({"id": 1}, pk="id") with pytest.raises(TransactionError), db.atomic(): - db["test"].insert({"id": 2}, pk="id") + db.table("test").insert({"id": 2}, pk="id") db.enable_wal() # The atomic() block must have rolled back cleanly and the # journal mode must be unchanged assert db.journal_mode == "delete" - assert [r["id"] for r in db["test"].rows] == [1] + assert [r["id"] for r in db.table("test").rows] == [1] def test_disable_wal_inside_transaction_raises(db_path_tmpdir): db, _path, _tmpdir = db_path_tmpdir db.enable_wal() - db["test"].insert({"id": 1}, pk="id") + db.table("test").insert({"id": 1}, pk="id") with pytest.raises(TransactionError), db.atomic(): - db["test"].insert({"id": 2}, pk="id") + db.table("test").insert({"id": 2}, pk="id") db.disable_wal() assert db.journal_mode == "wal" - assert [r["id"] for r in db["test"].rows] == [1] + assert [r["id"] for r in db.table("test").rows] == [1] def test_ensure_autocommit_on(db_path_tmpdir): @@ -65,9 +65,9 @@ def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): db, _path, _tmpdir = db_path_tmpdir db.enable_wal() with db.atomic(): - db["test"].insert({"id": 1}, pk="id") + db.table("test").insert({"id": 1}, pk="id") db.enable_wal() - assert [r["id"] for r in db["test"].rows] == [1] + assert [r["id"] for r in db.table("test").rows] == [1] def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): @@ -75,7 +75,7 @@ def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): # effect, silently breaking the caller's rollback guarantee - so # entering autocommit mode with a transaction open is an error db, _path, _tmpdir = db_path_tmpdir - db["test"].insert({"id": 1}, pk="id") + db.table("test").insert({"id": 1}, pk="id") db.begin() db.execute("insert into test (id) values (2)") with pytest.raises(TransactionError), db.ensure_autocommit_on(): @@ -83,4 +83,4 @@ def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): # The transaction is still open and can still be rolled back assert db.conn.in_transaction db.rollback() - assert [r["id"] for r in db["test"].rows] == [1] + assert [r["id"] for r in db.table("test").rows] == [1] From ebb04a97de765ce5f0b6d1149c992062fa25629a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 13:41:33 -0700 Subject: [PATCH 096/110] Fixes for Pyright, closes #833 --- .github/workflows/test.yml | 3 + Justfile | 3 +- pyproject.toml | 1 + sqlite_utils/cli.py | 17 ++-- sqlite_utils/db.py | 135 ++++++++++++++++++-------------- sqlite_utils/utils.py | 11 ++- tests/test_cli.py | 2 +- tests/test_cli_bulk.py | 1 + tests/test_cli_insert.py | 1 + tests/test_constructor.py | 3 +- tests/test_create.py | 1 + tests/test_foreign_keys.py | 2 +- tests/test_fts.py | 2 +- tests/test_list_mode.py | 4 +- tests/test_register_function.py | 6 +- tests/test_upsert.py | 2 +- 16 files changed, 109 insertions(+), 85 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 923de2e..6c720a1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,6 +43,9 @@ jobs: run: pytest --sqlite-autocommit - name: run mypy run: mypy sqlite_utils tests + - name: run pyright regression checks + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' + run: pyright sqlite_utils tests - name: run flake8 run: flake8 - name: run ty diff --git a/Justfile b/Justfile index be41523..e93075f 100644 --- a/Justfile +++ b/Justfile @@ -8,11 +8,12 @@ @run *options: uv run -- {{options}} -# Run linters: black, flake8, mypy, ty, cog +# Run linters: black, flake8, mypy, pyright, ty, cog @lint: just run black . --check uv run flake8 uv run mypy sqlite_utils tests + uv run pyright sqlite_utils tests uv run ty check sqlite_utils uv run cog --check README.md docs/*.rst uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt diff --git a/pyproject.toml b/pyproject.toml index 6bc0a64..9b4d6f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dev = [ # flake8 "flake8", "flake8-pyproject", + "pyright>=1.1.411", "ty>=0.0.37", # For stable cog: "tabulate>=0.10.0", diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index d9c7728..c90c137 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1093,7 +1093,7 @@ def insert_upsert_implementation( column_type_overrides = {column: ctype.upper() for column, ctype in (types or [])} def _insert_docs(docs, tracker=None): - extra_kwargs = { + extra_kwargs: dict[str, Any] = { "ignore": ignore, "replace": replace, "truncate": truncate, @@ -3275,15 +3275,10 @@ def convert( raise click.ClickException(str(e)) if dry_run: # Pull first 20 values for first column and preview them - if multi: - - def preview(v): + def preview(v): + if multi: return json.dumps(fn(v), default=repr, ensure_ascii=False) if v else v - - else: - - def preview(v): - return fn(v) if v else v + return fn(v) if v else v db.conn.create_function("preview_transform", 1, preview) sql = """ @@ -3788,12 +3783,12 @@ def _rows_from_code(code): code = pathlib.Path(code).read_text() except FileNotFoundError: raise click.ClickException(f"File not found: {code}") - namespace = {} + namespace: dict[str, Any] = {} try: exec(code, namespace) # noqa: S102 except SyntaxError as ex: raise click.ClickException(f"Error in --code: {ex}") - rows = namespace.get("rows") + rows: Any = namespace.get("rows") if callable(rows): rows = rows() if isinstance(rows, dict): diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 82a95c1..66dc700 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -18,6 +18,7 @@ from dataclasses import dataclass, field from types import TracebackType from typing import ( Any, + TypeVar, Union, cast, ) @@ -256,8 +257,8 @@ class ForeignKey: column: str | None = field(compare=False) other_table: str other_column: str | None = field(compare=False) - columns: tuple[str, ...] = () - other_columns: tuple[str, ...] = () + columns: tuple[str, ...] | list[str] = () + other_columns: tuple[str, ...] | list[str] = () is_compound: bool = False on_delete: str = "NO ACTION" on_update: str = "NO ACTION" @@ -320,6 +321,8 @@ ForeignKeyIndicator = ( ForeignKeysType = Iterable[ForeignKeyIndicator] | list[ForeignKeyIndicator] +PrimaryKey = str | tuple[str, ...] | list[str] + class Default: pass @@ -327,6 +330,8 @@ class Default: DEFAULT = Default() +T = TypeVar("T") + Tracer = Callable[[str, Sequence[Any] | dict[str, Any] | None], None] @@ -1853,8 +1858,8 @@ class Database: fk_object = self._resolve_foreign_key_casing( fk_object, table_obj.columns_dict ) - columns = fk_object.columns - other_columns = fk_object.other_columns + columns = tuple(fk_object.columns) + other_columns = tuple(fk_object.other_columns) for column in columns: if column not in table_obj.columns_dict: raise AlterError(f"No such column: {column} in {table}") @@ -1914,9 +1919,10 @@ class Database: existing_indexes = {tuple(i.columns) for i in table.indexes} for fk in table.foreign_keys: # A compound foreign key gets a single composite index - if fk.columns not in existing_indexes: + fk_columns = tuple(fk.columns) + if fk_columns not in existing_indexes: table.create_index(fk.columns, find_unique_name=True) - existing_indexes.add(fk.columns) + existing_indexes.add(fk_columns) def vacuum(self) -> None: "Run a SQLite ``VACUUM`` against the database." @@ -2453,7 +2459,7 @@ class Table(Queryable): replace: bool = False, ignore: bool = False, transform: bool = False, - strict: bool | Default = DEFAULT, + strict: bool | Default | None = DEFAULT, ) -> "Table": """ Create a table with the specified columns. @@ -2524,7 +2530,7 @@ class Table(Queryable): replace=replace, ignore=ignore, transform=transform, - strict=strict, # type: ignore[arg-type] + strict=cast(bool, strict), ) return self @@ -2860,7 +2866,7 @@ class Table(Queryable): for name, type_ in current_column_pairs: type_ = types.get(name) or type_ if name in drop: - del [copy_from_to[name]] + del copy_from_to[name] continue new_name = rename.get(name) or name new_column_pairs.append((new_name, type_)) @@ -3343,7 +3349,10 @@ class Table(Queryable): :param on_update: ``ON UPDATE`` action for the foreign key. """ columns = (column,) if isinstance(column, str) else tuple(column) + if not columns: + raise ValueError("column must contain at least one column name") columns = tuple(resolve_casing(c, self.columns_dict) for c in columns) + assert columns # Ensure columns exist for col in columns: if col not in self.columns_dict: @@ -3354,7 +3363,7 @@ class Table(Queryable): raise ValueError( "other_table must be specified for a compound foreign key" ) - other_table = self.guess_foreign_table(columns[0]) + other_table = self.guess_foreign_table(next(iter(columns))) # If other_column is not specified, detect the primary key on other_table if other_column is None: if len(columns) > 1: @@ -3801,8 +3810,10 @@ class Table(Queryable): for row in cursor: yield dict(zip(columns, row)) - def value_or_default(self, key: str, value: Any) -> Any: - return self._defaults[key] if value is DEFAULT else value + def value_or_default(self, key: str, value: T | Default) -> T: + if value is DEFAULT: + return cast(T, self._defaults[key]) + return cast(T, value) def delete(self, pk_values: list | tuple | str | float) -> "Table": """ @@ -3987,7 +3998,7 @@ class Table(Queryable): def _convert_multi( self, column, fn, drop, show_progress, where=None, where_args=None - ): + ) -> "Table": # First we execute the function pk_to_values = {} new_column_types: dict[str, set[type]] = {} @@ -4033,6 +4044,7 @@ class Table(Queryable): bar.update(1) if drop: self.transform(drop=(column,)) + return self def build_insert_queries_and_params( self, @@ -4336,8 +4348,8 @@ class Table(Queryable): def insert( self, record: dict[str, Any], - pk=DEFAULT, - foreign_keys=DEFAULT, + pk: PrimaryKey | Default | None = DEFAULT, + foreign_keys: ForeignKeysType | Default | None = DEFAULT, column_order: list[str] | Default | None = DEFAULT, not_null: Iterable[str] | Default | None = DEFAULT, defaults: dict[str, Any] | Default | None = DEFAULT, @@ -4405,24 +4417,24 @@ class Table(Queryable): def insert_all( self, records: Iterable[dict[str, Any]] | Iterable[Sequence[Any]], - pk=DEFAULT, - foreign_keys=DEFAULT, - column_order=DEFAULT, - not_null=DEFAULT, - defaults=DEFAULT, - batch_size=DEFAULT, - hash_id=DEFAULT, - hash_id_columns=DEFAULT, - alter=DEFAULT, - ignore=DEFAULT, - replace=DEFAULT, - truncate=False, - extracts=DEFAULT, - conversions=DEFAULT, - columns=DEFAULT, - upsert=False, - analyze=False, - strict=DEFAULT, + pk: PrimaryKey | Default | None = DEFAULT, + foreign_keys: ForeignKeysType | Default | None = DEFAULT, + column_order: list[str] | Default | None = DEFAULT, + not_null: Iterable[str] | Default | None = DEFAULT, + defaults: dict[str, Any] | Default | None = DEFAULT, + batch_size: int | Default = DEFAULT, + hash_id: str | Default | None = DEFAULT, + hash_id_columns: Iterable[str] | Default | None = DEFAULT, + alter: bool | Default | None = DEFAULT, + ignore: bool | Default | None = DEFAULT, + replace: bool | Default | None = DEFAULT, + truncate: bool = False, + extracts: dict[str, str] | list[str] | Default | None = DEFAULT, + conversions: dict[str, str] | Default | None = DEFAULT, + columns: dict[str, Any] | Default | None = DEFAULT, + upsert: bool = False, + analyze: bool = False, + strict: bool | Default | None = DEFAULT, ) -> "Table": """ Like ``.insert()`` but takes a list of records and ensures that the table @@ -4715,6 +4727,7 @@ class Table(Queryable): elif isinstance(pk, str): self.last_pk = row[resolve_casing(pk, row)] else: + assert pk is not None self.last_pk = tuple( row[resolve_casing(p, row)] for p in pk ) @@ -4732,6 +4745,7 @@ class Table(Queryable): pk_index = column_names.index(resolve_casing(pk, column_names)) self.last_pk = first_record_list[pk_index] else: + assert pk is not None self.last_pk = tuple( first_record_list[ column_names.index(resolve_casing(p, column_names)) @@ -4743,6 +4757,7 @@ class Table(Queryable): if hash_id: self.last_pk = hash_record(first_record_dict, hash_id_columns) else: + assert pk is not None self.last_pk = ( first_record_dict[resolve_casing(pk, first_record_dict)] if isinstance(pk, str) @@ -4759,19 +4774,19 @@ class Table(Queryable): def upsert( self, - record, - pk=DEFAULT, - foreign_keys=DEFAULT, - column_order=DEFAULT, - not_null=DEFAULT, - defaults=DEFAULT, - hash_id=DEFAULT, - hash_id_columns=DEFAULT, - alter=DEFAULT, - extracts=DEFAULT, - conversions=DEFAULT, - columns=DEFAULT, - strict=DEFAULT, + record: dict[str, Any], + pk: PrimaryKey | Default | None = DEFAULT, + foreign_keys: ForeignKeysType | Default | None = DEFAULT, + column_order: list[str] | Default | None = DEFAULT, + not_null: Iterable[str] | Default | None = DEFAULT, + defaults: dict[str, Any] | Default | None = DEFAULT, + hash_id: str | Default | None = DEFAULT, + hash_id_columns: Iterable[str] | Default | None = DEFAULT, + alter: bool | Default | None = DEFAULT, + extracts: dict[str, str] | list[str] | Default | None = DEFAULT, + conversions: dict[str, str] | Default | None = DEFAULT, + columns: dict[str, Any] | Default | None = DEFAULT, + strict: bool | Default | None = DEFAULT, ) -> "Table": """ Like ``.insert()`` but performs an ``UPSERT``, where records are inserted if they do @@ -4798,20 +4813,20 @@ class Table(Queryable): def upsert_all( self, records: Iterable[dict[str, Any]] | Iterable[Sequence[Any]], - pk=DEFAULT, - foreign_keys=DEFAULT, - column_order=DEFAULT, - not_null=DEFAULT, - defaults=DEFAULT, - batch_size=DEFAULT, - hash_id=DEFAULT, - hash_id_columns=DEFAULT, - alter=DEFAULT, - extracts=DEFAULT, - conversions=DEFAULT, - columns=DEFAULT, - analyze=False, - strict=DEFAULT, + pk: PrimaryKey | Default | None = DEFAULT, + foreign_keys: ForeignKeysType | Default | None = DEFAULT, + column_order: list[str] | Default | None = DEFAULT, + not_null: Iterable[str] | Default | None = DEFAULT, + defaults: dict[str, Any] | Default | None = DEFAULT, + batch_size: int | Default = DEFAULT, + hash_id: str | Default | None = DEFAULT, + hash_id_columns: Iterable[str] | Default | None = DEFAULT, + alter: bool | Default | None = DEFAULT, + extracts: dict[str, str] | list[str] | Default | None = DEFAULT, + conversions: dict[str, str] | Default | None = DEFAULT, + columns: dict[str, Any] | Default | None = DEFAULT, + analyze: bool = False, + strict: bool | Default | None = DEFAULT, ) -> "Table": """ Like ``.upsert()`` but can be applied to a list of records. diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index ed5a558..3145a6f 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -14,6 +14,7 @@ from typing import ( TYPE_CHECKING, Any, BinaryIO, + Generic, TypeVar, Union, cast, @@ -344,7 +345,11 @@ def rows_from_file( reader = csv.DictReader(decoded_fp, dialect=dialect) else: reader = csv.DictReader(decoded_fp) - rows = _extra_key_strategy(reader, ignore_extras, extras_key) + rows = _extra_key_strategy( + cast(Iterable[dict[str | None, object]], reader), + ignore_extras, + extras_key, + ) return _CloseableIterator(iter(rows), decoded_fp), Format.CSV elif format == Format.TSV: rows, _ = rows_from_file( @@ -487,12 +492,12 @@ class ValueTracker: del self.couldbe[key] -class NullProgressBar: +class NullProgressBar(Generic[T]): def __init__(self, *args: Iterable[T]) -> None: self.args = args def __iter__(self) -> Iterator[T]: - yield from self.args[0] # type: ignore + yield from self.args[0] def update(self, value: int) -> None: pass diff --git a/tests/test_cli.py b/tests/test_cli.py index 012900c..f60c7d5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1220,7 +1220,7 @@ def test_rows(db_path, args, expected): {"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}, ], - column_order=("id", "name", "age"), + column_order=["id", "name", "age"], ) result = CliRunner().invoke(cli.cli, ["rows", db_path, "dogs"] + args) assert expected == result.output.strip() diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py index 24889b3..c5c9dcd 100644 --- a/tests/test_cli_bulk.py +++ b/tests/test_cli_bulk.py @@ -91,6 +91,7 @@ def test_cli_bulk_batch_size(test_db_and_path): stdin=subprocess.PIPE, stdout=sys.stdout, ) + assert proc.stdin is not None # Writing one record should not commit proc.stdin.write(b'{"id": 3, "name": "Three"}\n\n') proc.stdin.flush() diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 01e7e94..0117862 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -577,6 +577,7 @@ def test_insert_streaming_batch_size_1(db_path): stdin=subprocess.PIPE, stdout=sys.stdout, ) + assert proc.stdin is not None proc.stdin.write(b'{"name": "Azi"}\n') proc.stdin.flush() diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 2d0a298..412b66c 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -83,7 +83,8 @@ def test_autocommit_connections_are_rejected(tmpdir, autocommit): ) def test_legacy_transaction_control_connection_is_accepted(tmpdir): conn = sqlite3.connect( - str(tmpdir / "test.db"), autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL + str(tmpdir / "test.db"), + autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL, # type: ignore[arg-type] ) db = Database(conn) db.table("t").insert({"id": 1}, pk="id") diff --git a/tests/test_create.py b/tests/test_create.py index 0af68a6..83ce403 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1182,6 +1182,7 @@ def test_works_with_pathlib_path(tmpdir): @pytest.mark.skipif(pd is None, reason="pandas and numpy are not installed") def test_create_table_numpy(fresh_db): + assert pd is not None df = pd.DataFrame({"col 1": range(3), "col 2": range(3)}) fresh_db.table("pandas").insert_all(df.to_dict(orient="records")) assert [ diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 271125c..f916c65 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -608,7 +608,7 @@ def test_foreign_key_is_immutable(): fk = ForeignKey("c", "pid", "p", "id") with pytest.raises(dataclasses.FrozenInstanceError): - fk.table = "other" + setattr(fk, "table", "other") def test_foreign_key_equality_and_hash_include_actions(): diff --git a/tests/test_fts.py b/tests/test_fts.py index 312b032..04b5bc3 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -510,7 +510,7 @@ def test_view_has_no_enable_fts(): db.create_view("hello", "select 1 + 1") # Views deliberately do not have an enable_fts() method with pytest.raises(AttributeError): - db.view("hello").enable_fts() # type: ignore[union-attr] + db.view("hello").enable_fts() # type: ignore[attr-defined] @pytest.mark.parametrize( diff --git a/tests/test_list_mode.py b/tests/test_list_mode.py index b9ab812..75f5a76 100644 --- a/tests/test_list_mode.py +++ b/tests/test_list_mode.py @@ -99,7 +99,7 @@ def test_list_mode_error_non_string_columns(): yield ["a", "b", "c"] with pytest.raises(ValueError, match="must be a list of column name strings"): - db.table("bad").insert_all(bad_data()) + db.table("bad").insert_all(bad_data()) # type: ignore[arg-type] def test_list_mode_error_mixed_types(): @@ -111,7 +111,7 @@ def test_list_mode_error_mixed_types(): yield {"id": 1, "name": "Alice"} # Should be a list, not dict with pytest.raises(ValueError, match="must also be lists"): - db.table("bad").insert_all(bad_data()) + db.table("bad").insert_all(bad_data()) # type: ignore[arg-type] def test_list_mode_empty_after_headers(): diff --git a/tests/test_register_function.py b/tests/test_register_function.py index 618bf1e..63f0570 100644 --- a/tests/test_register_function.py +++ b/tests/test_register_function.py @@ -86,21 +86,21 @@ def test_register_function_deterministic_tries_again_if_exception_raised(fresh_d def test_register_function_replace(fresh_db): @fresh_db.register_function() - def one(): + def one(): # pyright: ignore[reportRedeclaration] return "one" assert "one" == fresh_db.execute("select one()").fetchone()[0] # This will silently fail to replaec the function @fresh_db.register_function() - def one(): # noqa + def one(): # pyright: ignore[reportRedeclaration] return "two" assert "one" == fresh_db.execute("select one()").fetchone()[0] # This will replace it @fresh_db.register_function(replace=True) - def one(): # noqa + def one(): # pyright: ignore[reportRedeclaration] return "two" assert "two" == fresh_db.execute("select one()").fetchone()[0] diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 0f44cc7..8274557 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -8,7 +8,7 @@ from sqlite_utils.db import PrimaryKeyRequired def test_upsert(use_old_upsert): db = Database(memory=True, use_old_upsert=use_old_upsert) table = db.table("table") - table.insert({"id": 1, "name": "Cleo"}, pk="id") + table.insert_all([{"id": 1, "name": "Cleo"}], pk="id", replace=True) table.upsert({"id": 1, "age": 5}, pk="id", alter=True) assert list(table.rows) == [{"id": 1, "name": "Cleo", "age": 5}] assert table.last_pk == 1 From 25c632fbbc286b6e5d622a975a2231c12ce6837c Mon Sep 17 00:00:00 2001 From: Rami Abdelrazzaq Date: Wed, 12 Aug 2026 16:05:09 -0500 Subject: [PATCH 097/110] Handle empty input in rows_from_file Closes #808 --- sqlite_utils/utils.py | 2 ++ tests/test_rows_from_file.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 3145a6f..06404eb 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -373,6 +373,8 @@ def rows_from_file( raise TypeError( "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO" ) + if not first_bytes: + return (), Format.CSV if first_bytes.startswith((b"[", b"{")): # TODO: Detect newline-JSON return rows_from_file(buffered, format=Format.JSON) diff --git a/tests/test_rows_from_file.py b/tests/test_rows_from_file.py index 8c080d6..3de3582 100644 --- a/tests/test_rows_from_file.py +++ b/tests/test_rows_from_file.py @@ -20,6 +20,13 @@ def test_rows_from_file_detect_format(input, expected_format): assert rows_list == [{"id": "1", "name": "Cleo"}] +@pytest.mark.parametrize("input", (b"", b" \n\t")) +def test_rows_from_file_empty_input(input): + rows, format = rows_from_file(BytesIO(input)) + assert format == Format.CSV + assert list(rows) == [] + + @pytest.mark.parametrize( "ignore_extras,extras_key,expected", ( From c5063f67b10ff194392dcbce7b64f409f866dd72 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 14:14:27 -0700 Subject: [PATCH 098/110] Use quoted SQL identifiers in convert --dry-run, closes #829 --- sqlite_utils/cli.py | 10 +++++----- tests/test_cli_convert.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c90c137..a8baff8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3283,12 +3283,12 @@ def convert( db.conn.create_function("preview_transform", 1, preview) sql = """ select - [{column}] as value, - preview_transform([{column}]) as preview - from [{table}]{where} limit 10 + {column} as value, + preview_transform({column}) as preview + from {table}{where} limit 10 """.format( - column=columns[0], - table=table, + column=quote_identifier(columns[0]), + table=quote_identifier(table), where=f" where {where}" if where is not None else "", ) for row in db.conn.execute(sql, where_args).fetchall(): diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 1101f0f..9f59d59 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -181,6 +181,34 @@ def test_convert_dryrun(test_db_and_path): assert result.output.strip().split("\n")[-1] == "Would affect 1 row" +def test_convert_dryrun_table_and_column_names_containing_closing_bracket( + fresh_db_and_path, +): + db, db_path = fresh_db_and_path + table_name = "table]name" + column_name = "column]name" + db[table_name].insert({column_name: "hello"}) + + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + table_name, + column_name, + "value.upper()", + "--dry-run", + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output.strip() == ( + "hello\n --- becomes:\nHELLO\n\nWould affect 1 row" + ) + assert list(db[table_name].rows) == [{column_name: "hello"}] + + def test_convert_multi_dryrun(test_db_and_path): db_path = test_db_and_path[1] result = CliRunner().invoke( From e6be6267a4eda2d35e57a50400208fe1bb66d6d3 Mon Sep 17 00:00:00 2001 From: nyxst4ck Date: Wed, 12 Aug 2026 18:15:17 -0300 Subject: [PATCH 099/110] Use quote_identifier() in indexes/xindexes PRAGMA statements (#825) Closes #824 --- sqlite_utils/db.py | 14 ++++---------- tests/test_introspect.py | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 66dc700..a59597b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2375,14 +2375,11 @@ class Table(Queryable): @property def indexes(self) -> list[Index]: "List of indexes defined on this table." - sql = f'PRAGMA index_list("{self.name}")' + sql = f"PRAGMA index_list({quote_identifier(self.name)})" indexes = [] for row in self.db.execute_returning_dicts(sql): index_name = row["name"] - index_name_quoted = ( - f'"{index_name}"' if not index_name.startswith('"') else index_name - ) - column_sql = f"PRAGMA index_info({index_name_quoted})" + column_sql = f"PRAGMA index_info({quote_identifier(index_name)})" columns = [] for seqno, cid, name in self.db.execute(column_sql).fetchall(): columns.append(name) @@ -2397,14 +2394,11 @@ class Table(Queryable): @property def xindexes(self) -> list[XIndex]: "List of indexes defined on this table using the more detailed ``XIndex`` format." - sql = f'PRAGMA index_list("{self.name}")' + sql = f"PRAGMA index_list({quote_identifier(self.name)})" indexes = [] for row in self.db.execute_returning_dicts(sql): index_name = row["name"] - index_name_quoted = ( - f'"{index_name}"' if not index_name.startswith('"') else index_name - ) - column_sql = f"PRAGMA index_xinfo({index_name_quoted})" + column_sql = f"PRAGMA index_xinfo({quote_identifier(index_name)})" index_columns = [] for info in self.db.execute(column_sql).fetchall(): index_columns.append(XIndexColumn(*info)) diff --git a/tests/test_introspect.py b/tests/test_introspect.py index b0953f1..03b02cc 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -161,6 +161,31 @@ def test_xindexes(fresh_db): ] +def test_indexes_with_double_quotes_in_identifiers(fresh_db): + fresh_db['Go"sh'].insert({"id": 1, 'c"1': 2}, pk="id") + fresh_db['Go"sh'].create_index(['c"1']) + assert [(index.name, index.columns) for index in fresh_db['Go"sh'].indexes] == [ + ('idx_Go"sh_c"1', ['c"1']) + ] + assert fresh_db['Go"sh'].xindexes == [ + XIndex( + name='idx_Go"sh_c"1', + columns=[ + XIndexColumn(seqno=0, cid=1, name='c"1', desc=0, coll="BINARY", key=1), + XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll="BINARY", key=0), + ], + ) + ] + + +def test_transform_table_with_double_quotes_in_identifiers(fresh_db): + fresh_db['Go"sh'].insert({"id": 1, 'c"1': 2, "c2": 3}, pk="id") + fresh_db['Go"sh'].create_index(['c"1']) + fresh_db['Go"sh'].transform(types={"c2": str}) + assert fresh_db['Go"sh'].columns_dict["c2"] is str + assert [index.columns for index in fresh_db['Go"sh'].indexes] == [['c"1']] + + @pytest.mark.parametrize( "column,expected_table_guess", ( From 88b48fa1674c396bfda330d1c609bc7108f952f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 14:19:30 -0700 Subject: [PATCH 100/110] Fixed introspection of default values TRUE / FALSE / NULL Closes #836 --- sqlite_utils/db.py | 7 +++++++ tests/test_create.py | 20 ++++++++++++++++++++ tests/test_introspect.py | 15 +++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a59597b..2478189 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -5270,6 +5270,13 @@ def _decode_default_value(value: str) -> object: # It's a binary string, stored as hex to_decode = value[2:-1] return binascii.unhexlify(to_decode) + upper = value.upper() + if upper == "TRUE": + return True + if upper == "FALSE": + return False + if upper == "NULL": + return None # If it is a string containing a floating point number: try: return float(value) diff --git a/tests/test_create.py b/tests/test_create.py index 83ce403..e900aee 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1509,6 +1509,26 @@ def test_create_transform(fresh_db, cols, kwargs, expected_schema, should_transf assert fresh_db.table("demo").count == 1 +def test_create_transform_keyword_literal_defaults_unchanged(fresh_db): + fresh_db.execute( + "create table demo (" + "id integer primary key, " + "enabled integer default TRUE, " + "disabled integer default FALSE, " + "nullable text default NULL" + ")" + ) + traces = [] + with fresh_db.tracer(lambda sql, parameters: traces.append((sql, parameters))): + fresh_db.table("demo").create( + {"id": int, "enabled": int, "disabled": int, "nullable": str}, + pk="id", + defaults={"enabled": True, "disabled": False, "nullable": None}, + transform=True, + ) + assert not any(sql.startswith("CREATE TABLE") for sql, _ in traces) + + def test_rename_table(fresh_db): fresh_db.table("t").insert({"foo": "bar"}) assert ["t"] == fresh_db.table_names() diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 03b02cc..343424d 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -393,6 +393,21 @@ def test_table_default_values_escaped_quotes(fresh_db): assert fresh_db.table("t").default_values == {"name": "O'Brien"} +def test_table_default_values_keyword_literals(fresh_db): + fresh_db.execute( + "create table t (" + "enabled integer default TRUE, " + "disabled integer default false, " + "nullable text default NULL" + ")" + ) + assert fresh_db.table("t").default_values == { + "enabled": True, + "disabled": False, + "nullable": None, + } + + def test_pks_use_primary_key_declaration_order(fresh_db): # PRIMARY KEY (a, b) declared against columns stored in order (b, a) - # pks must follow the declaration order, which is what SQLite uses to From e4784ec1200b7408a037c50009dc07d88a5ac577 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 14:38:20 -0700 Subject: [PATCH 101/110] Changelog updates Refs #808, #811, #816, #821, #824, #825, #828, #829, #833, #836, #837 --- docs/changelog.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2950dd4..8540e1b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,14 @@ Unreleased - ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) - ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) - ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) +- ``table.default_values`` now unescapes doubled single quotes in string defaults, so a default such as ``'O''Brien'`` is returned as ``"O'Brien"``. Thanks, `ikatyal2110 `__. (`#811 `__) +- ``table.default_values`` now decodes unquoted ``TRUE``, ``FALSE`` and ``NULL`` default literals as ``True``, ``False`` and ``None`` respectively. (:issue:`836`) +- ``table.enable_fts(..., tokenize=...)`` and ``sqlite-utils enable-fts --tokenize`` now safely quote the tokenizer argument, preventing a crafted value from injecting additional SQL. Thanks, `Bunlong Heng `__. (`#828 `__) +- ``rows_where()``, ``pks_and_rows_where()``, ``search()`` and ``search_sql()`` now support ``offset=`` without requiring ``limit=``. The ``sqlite-utils rows --offset`` option now works without ``--limit`` too. Thanks, `ethanhawkes-gif `__. (:issue:`816`, `#821 `__) +- Empty or whitespace-only input passed to ``rows_from_file()`` is now handled as an empty CSV file instead of raising ``csv.Error``. Thanks, `Rami Abdelrazzaq `__. (:issue:`808`, `#837 `__) +- ``sqlite-utils convert --dry-run`` now works for table and column names containing closing square brackets. (:issue:`829`) +- ``table.indexes`` and ``table.xindexes`` now work for table, index and column names containing double quotes. This also fixes ``table.transform()`` for tables with those identifiers. Thanks, `nyxst4ck `__. (:issue:`824`, `#825 `__) +- Improved type annotations throughout the package and added Pyright regression checks to CI. (:issue:`833`) .. _v3_39_1: From 57192ef4e36c334bc2946a10547bf64d63621127 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 14:42:24 -0700 Subject: [PATCH 102/110] table.transform(rename=...) now preserves indexes, closes #822 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 98 +++++++++++++++++++++++++++++++---------- tests/test_transform.py | 80 ++++++++++++++++++++++++++++----- 3 files changed, 146 insertions(+), 33 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8540e1b..31b4ea3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ Unreleased - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) - ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) - ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) +- ``table.transform(rename=...)`` now preserves explicit indexes on renamed columns by dropping and recreating those indexes against the new column names. Previously this raised a ``TransformError``. (:issue:`822`) - ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) - ``table.default_values`` now unescapes doubled single quotes in string defaults, so a default such as ``'O''Brien'`` is returned as ``"O'Brien"``. Thanks, `ikatyal2110 `__. (`#811 `__) - ``table.default_values`` now decodes unquoted ``TRUE``, ``FALSE`` and ``NULL`` default literals as ``True``, ``False`` and ``None`` respectively. (:issue:`836`) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2478189..edefbab 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2947,6 +2947,80 @@ class Table(Queryable): new_cols=", ".join(quote_identifier(col) for col in new_cols), ) sqls.append(copy_sql) + # Capture indexes before the old table is changed. Simple indexes that + # reference renamed columns are recreated from structured PRAGMA + # metadata instead of editing their stored CREATE INDEX SQL. + index_drop_sqls = [] + index_create_sqls = [] + xindexes_by_name = {index.name: index for index in self.xindexes} + for index in self.indexes: + if index.origin == "pk": + continue + index_sql = self.db.execute( + """SELECT sql FROM sqlite_master WHERE type = 'index' AND name = :index_name;""", + {"index_name": index.name}, + ).fetchall()[0][0] + if index_sql is None: + raise TransformError( + f"Index '{index.name}' on table '{self.name}' does not have a " + "CREATE INDEX statement. You must manually drop this index prior to running this " + "transformation and manually recreate the new index after running this transformation." + ) + dropped_index_column = next( + (column for column in index.columns if column in drop), None + ) + renamed_index_column = next( + (column for column in index.columns if column in rename), None + ) + if dropped_index_column is not None: + raise TransformError( + f"Index '{index.name}' column '{dropped_index_column}' is not in updated table '{self.name}'. " + f"You must manually drop this index prior to running this transformation " + f"and manually recreate the new index after running this transformation. " + f"The original index sql statement is: `{index_sql}`. No changes have been applied to this table." + ) + xindex = xindexes_by_name[index.name] + indexed_columns = sorted( + (column for column in xindex.columns if column.key), + key=lambda column: column.seqno, + ) + if (rename or drop) and ( + index.partial or any(column.name is None for column in indexed_columns) + ): + raise TransformError( + f"Index '{index.name}' is a partial or expression index, so it " + f"cannot be safely recreated while columns are renamed or dropped. " + f"You must manually drop this index prior to running this transformation " + f"and manually recreate the new index after running this transformation. " + f"The original index sql statement is: `{index_sql}`. No changes have been applied to this table." + ) + if renamed_index_column is not None: + columns_sql = [] + for column in indexed_columns: + assert column.name is not None + column_sql = quote_identifier( + rename.get(column.name) or column.name + ) + if column.coll and column.coll.upper() != "BINARY": + column_sql += f" COLLATE {quote_identifier(column.coll)}" + if column.desc: + column_sql += " DESC" + columns_sql.append(column_sql) + index_sql = "CREATE {unique}INDEX {index_name} ON {table_name} ({columns})".format( + unique="UNIQUE " if index.unique else "", + index_name=quote_identifier(index.name), + table_name=quote_identifier(self.name), + columns=", ".join(columns_sql), + ) + index_drop_sqls.append( + f"DROP INDEX IF EXISTS {quote_identifier(index.name)};" + ) + elif keep_table: + index_drop_sqls.append( + f"DROP INDEX IF EXISTS {quote_identifier(index.name)};" + ) + index_create_sqls.append(index_sql) + sqls.extend(index_drop_sqls) # Drop (or keep) the old table, then rename the new one into place. # Since SQLite 3.25 ALTER TABLE ... RENAME TO rewrites references to # the renamed table in every view definition, which fails if a view @@ -2976,29 +3050,7 @@ class Table(Queryable): ) ) # Re-add existing indexes - for index in self.indexes: - if index.origin != "pk": - index_sql = self.db.execute( - """SELECT sql FROM sqlite_master WHERE type = 'index' AND name = :index_name;""", - {"index_name": index.name}, - ).fetchall()[0][0] - if index_sql is None: - raise TransformError( - f"Index '{index.name}' on table '{self.name}' does not have a " - "CREATE INDEX statement. You must manually drop this index prior to running this " - "transformation and manually recreate the new index after running this transformation." - ) - if keep_table: - sqls.append(f"DROP INDEX IF EXISTS {quote_identifier(index.name)};") - for col in index.columns: - if col in rename or col in drop: - raise TransformError( - f"Index '{index.name}' column '{col}' is not in updated table '{self.name}'. " - f"You must manually drop this index prior to running this transformation " - f"and manually recreate the new index after running this transformation. " - f"The original index sql statement is: `{index_sql}`. No changes have been applied to this table." - ) - sqls.append(index_sql) + sqls.extend(index_create_sqls) return sqls def extract( diff --git a/tests/test_transform.py b/tests/test_transform.py index 28fa4d7..5793f10 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -893,22 +893,15 @@ def test_transform_retains_indexes_with_foreign_keys(fresh_db): ), f"Indexes before transform: {indexes_before_transform}\nIndexes after transform: {dogs.indexes}" -@pytest.mark.parametrize( - "transform_params", - [ - {"rename": {"age": "dog_age"}}, - {"drop": ["age"]}, - ], -) -def test_transform_with_indexes_errors(fresh_db, transform_params): - # Should error with a compound (name, age) index if age is renamed or dropped +def test_transform_with_indexes_errors(fresh_db): + # Should error with a compound (name, age) index if age is dropped dogs = fresh_db.table("dogs") dogs.insert({"id": 1, "name": "Cleo", "age": 5}, pk="id") dogs.create_index(["name", "age"]) with pytest.raises(TransformError) as excinfo: - dogs.transform(**transform_params) + dogs.transform(drop=["age"]) assert ( "Index 'idx_dogs_name_age' column 'age' is not in updated table 'dogs'. " @@ -917,6 +910,73 @@ def test_transform_with_indexes_errors(fresh_db, transform_params): ) +@pytest.mark.parametrize( + ("table_name", "index_name"), + (("name", "idx_name"), ("t", "name")), +) +def test_transform_rename_column_with_index(fresh_db, table_name, index_name): + # https://github.com/simonw/sqlite-utils/issues/822 + # Use the same name for the table, column and index to ensure only the + # indexed column changes. + table = fresh_db.table(table_name) + table.insert({"id": 1, "name": "Cleo"}, pk="id") + table.create_index(["name"], index_name=index_name) + + sqls = table.transform_sql(rename={"name": "full_name"}, tmp_suffix="suffix") + drop_index_sql = f'DROP INDEX IF EXISTS "{index_name}";' + assert drop_index_sql in sqls + assert sqls.index(drop_index_sql) < sqls.index(f'DROP TABLE "{table_name}";') + + table.transform(rename={"name": "full_name"}) + + assert [column.name for column in table.columns] == ["id", "full_name"] + assert [(index.name, index.columns) for index in table.indexes] == [ + (index_name, ["full_name"]) + ] + + +def test_transform_recreates_renamed_index_from_metadata(fresh_db): + table = fresh_db.table("t") + table.insert({"alpha": "one", "beta": "two"}) + # Deliberately use unquoted SQL and index details that need to survive the + # reconstruction. Renaming both columns also guards against cascading + # string substitutions. + fresh_db.execute( + "CREATE UNIQUE INDEX swap_idx ON t(alpha COLLATE NOCASE DESC, beta)" + ) + + table.transform(rename={"alpha": "beta", "beta": "alpha"}) + + assert table.columns_dict == {"beta": str, "alpha": str} + assert [(index.name, index.unique, index.columns) for index in table.indexes] == [ + ("swap_idx", 1, ["beta", "alpha"]) + ] + key_columns = [column for column in table.xindexes[0].columns if column.key] + assert [(column.name, column.desc, column.coll) for column in key_columns] == [ + ("beta", 1, "NOCASE"), + ("alpha", 0, "BINARY"), + ] + + +@pytest.mark.parametrize( + "index_sql", + ( + "CREATE INDEX idx_t_name ON t(lower(name))", + "CREATE INDEX idx_t_name ON t(name) WHERE name IS NOT NULL", + ), +) +def test_transform_rename_complex_index_errors(fresh_db, index_sql): + table = fresh_db.table("t") + table.insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.execute(index_sql) + + with pytest.raises(TransformError, match="partial or expression index"): + table.transform(rename={"name": "full_name"}) + + assert table.columns_dict == {"id": int, "name": str} + assert [index.name for index in table.indexes] == ["idx_t_name"] + + def test_transform_with_unique_constraint_implicit_index(fresh_db): dogs = fresh_db.table("dogs") # Create a table with a UNIQUE constraint on 'name', which creates an implicit index From fcfccea8132e4aa6167a14f9afec5a690de7485c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 16:43:33 -0700 Subject: [PATCH 103/110] Support ANY column types for strict tables Closes #790, #820 --- docs/changelog.rst | 1 + docs/cli-reference.rst | 6 +-- docs/cli.rst | 17 +++++++- docs/python-api.rst | 23 +++++++++- sqlite_utils/__init__.py | 11 ++++- sqlite_utils/cli.py | 22 ++++++---- sqlite_utils/db.py | 14 +++++++ sqlite_utils/utils.py | 6 +++ tests/test_cli.py | 79 ++++++++++++++++++++++++++++++++++- tests/test_column_affinity.py | 3 ++ tests/test_create.py | 40 ++++++++++++++++++ tests/test_extract.py | 36 ++++++++++++++++ tests/test_transform.py | 50 ++++++++++++++++++++++ 13 files changed, 292 insertions(+), 16 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 31b4ea3..0ef85ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) +- New ``sqlite_utils.ANY`` marker type for creating and introspecting SQLite ``ANY`` columns. The Python API and CLI can create, add and transform these columns, and ``table.transform()`` and ``table.extract()`` now preserve ``ANY`` columns and their values in ``STRICT`` tables. (:issue:`790`) - ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) - ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) - ``table.transform(rename=...)`` now preserves explicit indexes on renamed columns by dropping and recreating those indexes against the new column names. Previously this raised a ``TransformError``. (:issue:`822`) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index a4ec402..c53d642 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -494,7 +494,7 @@ See :ref:`cli_transform_table`. Options: --type ... Change column type to INTEGER, TEXT, FLOAT, - REAL or BLOB + REAL, BLOB or ANY --drop TEXT Drop this column --rename ... Rename this column to X -o, --column-order TEXT Reorder columns @@ -963,7 +963,7 @@ See :ref:`cli_create_table`. height real \ photo blob --pk id - Valid column types are text, integer, real, float and blob. + Valid column types are text, integer, real, float, blob and any. Options: --pk TEXT Column to use as primary key @@ -1257,7 +1257,7 @@ See :ref:`cli_add_column`. :: Usage: sqlite-utils add-column [OPTIONS] PATH TABLE COL_NAME - [integer|int|float|real|text|str|blob|bytes] + [integer|int|float|real|text|str|blob|bytes|any] Add a column to the specified table diff --git a/docs/cli.rst b/docs/cli.rst index cf241aa..417911a 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1390,7 +1390,14 @@ Use ``--type column-name type`` to override the type automatically chosen when t This is useful for values such as ZIP codes, which may look like integers but should be stored as ``TEXT`` to preserve leading zeros. -The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL`` or ``BLOB``. Column types are matched case-insensitively. +The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL``, ``BLOB`` or ``ANY``. Column types are matched case-insensitively. + +``ANY`` is especially useful with ``--strict``. An ``ANY`` column in a strict table preserves values without coercion, so text such as ``000123`` remains text instead of being converted to an integer: + +.. code-block:: bash + + sqlite-utils insert events.db events events.csv --csv --strict \ + --type payload any As with detected column types, ``--type`` only affects tables created by the command. If the table already exists, its existing column types are left unchanged. @@ -2141,6 +2148,12 @@ You can create a table in `SQLite STRICT mode ` @@ -1569,7 +1582,7 @@ You can specify the ``col_type`` argument either using a SQLite type as a string The ``col_type`` is optional - if you omit it the type of ``TEXT`` will be used. -SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"``, ``"REAL"`` or ``"BLOB"``. +SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"``, ``"REAL"``, ``"BLOB"`` or ``"ANY"``. You can use the ``sqlite_utils.ANY`` marker instead of the ``"ANY"`` string. If you pass a Python type, it will be mapped to SQLite types as shown here:: @@ -1582,6 +1595,7 @@ If you pass a Python type, it will be mapped to SQLite types as shown here:: datetime.date: "TEXT" datetime.time: "TEXT" datetime.timedelta: "TEXT" + sqlite_utils.ANY: "ANY" # If numpy is installed np.int8: "INTEGER" @@ -1831,6 +1845,8 @@ Pass ``strict=False`` to convert a strict table back to a regular non-strict tab table.transform(strict=False) +If the table has ``ANY`` columns, converting it to non-strict mode can coerce text values that look numeric. For example, SQLite converts ``"000123"`` to the integer ``123`` when copying it into an ordinary ``ANY`` column. This is SQLite's documented distinction between `STRICT and ordinary ANY columns `__. + The default is ``strict=None``, which preserves the table's existing strict mode. Passing ``strict=True`` raises ``sqlite_utils.db.TransformError`` if the available SQLite version does not support strict tables. @@ -2458,6 +2474,11 @@ The ``.columns_dict`` property returns a dictionary version of the columns with >>> db.table("PlantType").columns_dict {'id': , 'value': } +SQLite ``ANY`` columns are represented by the ``sqlite_utils.ANY`` marker type:: + + >>> db.table("events").columns_dict + {'id': , 'payload': } + .. _python_api_introspection_default_values: .default_values diff --git a/sqlite_utils/__init__.py b/sqlite_utils/__init__.py index 0d25716..3f350e1 100644 --- a/sqlite_utils/__init__.py +++ b/sqlite_utils/__init__.py @@ -1,6 +1,13 @@ from .db import Database from .hookspecs import hookimpl, hookspec from .migrations import Migrations -from .utils import suggest_column_types +from .utils import ANY, suggest_column_types -__all__ = ["Database", "Migrations", "hookimpl", "hookspec", "suggest_column_types"] +__all__ = [ + "ANY", + "Database", + "Migrations", + "hookimpl", + "hookspec", + "suggest_column_types", +] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index a8baff8..c230902 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -76,7 +76,7 @@ def _close_databases(ctx): pass -VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "REAL", "BLOB") +VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "REAL", "BLOB", "ANY") UNICODE_ERROR = """ {} @@ -489,7 +489,17 @@ def dump(path, load_extension): @click.argument( "col_type", type=click.Choice( - ["integer", "int", "float", "real", "text", "str", "blob", "bytes"], + [ + "integer", + "int", + "float", + "real", + "text", + "str", + "blob", + "bytes", + "any", + ], case_sensitive=False, ), required=False, @@ -1758,7 +1768,7 @@ def create_table( height real \\ photo blob --pk id - Valid column types are text, integer, real, float and blob. + Valid column types are text, integer, real, float, blob and any. """ db = sqlite_utils.Database(path) _register_db_for_cleanup(db) @@ -2668,12 +2678,10 @@ def schema( "--type", type=( str, - click.Choice( - ["INTEGER", "TEXT", "FLOAT", "REAL", "BLOB"], case_sensitive=False - ), + click.Choice(list(VALID_COLUMN_TYPES), case_sensitive=False), ), multiple=True, - help="Change column type to INTEGER, TEXT, FLOAT, REAL or BLOB", + help="Change column type to INTEGER, TEXT, FLOAT, REAL, BLOB or ANY", ) @click.option("--drop", type=str, multiple=True, help="Drop this column") @click.option( diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index edefbab..9c0b402 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -39,6 +39,7 @@ from .create_table_parser import ( sql_ends_in_line_comment, ) from .utils import ( + ANY, OperationalError, chunks, column_affinity, @@ -366,6 +367,7 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = { decimal.Decimal: "REAL", None.__class__: "TEXT", uuid.UUID: "TEXT", + ANY: "ANY", # SQLite explicit types "TEXT": "TEXT", "INTEGER": "INTEGER", @@ -380,6 +382,8 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = { "real": "REAL", "blob": "BLOB", "bytes": "BLOB", + "ANY": "ANY", + "any": "ANY", } # If numpy is available, add more types if np: @@ -3092,6 +3096,15 @@ class Table(Queryable): if col in columns } if lookup_table.exists(): + if ( + self.strict + and ANY in lookup_columns_definition.values() + and not lookup_table.strict + ): + raise InvalidColumns( + f"Lookup table {table} already exists but is not STRICT, " + "so it cannot preserve ANY column values" + ) if not set(lookup_columns_definition.items()).issubset( lookup_table.columns_dict.items() ): @@ -3105,6 +3118,7 @@ class Table(Queryable): **lookup_columns_definition, }, pk="id", + strict=self.strict, ) lookup_columns = [(rename.get(col) or col) for col in columns] lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 06404eb..ee6695b 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -59,6 +59,10 @@ Row = dict[str, RowValue] T = TypeVar("T") +class ANY: + """Marker type for an SQLite ``ANY`` column.""" + + class _CloseableIterator(Iterator[Row]): """Iterator wrapper that closes a file when iteration is complete.""" @@ -178,6 +182,8 @@ def column_affinity(column_type: str) -> type: return bytes if "REAL" in column_type or "FLOA" in column_type or "DOUB" in column_type: return float + if column_type == "ANY": + return ANY # Default is 'NUMERIC', which we currently also treat as float return float diff --git a/tests/test_cli.py b/tests/test_cli.py index f60c7d5..064026a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,7 +9,7 @@ from pathlib import Path import pytest from click.testing import CliRunner -from sqlite_utils import Database, cli +from sqlite_utils import ANY, Database, cli from sqlite_utils.db import ForeignKey, Index @@ -355,6 +355,7 @@ def test_create_index_desc(db_path): ("blob", "BLOB", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), ("blob", "bytes", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), ("blob", "BYTES", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), + ("anything", "any", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "anything" ANY)'), ("default", None, 'CREATE TABLE "dogs" (\n "name" TEXT\n, "default" TEXT)'), ), ) @@ -2007,6 +2008,25 @@ def test_transform_strict_option_with_invalid_data(db_path): assert not any(name.startswith("dogs_new_") for name in db.table_names()) +def test_transform_column_to_any(db_path): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db.table("items").create({"data": str}, strict=True) + db.table("items").insert({"data": "000123"}) + + result = CliRunner().invoke( + cli.cli, ["transform", db_path, "items", "--type", "data", "any"] + ) + + assert result.exit_code == 0, result.output + assert db.table("items").columns_dict == {"data": ANY} + assert db.execute("select typeof(data), data from items").fetchone() == ( + "text", + "000123", + ) + + @pytest.mark.parametrize( "extra_args,expected_schema", ( @@ -2872,6 +2892,30 @@ def test_create_table_strict(strict): assert db.table("items").columns_dict == {"id": int, "w": float} +def test_create_table_strict_any(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + result = runner.invoke( + cli.cli, + [ + "create-table", + "test.db", + "items", + "id", + "integer", + "data", + "any", + "--strict", + ], + ) + assert result.exit_code == 0, result.output + assert db.table("items").strict is True + assert db.table("items").columns_dict == {"id": int, "data": ANY} + + @pytest.mark.parametrize("method", ("insert", "upsert")) @pytest.mark.parametrize("strict", (False, True)) def test_insert_upsert_strict(tmpdir, method, strict): @@ -2887,6 +2931,39 @@ def test_insert_upsert_strict(tmpdir, method, strict): assert db.table("items").strict == strict or not db.supports_strict +@pytest.mark.parametrize("method", ("insert", "upsert")) +def test_insert_upsert_strict_any(tmpdir, method): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db.close() + result = CliRunner().invoke( + cli.cli, + [ + method, + db_path, + "items", + "-", + "--csv", + "--pk", + "id", + "--type", + "data", + "any", + "--strict", + ], + input="id,data\n1,000123", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert db.table("items").columns_dict == {"id": int, "data": ANY} + assert db.execute("select typeof(data), data from items").fetchone() == ( + "text", + "000123", + ) + + def test_extract_bad_column_clean_error(db_path): db = Database(db_path) db.table("trees").insert({"id": 1, "species": "Palm"}, pk="id") diff --git a/tests/test_column_affinity.py b/tests/test_column_affinity.py index 8c619e1..2d7846e 100644 --- a/tests/test_column_affinity.py +++ b/tests/test_column_affinity.py @@ -1,5 +1,6 @@ import pytest +from sqlite_utils import ANY from sqlite_utils.utils import column_affinity EXAMPLES = [ @@ -26,6 +27,8 @@ EXAMPLES = [ ("DOUBLE", float), ("DOUBLE PRECISION", float), ("FLOAT", float), + ("ANY", ANY), + ("any", ANY), # Numeric, treated as float: ("NUMERIC", float), ("DECIMAL(10,5)", float), diff --git a/tests/test_create.py b/tests/test_create.py index e900aee..b738df9 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -7,6 +7,7 @@ import uuid import pytest +from sqlite_utils import ANY from sqlite_utils.db import ( AlterError, Database, @@ -1366,6 +1367,18 @@ def test_quote(fresh_db, input, expected): {"col": list}, '"col" TEXT', ), + ( + {"col": ANY}, + '"col" ANY', + ), + ( + {"col": "ANY"}, + '"col" ANY', + ), + ( + {"col": "any"}, + '"col" ANY', + ), ), ) def test_create_table_sql(fresh_db, columns, expected_sql_middle): @@ -1589,6 +1602,33 @@ def test_create_strict(fresh_db, strict): assert table.strict == strict or not fresh_db.supports_strict +def test_create_strict_with_any(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + table = fresh_db.table("items").create( + {"id": int, "data": ANY}, pk="id", strict=True + ) + table.insert_all( + [ + {"id": 1, "data": 42}, + {"id": 2, "data": "000123"}, + {"id": 3, "data": 3.14}, + {"id": 4, "data": b"bytes"}, + {"id": 5, "data": None}, + ] + ) + assert table.columns_dict == {"id": int, "data": ANY} + assert fresh_db.execute( + "select typeof(data), data from items order by id" + ).fetchall() == [ + ("integer", 42), + ("text", "000123"), + ("real", 3.14), + ("blob", b"bytes"), + ("null", None), + ] + + def test_bad_table_and_view_exceptions(fresh_db): fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.create_view("v", "select * from t") diff --git a/tests/test_extract.py b/tests/test_extract.py index 72579c4..f855041 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -2,6 +2,7 @@ import itertools import pytest +from sqlite_utils import ANY from sqlite_utils.db import InvalidColumns @@ -305,3 +306,38 @@ def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db): fresh_db.table("t1").extract(["species"], table="lk") fresh_db.table("t2").extract(["species"], table="lk") assert fresh_db.table("lk").count == 1 + + +def test_extract_preserves_strict_any(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (id integer primary key, data any) strict") + fresh_db.execute("insert into items values (1, ?)", ("000123",)) + + fresh_db["items"].extract("data", table="data_values") + + lookup = fresh_db["data_values"] + assert lookup.strict is True + assert lookup.columns_dict == {"id": int, "data": ANY} + assert fresh_db.execute( + "select typeof(data), data from data_values" + ).fetchone() == ("text", "000123") + + +def test_extract_strict_any_rejects_non_strict_lookup(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (data any) strict") + fresh_db.execute("insert into items values (?)", ("000123",)) + fresh_db.execute("create table data_values (id integer primary key, data any)") + + with pytest.raises( + InvalidColumns, + match="is not STRICT, so it cannot preserve ANY column values", + ): + fresh_db["items"].extract("data", table="data_values") + + assert fresh_db.execute("select typeof(data), data from items").fetchone() == ( + "text", + "000123", + ) diff --git a/tests/test_transform.py b/tests/test_transform.py index 5793f10..6a8a143 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -2,6 +2,7 @@ import sqlite3 import pytest +from sqlite_utils import ANY from sqlite_utils.db import Check, ForeignKey, TransactionError, TransformError from sqlite_utils.utils import OperationalError @@ -823,6 +824,55 @@ def test_transform_to_strict_not_supported(fresh_db, method_name): assert table.strict is False +def test_transform_preserves_any_column_in_strict_table(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (id integer primary key, data any) strict") + fresh_db.conn.executemany( + "insert into items values (?, ?)", + [ + (1, 42), + (2, "000123"), + (3, 3.14), + (4, b"bytes"), + (5, None), + ], + ) + table = fresh_db["items"] + + table.transform() + + assert table.strict is True + assert table.columns_dict == {"id": int, "data": ANY} + assert fresh_db.execute( + "select typeof(data), data from items order by id" + ).fetchall() == [ + ("integer", 42), + ("text", "000123"), + ("real", 3.14), + ("blob", b"bytes"), + ("null", None), + ] + + +def test_transform_any_column_from_strict_to_non_strict(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (data any) strict") + fresh_db.execute("insert into items values (?)", ("000123",)) + table = fresh_db["items"] + + table.transform(strict=False) + + assert table.strict is False + assert table.columns_dict == {"data": ANY} + # Ordinary non-STRICT ANY columns apply NUMERIC affinity + assert fresh_db.execute("select typeof(data), data from items").fetchone() == ( + "integer", + 123, + ) + + @pytest.mark.parametrize( "indexes, transform_params", [ From 2b52b5ed6f4a6e553e3620d8424374fc7cbf95fd Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 18:39:00 -0700 Subject: [PATCH 104/110] Preserve AUTOINCREMENT through transforms --- docs/changelog.rst | 1 + sqlite_utils/create_table_parser.py | 30 ++++++++++++++- sqlite_utils/db.py | 60 +++++++++++++++++++++++++++++ tests/test_create_table_parser.py | 31 +++++++++++++++ tests/test_transform.py | 18 +++++++++ 5 files changed, 139 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0ef85ca..624808f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Unreleased ---------- +- ``table.transform()`` now preserves ``AUTOINCREMENT`` primary keys and their sequence high-water marks. Previously a transform removed ``AUTOINCREMENT`` and could reuse deleted row IDs. (:issue:`602`) - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) - New ``sqlite_utils.ANY`` marker type for creating and introspecting SQLite ``ANY`` columns. The Python API and CLI can create, add and transform these columns, and ``table.transform()`` and ``table.extract()`` now preserve ``ANY`` columns and their values in ``STRICT`` tables. (:issue:`790`) - ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py index 2d891ae..d426286 100644 --- a/sqlite_utils/create_table_parser.py +++ b/sqlite_utils/create_table_parser.py @@ -1,4 +1,4 @@ -"""Helpers for parsing CHECK constraints from SQLite CREATE TABLE SQL. +"""Helpers for parsing constraints from SQLite CREATE TABLE SQL. SQLite does not expose CHECK constraints through a pragma, so preserving them across a table rebuild requires reading ``sqlite_schema.sql``. This module is @@ -564,6 +564,34 @@ def parse_checks(create_sql: str) -> list[Check]: return checks +def parse_autoincrement(create_sql: str) -> str | None: + """Return the AUTOINCREMENT column from a valid CREATE TABLE statement.""" + body_info = _table_body(create_sql) + if body_info is None: + return None + body, _ = body_info + for item, _, _ in _split_spans(body, _lex(body)): + item_tokens = _meaningful(_lex(item)) + if not item_tokens: + continue + head = item_tokens[0] + if ( + head.kind == "word" and head.text.upper() in _TABLE_CONSTRAINT_KEYWORDS + ) or head.is_keyword("CONSTRAINT"): + continue + column = _unquote(head.text) + index = 1 + while index < len(item_tokens): + token = item_tokens[index] + if token.text == "(": + index = _matching_paren(item_tokens, index) + 1 + continue + if token.is_keyword("AUTOINCREMENT"): + return column + index += 1 + return None + + def parse_column_comments(create_sql: str) -> dict[str, ColumnComments]: """Return comments immediately before and after each column definition.""" body_info = _table_body(create_sql) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9c0b402..48c5d5d 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -33,6 +33,7 @@ from .create_table_parser import ( ColumnComments, ParseError, check_references_identifier, + parse_autoincrement, parse_checks, parse_column_comments, rewrite_check_expression, @@ -1422,6 +1423,7 @@ class Database: strict: bool = False, _checks: Iterable[Check] | None = None, _column_comments: Mapping[str, ColumnComments] | None = None, + _autoincrement: str | None = None, ) -> str: """ Returns the SQL ``CREATE TABLE`` statement for creating the specified table. @@ -1525,10 +1527,22 @@ class Database: column_items.insert(0, (pk, int)) elif pk: pk = [resolve_casing(p, [c[0] for c in column_items]) for p in pk] + if _autoincrement is not None: + _autoincrement = resolve_casing( + _autoincrement, [c[0] for c in column_items] + ) + if _autoincrement != single_pk: + raise ValueError("AUTOINCREMENT requires a single-column primary key") for column_name, column_type in column_items: column_extras = [] if column_name == single_pk: column_extras.append("PRIMARY KEY") + if column_name == _autoincrement: + if COLUMN_TYPE_MAPPING[column_type] != "INTEGER": + raise ValueError( + "AUTOINCREMENT requires an INTEGER PRIMARY KEY column" + ) + column_extras.append("AUTOINCREMENT") if column_name in not_null: column_extras.append("NOT NULL") if column_name in defaults and defaults[column_name] is not None: @@ -2748,6 +2762,7 @@ class Table(Queryable): try: existing_checks = self.checks existing_column_comments = parse_column_comments(self.schema) + existing_autoincrement = parse_autoincrement(self.schema) except ParseError as ex: raise TransformError( f"Could not parse table schema for table {self.name!r}: {ex}" @@ -2870,6 +2885,11 @@ class Table(Queryable): new_column_pairs.append((new_name, type_)) copy_from_to[name] = new_name + if existing_autoincrement: + existing_autoincrement = resolve_casing( + existing_autoincrement, existing_columns + ) + if pk is DEFAULT: pks_renamed = tuple( rename.get(pk_name) or pk_name @@ -2880,6 +2900,28 @@ class Table(Queryable): else: pk = pks_renamed + create_table_autoincrement = None + if existing_autoincrement and existing_autoincrement not in drop: + renamed_autoincrement = ( + rename.get(existing_autoincrement) or existing_autoincrement + ) + single_pk = pk[0] if isinstance(pk, (list, tuple)) and len(pk) == 1 else pk + new_column_types = dict(new_column_pairs) + if ( + single_pk == renamed_autoincrement + and COLUMN_TYPE_MAPPING.get(new_column_types.get(renamed_autoincrement)) + == "INTEGER" + ): + create_table_autoincrement = renamed_autoincrement + + autoincrement_sequence = None + if create_table_autoincrement: + sequence_row = self.db.execute( + "SELECT seq FROM sqlite_sequence WHERE name = ?", [self.name] + ).fetchone() + if sequence_row is not None: + autoincrement_sequence = sequence_row[0] + # not_null may be a set or dict, need to convert to a set create_table_not_null = { rename.get(c.name) or c.name @@ -2931,6 +2973,7 @@ class Table(Queryable): strict=self.strict if strict is None else strict, _checks=create_table_checks, _column_comments=create_table_column_comments, + _autoincrement=create_table_autoincrement, ).strip() ) @@ -3053,6 +3096,23 @@ class Table(Queryable): "ON" if legacy_alter_table_was_on else "OFF" ) ) + if autoincrement_sequence is not None: + table_name_literal = self.db.quote(self.name) + sqls.extend( + ( + "UPDATE sqlite_sequence SET seq = MAX(seq, {sequence}) " + "WHERE name = {table_name};".format( + sequence=autoincrement_sequence, + table_name=table_name_literal, + ), + "INSERT INTO sqlite_sequence (name, seq) " + "SELECT {table_name}, {sequence} WHERE NOT EXISTS " + "(SELECT 1 FROM sqlite_sequence WHERE name = {table_name});".format( + sequence=autoincrement_sequence, + table_name=table_name_literal, + ), + ) + ) # Re-add existing indexes sqls.extend(index_create_sqls) return sqls diff --git a/tests/test_create_table_parser.py b/tests/test_create_table_parser.py index a7aa0c0..54bf221 100644 --- a/tests/test_create_table_parser.py +++ b/tests/test_create_table_parser.py @@ -8,6 +8,7 @@ from sqlite_utils.create_table_parser import ( Check, ColumnComments, ParseError, + parse_autoincrement, parse_checks, parse_column_comments, ) @@ -117,6 +118,36 @@ def test_virtual_table_has_no_checks(): ) +@pytest.mark.parametrize( + "sql,expected", + [ + ( + "CREATE TABLE t(id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT)", + "id", + ), + ( + 'CREATE TABLE t("quoted id" INTEGER PRIMARY KEY AUTOINCREMENT)', + "quoted id", + ), + ( + 'CREATE TABLE t("autoincrement" INTEGER PRIMARY KEY, value TEXT)', + None, + ), + ( + "CREATE TABLE t(id INTEGER PRIMARY KEY /* AUTOINCREMENT */, value TEXT)", + None, + ), + ( + "CREATE TABLE t(id INTEGER PRIMARY KEY, value TEXT CHECK(value != 'AUTOINCREMENT'))", + None, + ), + ], +) +def test_parse_autoincrement(sql, expected): + sqlite3.connect(":memory:").execute(sql) + assert parse_autoincrement(sql) == expected + + comment_or_space = st.sampled_from( [ " ", diff --git a/tests/test_transform.py b/tests/test_transform.py index 6a8a143..e6096cd 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1053,6 +1053,24 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db): ) +def test_transform_preserves_autoincrement_and_sequence(fresh_db): + fresh_db.execute( + "CREATE TABLE entries (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT)" + ) + entries = fresh_db.table("entries") + entries.insert_all(({"value": "one"}, {"value": "two"})) + entries.delete(2) + + entries.transform(rename={"value": "label"}) + + assert "PRIMARY KEY AUTOINCREMENT" in entries.schema + entries.insert({"label": "three"}) + assert list(entries.rows) == [ + {"id": 1, "label": "one"}, + {"id": 3, "label": "three"}, + ] + + def test_transform_preserves_view(fresh_db): # https://github.com/simonw/sqlite-utils/issues/831 dogs = fresh_db.table("dogs") From 75ba58846206b2c1beb39836134e763bf34177aa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 18:46:18 -0700 Subject: [PATCH 105/110] Preserve composite UNIQUE constraints in transforms --- docs/changelog.rst | 1 + sqlite_utils/create_table_parser.py | 193 ++++++++++++++++++++++++++++ sqlite_utils/db.py | 119 +++++++++++++++++ tests/test_create_table_parser.py | 48 +++++++ tests/test_transform.py | 74 +++++++++-- 5 files changed, 425 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 624808f..fe215fd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Unreleased ---------- +- ``table.transform()`` now preserves column-level and composite ``UNIQUE`` constraints, including constraint names, collations, sort order and ``ON CONFLICT`` behavior. Renaming columns updates those constraints, while dropping any constituent column removes the entire constraint. (:issue:`762`) - ``table.transform()`` now preserves ``AUTOINCREMENT`` primary keys and their sequence high-water marks. Previously a transform removed ``AUTOINCREMENT`` and could reuse deleted row IDs. (:issue:`602`) - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) - New ``sqlite_utils.ANY`` marker type for creating and introspecting SQLite ``ANY`` columns. The Python API and CLI can create, add and transform these columns, and ``table.transform()`` and ``table.extract()`` now preserve ``ANY`` columns and their values in ``STRICT`` tables. (:issue:`790`) diff --git a/sqlite_utils/create_table_parser.py b/sqlite_utils/create_table_parser.py index d426286..7377f4f 100644 --- a/sqlite_utils/create_table_parser.py +++ b/sqlite_utils/create_table_parser.py @@ -32,6 +32,24 @@ class ColumnComments: after: str = "" +@dataclass(frozen=True) +class UniqueColumn: + name: str + collation: str = "" + order: str = "" + + +@dataclass +class Unique: + columns: tuple[UniqueColumn, ...] + name: str = "" + column: str = "" + conflict: str = "" + sql: str = field(default="", compare=False, repr=False) + start: int = field(default=-1, compare=False, repr=False) + end: int = field(default=-1, compare=False, repr=False) + + class ParseError(ValueError): pass @@ -592,6 +610,181 @@ def parse_autoincrement(create_sql: str) -> str | None: return None +_CONFLICT_ACTIONS = frozenset(("ROLLBACK", "ABORT", "FAIL", "IGNORE", "REPLACE")) + + +def _conflict_after(tokens: list[_Token], index: int) -> tuple[str, int]: + if index >= len(tokens) or not tokens[index].is_keyword("ON"): + return "", index + if index + 2 >= len(tokens) or not tokens[index + 1].is_keyword("CONFLICT"): + raise ParseError("ON after UNIQUE must be followed by CONFLICT and an action") + action = tokens[index + 2].text.upper() + if tokens[index + 2].kind != "word" or action not in _CONFLICT_ACTIONS: + raise ParseError("Invalid UNIQUE ON CONFLICT action") + return action, index + 3 + + +def _unique_columns( + item: str, tokens: list[_Token], open_index: int +) -> tuple[tuple[UniqueColumn, ...], int]: + close = _matching_paren(tokens, open_index) + inner = item[tokens[open_index].end : tokens[close].start] + columns: list[UniqueColumn] = [] + for raw_column in _split_ranges(inner, _lex(inner)): + column_tokens = _meaningful(_lex(raw_column)) + if not column_tokens or column_tokens[0].kind not in ( + "word", + "identifier", + "string", + ): + raise ParseError("UNIQUE constraint has an invalid column") + name = _unquote(column_tokens[0].text) + collation = "" + order = "" + index = 1 + if index < len(column_tokens) and column_tokens[index].is_keyword("COLLATE"): + if index + 1 >= len(column_tokens): + raise ParseError("COLLATE in UNIQUE constraint is missing its name") + collation = _unquote(column_tokens[index + 1].text) + index += 2 + if index < len(column_tokens) and ( + column_tokens[index].is_keyword("ASC") + or column_tokens[index].is_keyword("DESC") + ): + order = column_tokens[index].text.upper() + index += 1 + if index != len(column_tokens): + raise ParseError("UNIQUE constraint has an invalid indexed column") + columns.append(UniqueColumn(name, collation=collation, order=order)) + if not columns: + raise ParseError("UNIQUE constraint must include at least one column") + return tuple(columns), close + 1 + + +def _column_uniques( + item: str, tokens: list[_Token], column: str, base_offset: int +) -> list[Unique]: + uniques: list[Unique] = [] + collation = "" + collation_index = 1 + while collation_index < len(tokens): + token = tokens[collation_index] + if token.text == "(": + collation_index = _matching_paren(tokens, collation_index) + 1 + continue + if token.is_keyword("COLLATE"): + if collation_index + 1 >= len(tokens): + raise ParseError("COLLATE is missing its name") + collation = _unquote(tokens[collation_index + 1].text) + collation_index += 2 + continue + collation_index += 1 + pending_name = "" + pending_start: int | None = None + index = 1 + while index < len(tokens): + token = tokens[index] + if token.text == "(": + index = _matching_paren(tokens, index) + 1 + continue + if token.is_keyword("CONSTRAINT"): + if index + 1 >= len(tokens): + raise ParseError("CONSTRAINT is missing its name") + pending_name = _unquote(tokens[index + 1].text) + pending_start = index + index += 2 + continue + if token.is_keyword("UNIQUE"): + source_start = tokens[ + pending_start if pending_start is not None else index + ].start + conflict, next_index = _conflict_after(tokens, index + 1) + source_end = tokens[next_index - 1].end + uniques.append( + Unique( + (UniqueColumn(column, collation=collation),), + name=pending_name, + column=column, + conflict=conflict, + sql=item[source_start:source_end], + start=base_offset + source_start, + end=base_offset + source_end, + ) + ) + pending_name = "" + pending_start = None + index = next_index + continue + if ( + token.kind == "word" + and token.text.upper() in _OTHER_COLUMN_CONSTRAINT_KEYWORDS + ): + pending_name = "" + pending_start = None + index += 1 + return uniques + + +def parse_uniques(create_sql: str) -> list[Unique]: + """Return column-level and table-level UNIQUE constraints.""" + body_info = _table_body(create_sql) + if body_info is None: + return [] + body, body_start = body_info + uniques: list[Unique] = [] + for item, item_start, _ in _split_spans(body, _lex(body)): + item_tokens = _meaningful(_lex(item)) + if not item_tokens: + continue + item_index = 0 + constraint_name = "" + if item_tokens[item_index].is_keyword("CONSTRAINT"): + if len(item_tokens) < 2: + raise ParseError("CONSTRAINT is missing its name") + constraint_name = _unquote(item_tokens[1].text) + item_index = 2 + head = item_tokens[item_index] if item_index < len(item_tokens) else None + if head and head.is_keyword("UNIQUE"): + if ( + item_index + 1 >= len(item_tokens) + or item_tokens[item_index + 1].text != "(" + ): + raise ParseError("Table UNIQUE must be followed by a column list") + columns, next_index = _unique_columns(item, item_tokens, item_index + 1) + conflict, next_index = _conflict_after(item_tokens, next_index) + if next_index != len(item_tokens): + raise ParseError("Unexpected SQL after UNIQUE constraint") + source_start = item_tokens[0].start + source_end = item_tokens[next_index - 1].end + uniques.append( + Unique( + columns, + name=constraint_name, + conflict=conflict, + sql=item[source_start:source_end], + start=body_start + item_start + source_start, + end=body_start + item_start + source_end, + ) + ) + continue + if ( + head + and head.kind == "word" + and head.text.upper() in _TABLE_CONSTRAINT_KEYWORDS + ): + continue + column = _unquote(item_tokens[0].text) + uniques.extend( + _column_uniques( + item, + item_tokens, + column, + body_start + item_start, + ) + ) + return uniques + + def parse_column_comments(create_sql: str) -> dict[str, ColumnComments]: """Return comments immediately before and after each column definition.""" body_info = _table_body(create_sql) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 48c5d5d..48987a6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -32,10 +32,13 @@ from .create_table_parser import ( Check, ColumnComments, ParseError, + Unique, + UniqueColumn, check_references_identifier, parse_autoincrement, parse_checks, parse_column_comments, + parse_uniques, rewrite_check_expression, sql_ends_in_line_comment, ) @@ -104,6 +107,24 @@ def _check_constraint_sql(check: Check) -> str: return f"{prefix}CHECK ({check.check}{newline})" +def _unique_constraint_sql(unique: Unique) -> str: + prefix = f"CONSTRAINT {quote_identifier(unique.name)} " if unique.name else "" + if unique.column: + constraint = "UNIQUE" + else: + columns = [] + for column in unique.columns: + column_sql = quote_identifier(column.name) + if column.collation: + column_sql += f" COLLATE {quote_identifier(column.collation)}" + if column.order: + column_sql += f" {column.order}" + columns.append(column_sql) + constraint = "UNIQUE ({})".format(", ".join(columns)) + conflict = f" ON CONFLICT {unique.conflict}" if unique.conflict else "" + return f"{prefix}{constraint}{conflict}" + + def _column_definition_with_comments( definition: str, comments: ColumnComments | None ) -> str: @@ -1424,6 +1445,7 @@ class Database: _checks: Iterable[Check] | None = None, _column_comments: Mapping[str, ColumnComments] | None = None, _autoincrement: str | None = None, + _uniques: Iterable[Unique] | None = None, ) -> str: """ Returns the SQL ``CREATE TABLE`` statement for creating the specified table. @@ -1486,6 +1508,60 @@ class Database: checks_by_column.setdefault(column, []).append(check) else: table_checks.append(check) + uniques_by_column: dict[str, list[Unique]] = {} + table_uniques: list[Unique] = [] + for unique in _uniques or (): + resolved_unique = Unique( + tuple( + UniqueColumn( + resolve_casing(column.name, columns), + collation=column.collation, + order=column.order, + ) + for column in unique.columns + ), + name=unique.name, + column=( + resolve_casing(unique.column, columns) if unique.column else "" + ), + conflict=unique.conflict, + ) + missing = [ + column.name + for column in resolved_unique.columns + if column.name not in columns + ] + if missing: + raise AlterError( + "No such column for UNIQUE constraint: {}".format( + ", ".join(missing) + ) + ) + if resolved_unique.column: + if ( + len(resolved_unique.columns) != 1 + or resolved_unique.columns[0].name != resolved_unique.column + ): + raise AlterError("Invalid column-level UNIQUE constraint") + if any( + column.collation or column.order + for column in resolved_unique.columns + ): + # Render this as a table constraint so the collation or sort + # order that governs uniqueness can be represented explicitly. + table_uniques.append( + Unique( + resolved_unique.columns, + name=resolved_unique.name, + conflict=resolved_unique.conflict, + ) + ) + else: + uniques_by_column.setdefault(resolved_unique.column, []).append( + resolved_unique + ) + else: + table_uniques.append(resolved_unique) if not columns: raise ValueError("Tables must have at least one column") if not all(n in columns for n in not_null): @@ -1554,6 +1630,10 @@ class Database: column_extras.append( f"REFERENCES {quote_identifier(fk.other_table)}({quote_identifier(cast(str, fk.other_column))}){_fk_actions_sql(fk)}" ) + column_extras.extend( + _unique_constraint_sql(unique) + for unique in uniques_by_column.get(column_name, ()) + ) column_extras.extend( _check_constraint_sql(check) for check in checks_by_column.get(column_name, ()) @@ -1600,6 +1680,9 @@ class Database: actions=_fk_actions_sql(fk), ) ) + column_defs.extend( + f" {_unique_constraint_sql(unique)}" for unique in table_uniques + ) column_defs.extend( f" {_check_constraint_sql(check)}" for check in table_checks ) @@ -2763,6 +2846,7 @@ class Table(Queryable): existing_checks = self.checks existing_column_comments = parse_column_comments(self.schema) existing_autoincrement = parse_autoincrement(self.schema) + existing_uniques = parse_uniques(self.schema) except ParseError as ex: raise TransformError( f"Could not parse table schema for table {self.name!r}: {ex}" @@ -2789,6 +2873,37 @@ class Table(Queryable): ) ) + create_table_uniques: list[Unique] = [] + for unique in existing_uniques: + columns = tuple( + UniqueColumn( + resolve_casing(column.name, existing_columns), + collation=column.collation, + order=column.order, + ) + for column in unique.columns + ) + if any(column.name in drop for column in columns): + continue + owner = ( + resolve_casing(unique.column, existing_columns) if unique.column else "" + ) + create_table_uniques.append( + Unique( + tuple( + UniqueColumn( + rename.get(column.name) or column.name, + collation=column.collation, + order=column.order, + ) + for column in columns + ), + name=unique.name, + column=rename.get(owner) or owner, + conflict=unique.conflict, + ) + ) + create_table_column_comments: dict[str, ColumnComments] = {} for column, comments in existing_column_comments.items(): owner = resolve_casing(column, existing_columns) @@ -2974,6 +3089,7 @@ class Table(Queryable): _checks=create_table_checks, _column_comments=create_table_column_comments, _autoincrement=create_table_autoincrement, + _uniques=create_table_uniques, ).strip() ) @@ -3008,6 +3124,9 @@ class Table(Queryable): {"index_name": index.name}, ).fetchall()[0][0] if index_sql is None: + if index.origin == "u": + # UNIQUE constraints are reproduced in CREATE TABLE above. + continue raise TransformError( f"Index '{index.name}' on table '{self.name}' does not have a " "CREATE INDEX statement. You must manually drop this index prior to running this " diff --git a/tests/test_create_table_parser.py b/tests/test_create_table_parser.py index 54bf221..74a089c 100644 --- a/tests/test_create_table_parser.py +++ b/tests/test_create_table_parser.py @@ -8,9 +8,12 @@ from sqlite_utils.create_table_parser import ( Check, ColumnComments, ParseError, + Unique, + UniqueColumn, parse_autoincrement, parse_checks, parse_column_comments, + parse_uniques, ) @@ -148,6 +151,51 @@ def test_parse_autoincrement(sql, expected): assert parse_autoincrement(sql) == expected +def test_parse_column_and_table_uniques(): + sql = """ + CREATE TABLE memberships ( + email TEXT COLLATE RTRIM CONSTRAINT unique_email UNIQUE ON CONFLICT IGNORE, + account_id INTEGER, + CONSTRAINT unique_membership UNIQUE ( + account_id DESC, + email COLLATE NOCASE ASC + ) ON CONFLICT REPLACE + ) + """ + sqlite3.connect(":memory:").execute(sql) + assert parse_uniques(sql) == [ + Unique( + (UniqueColumn("email", collation="RTRIM"),), + name="unique_email", + column="email", + conflict="IGNORE", + ), + Unique( + ( + UniqueColumn("account_id", order="DESC"), + UniqueColumn("email", collation="NOCASE", order="ASC"), + ), + name="unique_membership", + conflict="REPLACE", + ), + ] + uniques = parse_uniques(sql) + assert uniques[0].sql == "CONSTRAINT unique_email UNIQUE ON CONFLICT IGNORE" + assert sql[uniques[1].start : uniques[1].end] == uniques[1].sql + + +def test_unique_like_text_in_comments_and_checks_is_ignored(): + sql = """ + CREATE TABLE t ( + value TEXT /* UNIQUE ON CONFLICT REPLACE */ + CHECK(value != 'UNIQUE(other)'), + other TEXT + ) + """ + sqlite3.connect(":memory:").execute(sql) + assert parse_uniques(sql) == [] + + comment_or_space = st.sampled_from( [ " ", diff --git a/tests/test_transform.py b/tests/test_transform.py index e6096cd..3be6c6f 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -1033,24 +1033,78 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db): fresh_db.execute(""" CREATE TABLE dogs ( id INTEGER PRIMARY KEY, - name TEXT UNIQUE, + name TEXT UNIQUE ON CONFLICT IGNORE, age INTEGER ); """) dogs.insert({"id": 1, "name": "Cleo", "age": 5}) - # Attempt to transform the table without modifying 'name' - with pytest.raises(TransformError) as excinfo: - dogs.transform(types={"age": str}) + dogs.transform(types={"age": str}, rename={"name": "dog_name"}) + + assert 'dog_name" TEXT UNIQUE ON CONFLICT IGNORE' in dogs.schema + dogs.insert({"id": 2, "dog_name": "Cleo", "age": "6"}) + assert list(dogs.rows) == [{"id": 1, "dog_name": "Cleo", "age": "5"}] + + +def test_transform_preserves_composite_unique_constraint(fresh_db): + fresh_db.execute(""" + CREATE TABLE memberships ( + account_id INTEGER, + email TEXT, + note TEXT, + CONSTRAINT unique_membership + UNIQUE (account_id DESC, email COLLATE NOCASE) + ON CONFLICT ABORT + ) + """) + memberships = fresh_db.table("memberships") + memberships.insert({"account_id": 1, "email": "one@example.com", "note": "x"}) + + memberships.transform(rename={"account_id": "organization_id"}, types={"note": str}) assert ( - "Index 'sqlite_autoindex_dogs_1' on table 'dogs' does not have a CREATE INDEX statement." - in str(excinfo.value) - ) - assert ( - "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation." - in str(excinfo.value) + 'CONSTRAINT "unique_membership" UNIQUE ' + '("organization_id" DESC, "email" COLLATE "NOCASE") ON CONFLICT ABORT' + in memberships.schema ) + with pytest.raises(sqlite3.IntegrityError): + memberships.insert( + {"organization_id": 1, "email": "ONE@example.com", "note": "y"} + ) + + +def test_transform_preserves_column_unique_collation(fresh_db): + fresh_db.execute(""" + CREATE TABLE people ( + id INTEGER PRIMARY KEY, + name TEXT COLLATE NOCASE UNIQUE + ) + """) + people = fresh_db.table("people") + people.insert({"id": 1, "name": "Cleo"}) + + people.transform(rename={"name": "full_name"}) + + assert 'UNIQUE ("full_name" COLLATE "NOCASE")' in people.schema + with pytest.raises(sqlite3.IntegrityError): + people.insert({"id": 2, "full_name": "cleo"}) + + +def test_transform_drops_entire_composite_unique_constraint(fresh_db): + fresh_db.execute(""" + CREATE TABLE memberships ( + account_id INTEGER, + email TEXT, + UNIQUE (account_id, email) + ) + """) + memberships = fresh_db.table("memberships") + memberships.insert({"account_id": 1, "email": "one@example.com"}) + + memberships.transform(drop={"email"}) + + assert "UNIQUE" not in memberships.schema + memberships.insert({"account_id": 1}) def test_transform_preserves_autoincrement_and_sequence(fresh_db): From e4935e064407bc995f77795c025c33cef52d742e Mon Sep 17 00:00:00 2001 From: ikatyal2110 <134458944+ikatyal2110@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:56:51 -0500 Subject: [PATCH 106/110] transform: coerce empty strings to NULL when converting TEXT columns to numeric types (#805) * transform: coerce empty strings to NULL when converting TEXT columns to numeric types When a TEXT column is transformed to INTEGER, FLOAT, or REAL and a row contains an empty string, the empty string is now converted to NULL during the INSERT...SELECT copy, matching the expected behavior described in #488. Fixes #488 --- docs/changelog.rst | 1 + docs/cli.rst | 2 +- docs/python-api.rst | 2 ++ sqlite_utils/db.py | 21 ++++++++++++++++++++- tests/test_transform.py | 36 +++++++++++++++++++++++++++++++++--- 5 files changed, 57 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fe215fd..b3203ad 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ Unreleased - ``sqlite-utils convert --dry-run`` now works for table and column names containing closing square brackets. (:issue:`829`) - ``table.indexes`` and ``table.xindexes`` now work for table, index and column names containing double quotes. This also fixes ``table.transform()`` for tables with those identifiers. Thanks, `nyxst4ck `__. (:issue:`824`, `#825 `__) - Improved type annotations throughout the package and added Pyright regression checks to CI. (:issue:`833`) +- Changing a ``TEXT`` column to ``INTEGER``, ``FLOAT`` or ``REAL`` using ``table.transform()`` or ``sqlite-utils transform`` now converts exact empty strings to ``NULL``. Previously they remained empty strings in the numeric column. Thanks, `ikatyal2110 `__. (:issue:`488`, `#805 `__) .. _v3_39_1: diff --git a/docs/cli.rst b/docs/cli.rst index 417911a..78c33b8 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2236,7 +2236,7 @@ The ``transform`` command allows you to apply complex transformations to a table Every option for this table (with the exception of ``--pk-none``) can be specified multiple times. The options are as follows: ``--type column-name new-type`` - Change the type of the specified column. Valid types are ``integer``, ``text``, ``float``, ``real``, ``blob`` and ``any``. + Change the type of the specified column. Valid types are ``integer``, ``text``, ``float``, ``real``, ``blob`` and ``any``. Changing a ``TEXT`` column to ``INTEGER``, ``FLOAT`` or ``REAL`` converts exact empty-string values to ``NULL``. ``--drop column-name`` Drop the specified column. diff --git a/docs/python-api.rst b/docs/python-api.rst index 88cc3e3..d515642 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1826,6 +1826,8 @@ To alter the type of a column, use the ``types=`` argument: # Convert the 'age' column to an integer, and 'weight' to a float table.transform(types={"age": int, "weight": float}) +When a ``TEXT`` column is changed to ``INTEGER``, ``FLOAT`` or ``REAL``, exact empty-string values are stored as ``NULL``. Other values, including whitespace-only strings, are copied normally. + See :ref:`python_api_add_column` for a list of available types. .. _python_api_transform_strict: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 48987a6..37825bb 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3093,6 +3093,19 @@ class Table(Queryable): ).strip() ) + # Columns being changed from TEXT to a numeric type: coerce empty strings to NULL + _numeric_sql_types = {"INTEGER", "REAL", "FLOAT", "NUMERIC"} + text_to_numeric_cols = { + col_name + for col_name, new_type in types.items() + if existing_columns.get(col_name) == str + and COLUMN_TYPE_MAPPING.get( + new_type, + new_type.upper() if isinstance(new_type, str) else "", + ) + in _numeric_sql_types + } + # Copy across data, respecting any renamed columns new_cols = [] old_cols = [] @@ -3103,10 +3116,16 @@ class Table(Queryable): if "rowid" not in new_cols: new_cols.insert(0, "rowid") old_cols.insert(0, "rowid") + + def _copy_expr(col): + if col in text_to_numeric_cols: + return "NULLIF({}, '')".format(quote_identifier(col)) + return quote_identifier(col) + copy_sql = "INSERT INTO {} ({new_cols})\n SELECT {old_cols} FROM {};".format( quote_identifier(new_table_name), quote_identifier(self.name), - old_cols=", ".join(quote_identifier(col) for col in old_cols), + old_cols=", ".join(_copy_expr(col) for col in old_cols), new_cols=", ".join(quote_identifier(col) for col in new_cols), ) sqls.append(copy_sql) diff --git a/tests/test_transform.py b/tests/test_transform.py index 3be6c6f..8738713 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -27,7 +27,7 @@ from sqlite_utils.utils import OperationalError {"types": {"age": int}}, [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER\n);', - 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";', 'DROP TABLE "dogs";', "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', @@ -63,7 +63,7 @@ from sqlite_utils.utils import OperationalError {"types": {"age": int}, "rename": {"age": "dog_age"}}, [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" INTEGER\n);', - 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";', 'DROP TABLE "dogs";', "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', @@ -168,7 +168,7 @@ def test_transform_sql_table_with_primary_key( {"types": {"age": int}}, [ 'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" INTEGER\n);', - 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";', + 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";', 'DROP TABLE "dogs";', "PRAGMA legacy_alter_table=ON;", 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";', @@ -1125,6 +1125,36 @@ def test_transform_preserves_autoincrement_and_sequence(fresh_db): ] +@pytest.mark.parametrize( + "new_type,expected_value,expected_type", + [ + (int, 42, int), + (float, 42.0, float), + ("integer", 42, int), + ("float", 42.0, float), + ("REAL", 42.0, float), + ], +) +def test_transform_empty_string_to_null_for_numeric_types( + fresh_db, new_type, expected_value, expected_type +): + fresh_db["test"].insert_all( + [ + {"id": 1, "value": "42"}, + {"id": 2, "value": ""}, + {"id": 3, "value": None}, + {"id": 4, "value": " "}, + ] + ) + fresh_db["test"].transform(types={"value": new_type}) + rows = {r["id"]: r["value"] for r in fresh_db["test"].rows} + assert rows[1] == expected_value + assert type(rows[1]) is expected_type + assert rows[2] is None + assert rows[3] is None + assert rows[4] == " " + + def test_transform_preserves_view(fresh_db): # https://github.com/simonw/sqlite-utils/issues/831 dogs = fresh_db.table("dogs") From 1d98613f28b8edab5fd0deb5ba65d54fb286e7ba Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Aug 2026 13:09:42 -0700 Subject: [PATCH 107/110] Release 4.2 Refs #488, #602, #762, #790, #805, #808, #811, #816, #821, #822, #824, #825, #828, #829, #831, #833, #834, #836, #837 --- docs/changelog.rst | 22 +++++++++++++--------- pyproject.toml | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b3203ad..31bd961 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,19 +4,13 @@ Changelog =========== -.. _unreleased: +.. _v4_2: -Unreleased ----------- +4.2 (2026-08-13) +---------------- -- ``table.transform()`` now preserves column-level and composite ``UNIQUE`` constraints, including constraint names, collations, sort order and ``ON CONFLICT`` behavior. Renaming columns updates those constraints, while dropping any constituent column removes the entire constraint. (:issue:`762`) -- ``table.transform()`` now preserves ``AUTOINCREMENT`` primary keys and their sequence high-water marks. Previously a transform removed ``AUTOINCREMENT`` and could reuse deleted row IDs. (:issue:`602`) - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) - New ``sqlite_utils.ANY`` marker type for creating and introspecting SQLite ``ANY`` columns. The Python API and CLI can create, add and transform these columns, and ``table.transform()`` and ``table.extract()`` now preserve ``ANY`` columns and their values in ``STRICT`` tables. (:issue:`790`) -- ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) -- ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) -- ``table.transform(rename=...)`` now preserves explicit indexes on renamed columns by dropping and recreating those indexes against the new column names. Previously this raised a ``TransformError``. (:issue:`822`) -- ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) - ``table.default_values`` now unescapes doubled single quotes in string defaults, so a default such as ``'O''Brien'`` is returned as ``"O'Brien"``. Thanks, `ikatyal2110 `__. (`#811 `__) - ``table.default_values`` now decodes unquoted ``TRUE``, ``FALSE`` and ``NULL`` default literals as ``True``, ``False`` and ``None`` respectively. (:issue:`836`) - ``table.enable_fts(..., tokenize=...)`` and ``sqlite-utils enable-fts --tokenize`` now safely quote the tokenizer argument, preventing a crafted value from injecting additional SQL. Thanks, `Bunlong Heng `__. (`#828 `__) @@ -27,6 +21,16 @@ Unreleased - Improved type annotations throughout the package and added Pyright regression checks to CI. (:issue:`833`) - Changing a ``TEXT`` column to ``INTEGER``, ``FLOAT`` or ``REAL`` using ``table.transform()`` or ``sqlite-utils transform`` now converts exact empty strings to ``NULL``. Previously they remained empty strings in the numeric column. Thanks, `ikatyal2110 `__. (:issue:`488`, `#805 `__) +``table.transform()`` can handle many more edge-cases: + +- ``table.transform()`` now preserves column-level and composite ``UNIQUE`` constraints, including constraint names, collations, sort order and ``ON CONFLICT`` behavior. Renaming columns updates those constraints, while dropping any constituent column removes the entire constraint. (:issue:`762`) +- ``table.transform()`` now preserves ``AUTOINCREMENT`` primary keys and their sequence high-water marks. Previously a transform removed ``AUTOINCREMENT`` and could reuse deleted row IDs. (:issue:`602`) +- ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) +- ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) +- ``table.transform(rename=...)`` now preserves explicit indexes on renamed columns by dropping and recreating those indexes against the new column names. Previously this raised a ``TransformError``. (:issue:`822`) +- ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`) + + .. _v3_39_1: 3.39.1 (2026-07-25) diff --git a/pyproject.toml b/pyproject.toml index 9b4d6f5..6dac11c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.1.1" +version = "4.2" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From f6d73112c8368cd6eb2ac596966e8148747c7b4e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Aug 2026 16:52:03 -0700 Subject: [PATCH 108/110] Fix for sqlite-utils 4.2 crashing bug (#843) - Remove from typing_extensions import Self - Smoke test: uv run --no-default-groups sqlite-utils --help Closes #842 --- .github/workflows/test.yml | 5 +++++ sqlite_utils/db.py | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6c720a1..5924fd8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,11 @@ jobs: run: | pip install uv uv run ty check sqlite_utils + - name: Check no accidental dev= dependencies needed + if: matrix.os == 'ubuntu-latest' + run: | + pip install uv + uv run --no-default-groups sqlite-utils --help - name: Check formatting run: black . --check - name: Check if cog needs to be run diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 37825bb..c011d9b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -24,8 +24,6 @@ from typing import ( ) from sqlite_fts4 import rank_bm25 -from typing_extensions import Self - from sqlite_utils.plugins import ensure_plugins_loaded, pm from .create_table_parser import ( @@ -637,7 +635,7 @@ class Database: pm.hook.prepare_connection(conn=self.conn) self.strict = strict - def __enter__(self) -> Self: + def __enter__(self): return self def __exit__( From 28dc6278cc03a9245325d056e6986818544abc68 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Aug 2026 16:52:30 -0700 Subject: [PATCH 109/110] Release 4.2.1 Refs #842, #843 --- docs/changelog.rst | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 31bd961..5d024e1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _v4_2_1: + +4.2.1 (2026-08-13) +------------------ + +- Fix for ``No module named 'typing_extensions'`` crashing bug accidentally shipped in version 4.2. (:issue:`842`) + .. _v4_2: 4.2 (2026-08-13) diff --git a/pyproject.toml b/pyproject.toml index 6dac11c..92650e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.2" +version = "4.2.1" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 56dd09702fdb9e899f577ffd51693c1f2176cb08 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 13 Aug 2026 17:01:47 -0700 Subject: [PATCH 110/110] Run no-default-groups smoke test from Justfile Refs #842 I had to add --isolated because otherwise this test would pass if a .venv folder already existed with the dev dependencies installed in it. --- Justfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index e93075f..7347534 100644 --- a/Justfile +++ b/Justfile @@ -2,9 +2,12 @@ @default: test lint # Run pytest with supplied options -@test *options: +@test *options: test-no-dev-dependencies uv run pytest {{options}} +@test-no-dev-dependencies: + uv run --isolated --no-default-groups sqlite-utils --help > /dev/null + @run *options: uv run -- {{options}}