From 8f0c06e1889513ed0f01cb57783ddf07c442d4be Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 17 May 2026 16:52:48 -0700 Subject: [PATCH 001/120] Test against Python 3.15-dev, bump ty and Black (#738) * Add Python 3.15-dev to test matrix * Run ty check only on 3.14 * Bump Black version * Update tabulate and use that in * Bump to latest ty --- .github/workflows/test.yml | 4 +- docs/cli-reference.rst | 139 ++++++++++++++++++----------------- docs/cli.rst | 1 + pyproject.toml | 6 +- sqlite_utils/cli.py | 26 +++---- sqlite_utils/db.py | 104 +++++++++++--------------- sqlite_utils/utils.py | 34 +++++---- tests/conftest.py | 6 +- tests/test_analyze_tables.py | 9 +-- tests/test_cli.py | 21 ++---- tests/test_cli_convert.py | 6 +- tests/test_create.py | 1 - tests/test_default_value.py | 1 - tests/test_duplicate.py | 6 +- tests/test_extract.py | 9 +-- tests/test_introspect.py | 12 +-- tests/test_transform.py | 6 +- 17 files changed, 175 insertions(+), 216 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6eb7e38..0f65d6a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15-dev"] numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest, macos-14] steps: @@ -46,7 +46,7 @@ jobs: - name: run flake8 run: flake8 - name: run ty - if: matrix.os != 'windows-latest' + if: matrix.os != 'windows-latest' && matrix.python-version == '3.14' run: | pip install uv uv run ty check sqlite_utils diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 6231dbf..c3bf7a0 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -117,15 +117,15 @@ See :ref:`cli_query`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of asciidoc, double_grid, - double_outline, fancy_grid, fancy_outline, github, - grid, heavy_grid, heavy_outline, html, jira, - latex, latex_booktabs, latex_longtable, latex_raw, - mediawiki, mixed_grid, mixed_outline, moinmoin, - orgtbl, outline, pipe, plain, presto, pretty, - psql, rounded_grid, rounded_outline, rst, simple, - simple_grid, simple_outline, textile, tsv, - unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, + fancy_outline, github, grid, heavy_grid, + heavy_outline, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, mixed_grid, + mixed_outline, moinmoin, orgtbl, outline, pipe, + plain, presto, pretty, psql, rounded_grid, + rounded_outline, rst, simple, simple_grid, + simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings -r, --raw Raw output, first column of first row @@ -186,15 +186,15 @@ See :ref:`cli_memory`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of asciidoc, double_grid, - double_outline, fancy_grid, fancy_outline, github, - grid, heavy_grid, heavy_outline, html, jira, - latex, latex_booktabs, latex_longtable, latex_raw, - mediawiki, mixed_grid, mixed_outline, moinmoin, - orgtbl, outline, pipe, plain, presto, pretty, - psql, rounded_grid, rounded_outline, rst, simple, - simple_grid, simple_outline, textile, tsv, - unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, + fancy_outline, github, grid, heavy_grid, + heavy_outline, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, mixed_grid, + mixed_outline, moinmoin, orgtbl, outline, pipe, + plain, presto, pretty, psql, rounded_grid, + rounded_outline, rst, simple, simple_grid, + simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings -r, --raw Raw output, first column of first row @@ -425,14 +425,15 @@ See :ref:`cli_search`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of asciidoc, double_grid, - double_outline, fancy_grid, fancy_outline, github, - grid, heavy_grid, heavy_outline, html, jira, latex, - latex_booktabs, latex_longtable, latex_raw, mediawiki, - mixed_grid, mixed_outline, moinmoin, orgtbl, outline, - pipe, plain, presto, pretty, psql, rounded_grid, - rounded_outline, rst, simple, simple_grid, - simple_outline, textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, fancy_outline, + github, grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl, + outline, pipe, plain, presto, pretty, psql, + rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, unsafehtml, + youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -690,14 +691,15 @@ See :ref:`cli_tables`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of asciidoc, double_grid, - double_outline, fancy_grid, fancy_outline, github, - grid, heavy_grid, heavy_outline, html, jira, latex, - latex_booktabs, latex_longtable, latex_raw, mediawiki, - mixed_grid, mixed_outline, moinmoin, orgtbl, outline, - pipe, plain, presto, pretty, psql, rounded_grid, - rounded_outline, rst, simple, simple_grid, - simple_outline, textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, fancy_outline, + github, grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl, + outline, pipe, plain, presto, pretty, psql, + rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, unsafehtml, + youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --columns Include list of columns for each table @@ -731,14 +733,15 @@ See :ref:`cli_views`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of asciidoc, double_grid, - double_outline, fancy_grid, fancy_outline, github, - grid, heavy_grid, heavy_outline, html, jira, latex, - latex_booktabs, latex_longtable, latex_raw, mediawiki, - mixed_grid, mixed_outline, moinmoin, orgtbl, outline, - pipe, plain, presto, pretty, psql, rounded_grid, - rounded_outline, rst, simple, simple_grid, - simple_outline, textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, fancy_outline, + github, grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl, + outline, pipe, plain, presto, pretty, psql, + rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, unsafehtml, + youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --columns Include list of columns for each view @@ -777,15 +780,15 @@ See :ref:`cli_rows`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of asciidoc, double_grid, - double_outline, fancy_grid, fancy_outline, github, - grid, heavy_grid, heavy_outline, html, jira, - latex, latex_booktabs, latex_longtable, latex_raw, - mediawiki, mixed_grid, mixed_outline, moinmoin, - orgtbl, outline, pipe, plain, presto, pretty, - psql, rounded_grid, rounded_outline, rst, simple, - simple_grid, simple_outline, textile, tsv, - unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, + fancy_outline, github, grid, heavy_grid, + heavy_outline, html, jira, latex, latex_booktabs, + latex_longtable, latex_raw, mediawiki, mixed_grid, + mixed_outline, moinmoin, orgtbl, outline, pipe, + plain, presto, pretty, psql, rounded_grid, + rounded_outline, rst, simple, simple_grid, + simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --load-extension TEXT Path to SQLite extension, with optional @@ -817,14 +820,15 @@ See :ref:`cli_triggers`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of asciidoc, double_grid, - double_outline, fancy_grid, fancy_outline, github, - grid, heavy_grid, heavy_outline, html, jira, latex, - latex_booktabs, latex_longtable, latex_raw, mediawiki, - mixed_grid, mixed_outline, moinmoin, orgtbl, outline, - pipe, plain, presto, pretty, psql, rounded_grid, - rounded_outline, rst, simple, simple_grid, - simple_outline, textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, fancy_outline, + github, grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl, + outline, pipe, plain, presto, pretty, psql, + rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, unsafehtml, + youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -856,14 +860,15 @@ See :ref:`cli_indexes`. --tsv Output TSV --no-headers Omit CSV headers -t, --table Output as a formatted table - --fmt TEXT Table format - one of asciidoc, double_grid, - double_outline, fancy_grid, fancy_outline, github, - grid, heavy_grid, heavy_outline, html, jira, latex, - latex_booktabs, latex_longtable, latex_raw, mediawiki, - mixed_grid, mixed_outline, moinmoin, orgtbl, outline, - pipe, plain, presto, pretty, psql, rounded_grid, - rounded_outline, rst, simple, simple_grid, - simple_outline, textile, tsv, unsafehtml, youtrack + --fmt TEXT Table format - one of asciidoc, colon_grid, + double_grid, double_outline, fancy_grid, fancy_outline, + github, grid, heavy_grid, heavy_outline, html, jira, + latex, latex_booktabs, latex_longtable, latex_raw, + mediawiki, mixed_grid, mixed_outline, moinmoin, orgtbl, + outline, pipe, plain, presto, pretty, psql, + rounded_grid, rounded_outline, rst, simple, + simple_grid, simple_outline, textile, tsv, unsafehtml, + youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings --load-extension TEXT Path to SQLite extension, with optional :entrypoint diff --git a/docs/cli.rst b/docs/cli.rst index a608160..415e4c2 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -257,6 +257,7 @@ Available ``--fmt`` options are: .. ]]] - ``asciidoc`` +- ``colon_grid`` - ``double_grid`` - ``double_outline`` - ``fancy_grid`` diff --git a/pyproject.toml b/pyproject.toml index a50a3a8..5bfa492 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ [dependency-groups] dev = [ - "black>=24.1.1", + "black>=26.3.1", "cogapp", "hypothesis", "pytest", @@ -47,7 +47,9 @@ dev = [ # flake8 "flake8", "flake8-pyproject", - "ty", + "ty>=0.0.37", + # For stable cog: + "tabulate>=0.10.0", ] docs = [ "beanbag-docutils>=2.0", diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 9b9ee20..5844dfc 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,7 +1,7 @@ import base64 from typing import Any import click -from click_default_group import DefaultGroup # type: ignore +from click_default_group import DefaultGroup from datetime import datetime, timezone import hashlib import pathlib @@ -42,7 +42,6 @@ from .utils import ( TypeTracker, ) - CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @@ -223,7 +222,7 @@ def tables( else: items = db.table_names(fts4=fts4, fts5=fts5) for name in items: - row = [name] + row: list[Any] = [name] if counts: row.append(method(name).count) if columns: @@ -2122,8 +2121,9 @@ def _execute_query( cursor = [[cursor.rowcount]] else: headers = [c[0] for c in cursor.description] + cursor_or_rows: Any = cursor if raw: - row = cursor.fetchone() # type: ignore[union-attr] + row = cursor_or_rows.fetchone() data = row[0] if row else None if isinstance(data, bytes): sys.stdout.buffer.write(data) @@ -2911,8 +2911,7 @@ def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least ) details = ( ( - textwrap.dedent( - """ + textwrap.dedent(""" {table}.{column}: ({i}/{total}) Total rows: {total_rows} @@ -2920,8 +2919,7 @@ def _analyze(db, tables, columns, save, common_limit=10, no_most=False, no_least Blank rows: {num_blank} Distinct values: {num_distinct}{most_common_rendered}{least_common_rendered} - """ - ) + """) .strip() .format( i=i + 1, @@ -2968,8 +2966,7 @@ def uninstall(packages, yes): def _generate_convert_help(): - help = textwrap.dedent( - """ + help = textwrap.dedent(""" Convert columns using Python code you supply. For example: \b @@ -2982,8 +2979,7 @@ def _generate_convert_help(): Use "-" for CODE to read Python code from standard input. The following common operations are available as recipe functions: - """ - ).strip() + """).strip() recipe_names = [ n for n in dir(recipes) @@ -2997,15 +2993,13 @@ def _generate_convert_help(): name, str(inspect.signature(fn)), textwrap.dedent(fn.__doc__.rstrip()) ) help += "\n\n" - help += textwrap.dedent( - """ + help += textwrap.dedent(""" You can use these recipes like so: \b sqlite-utils convert my.db mytable mycolumn \\ 'r.jsonsplit(value, delimiter=":")' - """ - ).strip() + """).strip() return help diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index aacdc89..ed3fc7a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -15,6 +15,7 @@ from collections.abc import Mapping import contextlib import datetime import decimal +import importlib import inspect import itertools import json @@ -22,7 +23,7 @@ import os import pathlib import re import secrets -from sqlite_fts4 import rank_bm25 # type: ignore +from sqlite_fts4 import rank_bm25 import textwrap from typing import ( cast, @@ -43,7 +44,7 @@ import uuid from sqlite_utils.plugins import pm try: - from sqlite_dump import iterdump # type: ignore[import-not-found] + iterdump = importlib.import_module("sqlite_dump").iterdump except ImportError: iterdump = None @@ -82,15 +83,17 @@ def quote_identifier(identifier: str) -> str: return '"{}"'.format(identifier.replace('"', '""')) +pd: Any = None try: - import pandas as pd # type: ignore + pd = importlib.import_module("pandas") except ImportError: - pd = None # type: ignore + pd = None +np: Any = None try: - import numpy as np # type: ignore + np = importlib.import_module("numpy") except ImportError: - np = None # type: ignore + np = None Column = namedtuple( "Column", ("cid", "name", "type", "notnull", "default_value", "is_pk") @@ -190,7 +193,10 @@ class Default: DEFAULT = Default() -COLUMN_TYPE_MAPPING = { +Tracer = Callable[[str, Optional[Union[Sequence[Any], Dict[str, Any]]]], None] + + +COLUMN_TYPE_MAPPING: Dict[Any, str] = { float: "REAL", int: "INTEGER", bool: "INTEGER", @@ -339,7 +345,7 @@ class Database: memory_name: Optional[str] = None, recreate: bool = False, recursive_triggers: bool = True, - tracer: Optional[Callable] = None, + tracer: Optional[Tracer] = None, use_counts_table: bool = False, execute_plugins: bool = True, use_old_upsert: bool = False, @@ -375,8 +381,8 @@ class Database: self.conn = sqlite3.connect(str(filename_or_conn)) else: assert not recreate, "recreate cannot be used with connections, only paths" - self.conn = filename_or_conn - self._tracer = tracer + self.conn = cast(sqlite3.Connection, filename_or_conn) + self._tracer: Optional[Tracer] = tracer if recursive_triggers: self.execute("PRAGMA recursive_triggers=on;") self._registered_functions: set = set() @@ -421,7 +427,7 @@ class Database: @contextlib.contextmanager def tracer( - self, tracer: Optional[Callable[[str, Optional[Sequence]], None]] = None + self, tracer: Optional[Tracer] = None ) -> Generator["Database", None, None]: """ Context manager to temporarily set a tracer function - all executed SQL queries will @@ -439,7 +445,7 @@ class Database: :param tracer: Callable accepting ``sql`` and ``parameters`` arguments """ prev_tracer = self._tracer - self._tracer = tracer or print + self._tracer = tracer or cast(Tracer, print) try: yield self finally: @@ -2275,12 +2281,10 @@ class Table(Queryable): "{}_{}".format(index_name, suffix) if suffix else index_name ) sql = ( - textwrap.dedent( - """ + textwrap.dedent(""" CREATE {unique}INDEX {if_not_exists}{index_name} ON {table_name} ({columns}); - """ - ) + """) .strip() .format( index_name=quote_identifier(created_index_name), @@ -2475,8 +2479,7 @@ class Table(Queryable): See :ref:`python_api_cached_table_counts` for details. """ sql = ( - textwrap.dedent( - """ + textwrap.dedent(""" {create_counts_table} CREATE TRIGGER IF NOT EXISTS {trigger_insert} AFTER INSERT ON {table} BEGIN @@ -2501,8 +2504,7 @@ class Table(Queryable): ); END; INSERT OR REPLACE INTO _counts VALUES ({table_quoted}, (select count(*) from {table})); - """ - ) + """) .strip() .format( create_counts_table=_COUNTS_TABLE_CREATE_SQL.format( @@ -2554,14 +2556,12 @@ class Table(Queryable): :param replace: Should any existing FTS index for this table be replaced by the new one? """ create_fts_sql = ( - textwrap.dedent( - """ + textwrap.dedent(""" CREATE VIRTUAL TABLE {table_fts} USING {fts_version} ( {columns},{tokenize} content={table} ) - """ - ) + """) .strip() .format( table=quote_identifier(self.name), @@ -2599,8 +2599,7 @@ class Table(Queryable): table = quote_identifier(self.name) table_fts = quote_identifier(self.name + "_fts") triggers = ( - textwrap.dedent( - """ + textwrap.dedent(""" CREATE TRIGGER {table_ai} AFTER INSERT ON {table} BEGIN INSERT INTO {table_fts} (rowid, {columns}) VALUES (new.rowid, {new_cols}); END; @@ -2611,8 +2610,7 @@ class Table(Queryable): INSERT INTO {table_fts} ({table_fts}, rowid, {columns}) VALUES('delete', old.rowid, {old_cols}); INSERT INTO {table_fts} (rowid, {columns}) VALUES (new.rowid, {new_cols}); END; - """ - ) + """) .strip() .format( table=table, @@ -2637,12 +2635,10 @@ class Table(Queryable): """ columns_quoted = ", ".join(quote_identifier(c) for c in columns) sql = ( - textwrap.dedent( - """ + textwrap.dedent(""" INSERT INTO {table_fts} (rowid, {columns}) SELECT rowid, {columns} FROM {table}; - """ - ) + """) .strip() .format( table=quote_identifier(self.name), @@ -2659,17 +2655,11 @@ class Table(Queryable): if fts_table: self.db[fts_table].drop() # Now delete the triggers that related to that table - sql = ( - textwrap.dedent( - """ + sql = textwrap.dedent(""" SELECT name FROM sqlite_master WHERE type = 'trigger' AND (sql LIKE '% INSERT INTO [{}]%' OR sql LIKE '% INSERT INTO "{}"%') - """ - ) - .strip() - .format(fts_table, fts_table) - ) + """).strip().format(fts_table, fts_table) trigger_names = [] for row in self.db.execute(sql).fetchall(): trigger_names.append(row[0]) @@ -2695,8 +2685,7 @@ class Table(Queryable): def detect_fts(self) -> Optional[str]: "Detect if table has a corresponding FTS virtual table and return it" - sql = textwrap.dedent( - """ + sql = textwrap.dedent(""" SELECT name FROM sqlite_master WHERE rootpage = 0 AND ( @@ -2707,8 +2696,7 @@ class Table(Queryable): AND sql LIKE '%VIRTUAL TABLE%USING FTS%' ) ) - """ - ).strip() + """).strip() args = { "like": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), "like2": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), @@ -2724,13 +2712,9 @@ class Table(Queryable): "Run the ``optimize`` operation against the associated full-text search index table." fts_table = self.detect_fts() if fts_table is not None: - self.db.execute( - """ + self.db.execute(""" INSERT INTO {table} ({table}) VALUES ("optimize"); - """.strip().format( - table=quote_identifier(fts_table) - ) - ) + """.strip().format(table=quote_identifier(fts_table))) return self def search_sql( @@ -2768,8 +2752,7 @@ class Table(Queryable): ) fts_table_quoted = quote_identifier(fts_table) virtual_table_using = self.db.table(fts_table).virtual_table_using - sql = textwrap.dedent( - """ + sql = textwrap.dedent(""" with {original} as ( select rowid, @@ -2786,8 +2769,7 @@ class Table(Queryable): order by {order_by} {limit_offset} - """ - ).strip() + """).strip() if virtual_table_using == "FTS5": rank_implementation = "{}.rank".format(fts_table_quoted) else: @@ -3517,7 +3499,7 @@ class Table(Queryable): raise ValueError( "When using list-based iteration, the first yielded value must be a list of column name strings" ) - column_names = list(first_record) + column_names = cast(List[str], list(first_record)) all_columns = column_names num_columns = len(column_names) # Get the actual first data record @@ -3559,7 +3541,8 @@ class Table(Queryable): chunk_as_dicts = [dict(zip(column_names, row)) for row in chunk] column_types = suggest_column_types(chunk_as_dicts) else: - column_types = suggest_column_types(chunk) # type: ignore[arg-type] + dict_chunk = cast(List[Dict[str, Any]], chunk) + column_types = suggest_column_types(dict_chunk) if extracts: for col in extracts: if col in column_types: @@ -3586,14 +3569,14 @@ class Table(Queryable): all_columns.insert(0, hash_id) else: all_columns_set: Set[str] = set() - for record in chunk: - all_columns_set.update(record.keys()) # type: ignore[union-attr] + for record in cast(List[Dict[str, Any]], chunk): + all_columns_set.update(record.keys()) all_columns = list(sorted(all_columns_set)) if hash_id: all_columns.insert(0, hash_id) else: if not list_mode: - for record in chunk: + for record in cast(List[Dict[str, Any]], chunk): all_columns += [ column for column in record if column not in all_columns ] @@ -3791,6 +3774,7 @@ class Table(Queryable): :param strict: Boolean, apply STRICT mode if creating the table. """ assert isinstance(lookup_values, dict) + assert pk is not None if extra_values is not None: assert isinstance(extra_values, dict) combined_values = dict(lookup_values) @@ -3810,7 +3794,7 @@ class Table(Queryable): ) ) try: - return rows[0][pk] # type: ignore[index] + return rows[0][pk] except IndexError: return self.insert( combined_values, diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 05e9a51..0ca98fc 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -3,6 +3,7 @@ import contextlib import csv import enum import hashlib +import importlib import io import itertools import json @@ -21,6 +22,7 @@ from typing import ( Set, Tuple, Type, + TYPE_CHECKING, TypeVar, Union, cast, @@ -30,22 +32,26 @@ import click from . import recipes -try: - import pysqlite3 as sqlite3 # type: ignore[import-not-found] # noqa: F401 - from pysqlite3 import dbapi2 # type: ignore[import-not-found] # noqa: F401 +if TYPE_CHECKING: + import sqlite3 # noqa: F401 + from sqlite3 import dbapi2 # noqa: F401 OperationalError = dbapi2.OperationalError -except ImportError: +else: try: - import sqlean as sqlite3 # type: ignore[import-not-found] # noqa: F401 - from sqlean import dbapi2 # type: ignore[import-not-found] # noqa: F401 - + sqlite3 = importlib.import_module("pysqlite3") + dbapi2 = importlib.import_module("pysqlite3.dbapi2") OperationalError = dbapi2.OperationalError except ImportError: - import sqlite3 # noqa: F401 - from sqlite3 import dbapi2 # noqa: F401 + try: + sqlite3 = importlib.import_module("sqlean") + dbapi2 = importlib.import_module("sqlean.dbapi2") + OperationalError = dbapi2.OperationalError + except ImportError: + import sqlite3 # noqa: F401 + from sqlite3 import dbapi2 # noqa: F401 - OperationalError = dbapi2.OperationalError + OperationalError = dbapi2.OperationalError SPATIALITE_PATHS = ( @@ -60,7 +66,7 @@ SPATIALITE_PATHS = ( ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit() # Type alias for row dictionaries - values can be various SQLite-compatible types -RowValue = Union[None, int, float, str, bytes, bool] +RowValue = Union[None, int, float, str, bytes, bool, List[str]] Row = Dict[str, RowValue] T = TypeVar("T") @@ -284,7 +290,7 @@ def _extra_key_strategy( else: extras_value = row.pop(None) row_out = cast(Row, row) - row_out[extras_key] = extras_value # type: ignore[assignment] + row_out[extras_key] = cast(RowValue, extras_value) yield row_out @@ -464,14 +470,14 @@ class ValueTracker: def test_integer(self, value: object) -> bool: try: - int(value) # type: ignore + int(cast(Any, value)) return True except (ValueError, TypeError): return False def test_float(self, value: object) -> bool: try: - float(value) # type: ignore[arg-type] + float(cast(Any, value)) return True except (ValueError, TypeError): return False diff --git a/tests/conftest.py b/tests/conftest.py index 3990d76..6fd35b8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,14 +42,12 @@ def fresh_db(): @pytest.fixture def existing_db(): database = Database(memory=True) - database.executescript( - """ + database.executescript(""" CREATE TABLE foo (text TEXT); INSERT INTO foo (text) values ("one"); INSERT INTO foo (text) values ("two"); INSERT INTO foo (text) values ("three"); - """ - ) + """) return database diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 4618eff..a2ce585 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -143,10 +143,7 @@ def db_to_analyze_path(db_to_analyze, tmpdir): def test_analyze_table(db_to_analyze_path): result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path]) - assert ( - result.output.strip() - == ( - """ + assert result.output.strip() == (""" stuff.id: (1/3) Total rows: 8 @@ -179,9 +176,7 @@ stuff.size: (3/3) Most common: 5: 5 - 3: 4""" - ).strip() - ) + 3: 4""").strip() def test_analyze_table_save(db_to_analyze_path): diff --git a/tests/test_cli.py b/tests/test_cli.py index 40c3595..40b3685 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -967,12 +967,9 @@ def test_query_json_with_json_cols(db_path): result = CliRunner().invoke( cli.cli, [db_path, "select id, name, friends from dogs"] ) - assert ( - r""" + assert r""" [{"id": 1, "name": "Cleo", "friends": "[{\"name\": \"Pancakes\"}, {\"name\": \"Bailey\"}]"}] - """.strip() - == result.output.strip() - ) + """.strip() == result.output.strip() # With --json-cols: result = CliRunner().invoke( cli.cli, [db_path, "select id, name, friends from dogs", "--json-cols"] @@ -1998,12 +1995,10 @@ def test_search_quote(tmpdir): def test_indexes(tmpdir): db_path = str(tmpdir / "test.db") db = Database(db_path) - db.conn.executescript( - """ + db.conn.executescript(""" create table Gosh (c1 text, c2 text, c3 text); create index Gosh_idx on Gosh(c2, c3 desc); - """ - ) + """) result = CliRunner().invoke( cli.cli, ["indexes", str(db_path)], @@ -2094,16 +2089,12 @@ def test_triggers(tmpdir, extra_args, expected): pk="id", ) db["counter"].insert({"count": 1}) - db.conn.execute( - textwrap.dedent( - """ + db.conn.execute(textwrap.dedent(""" CREATE TRIGGER blah AFTER INSERT ON articles BEGIN UPDATE counter SET count = count + 1; END - """ - ) - ) + """)) args = ["triggers", db_path] if extra_args: args.extend(extra_args) diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 387b318..443e72c 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -371,16 +371,14 @@ def test_convert_multi_complex_column_types(fresh_db_and_path): ], pk="id", ) - code = textwrap.dedent( - """ + code = textwrap.dedent(""" if value == 1: return {"is_str": "", "is_float": 1.2, "is_int": None} elif value == 2: return {"is_float": 1, "is_int": 12} elif value == 3: return {"is_bytes": b"blah"} - """ - ) + """) result = CliRunner().invoke( cli.cli, [ diff --git a/tests/test_create.py b/tests/test_create.py index ea09c45..b1a6ad1 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -20,7 +20,6 @@ import pathlib import pytest import uuid - try: import pandas as pd # type: ignore except ImportError: diff --git a/tests/test_default_value.py b/tests/test_default_value.py index 9ffdb14..c594c9f 100644 --- a/tests/test_default_value.py +++ b/tests/test_default_value.py @@ -1,6 +1,5 @@ import pytest - EXAMPLES = [ ("TEXT DEFAULT 'foo'", "'foo'", "'foo'"), ("TEXT DEFAULT 'foo)'", "'foo)'", "'foo)'"), diff --git a/tests/test_duplicate.py b/tests/test_duplicate.py index 552f697..28961d2 100644 --- a/tests/test_duplicate.py +++ b/tests/test_duplicate.py @@ -5,14 +5,12 @@ import pytest def test_duplicate(fresh_db): # Create table using native Sqlite statement: - fresh_db.execute( - """CREATE TABLE "table1" ( + fresh_db.execute("""CREATE TABLE "table1" ( "text_col" TEXT, "real_col" REAL, "int_col" INTEGER, "bool_col" INTEGER, - "datetime_col" TEXT)""" - ) + "datetime_col" TEXT)""") # Insert one row of mock data: dt = datetime.datetime.now() data = { diff --git a/tests/test_extract.py b/tests/test_extract.py index 435c1ae..d24c597 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -126,9 +126,7 @@ def test_extract_rowid_table(fresh_db): ' "common_name_latin_name_id" INTEGER REFERENCES "common_name_latin_name"("id")\n' ")" ) - assert ( - fresh_db.execute( - """ + assert fresh_db.execute(""" select tree.name, common_name_latin_name.common_name, @@ -136,10 +134,7 @@ def test_extract_rowid_table(fresh_db): from tree join common_name_latin_name on tree.common_name_latin_name_id = common_name_latin_name.id - """ - ).fetchall() - == [("Tree 1", "Palm", "Arecaceae")] - ) + """).fetchall() == [("Tree 1", "Palm", "Arecaceae")] def test_reuse_lookup_table(fresh_db): diff --git a/tests/test_introspect.py b/tests/test_introspect.py index ab61c15..4ff3f77 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -109,13 +109,11 @@ def test_table_repr(fresh_db): def test_indexes(fresh_db): - fresh_db.executescript( - """ + fresh_db.executescript(""" create table Gosh (c1 text, c2 text, c3 text); create index Gosh_c1 on Gosh(c1); create index Gosh_c2c3 on Gosh(c2, c3); - """ - ) + """) assert [ Index( seq=0, @@ -130,13 +128,11 @@ def test_indexes(fresh_db): def test_xindexes(fresh_db): - fresh_db.executescript( - """ + fresh_db.executescript(""" create table Gosh (c1 text, c2 text, c3 text); create index Gosh_c1 on Gosh(c1); create index Gosh_c2c3 on Gosh(c2, c3 desc); - """ - ) + """) assert fresh_db["Gosh"].xindexes == [ XIndex( name="Gosh_c2c3", diff --git a/tests/test_transform.py b/tests/test_transform.py index e763a6c..5eb501d 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -638,15 +638,13 @@ def test_transform_with_indexes_errors(fresh_db, transform_params): def test_transform_with_unique_constraint_implicit_index(fresh_db): dogs = fresh_db["dogs"] # Create a table with a UNIQUE constraint on 'name', which creates an implicit index - fresh_db.execute( - """ + fresh_db.execute(""" CREATE TABLE dogs ( id INTEGER PRIMARY KEY, name TEXT UNIQUE, age INTEGER ); - """ - ) + """) dogs.insert({"id": 1, "name": "Cleo", "age": 5}) # Attempt to transform the table without modifying 'name' From b702a512565b5a6edcb05a3e9c3df246ffa0e42a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 09:37:05 -0700 Subject: [PATCH 002/120] Add test for transform ... --sql command Coverage spotted this was missinge. --- tests/test_cli.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 40b3685..565fbc2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1734,6 +1734,29 @@ def test_transform(db_path, args, expected_schema): assert schema == expected_schema +def test_transform_sql(db_path): + db = Database(db_path) + with db.conn: + db["dogs"].insert( + {"id": 1, "age": 4, "name": "Cleo"}, + not_null={"age"}, + defaults={"age": 1}, + pk="id", + ) + original_schema = db["dogs"].schema + + result = CliRunner().invoke( + cli.cli, ["transform", db_path, "dogs", "--drop", "name", "--sql"] + ) + + assert result.exit_code == 0, result.output + assert 'CREATE TABLE "dogs_new_' in result.output + assert '"age" INTEGER NOT NULL DEFAULT' in result.output + assert 'DROP TABLE "dogs";' in result.output + assert 'ALTER TABLE "dogs_new_' in result.output + assert db["dogs"].schema == original_schema + + @pytest.mark.parametrize( "extra_args,expected_schema", ( From 3cc27d69bc86d6db47bb72fd135c972b4297131d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 09:40:21 -0700 Subject: [PATCH 003/120] New migrations system, ported from sqlite-migrate (#754) Closes #752 --- docs/changelog.rst | 9 +- docs/cli-reference.rst | 41 ++++- docs/cli.rst | 31 ++++ docs/index.rst | 1 + docs/migrations.rst | 171 +++++++++++++++++++++ sqlite_utils/__init__.py | 3 +- sqlite_utils/cli.py | 187 +++++++++++++++++++++-- sqlite_utils/migrations.py | 126 +++++++++++++++ tests/test_cli_migrate.py | 303 +++++++++++++++++++++++++++++++++++++ tests/test_migrations.py | 110 ++++++++++++++ 10 files changed, 963 insertions(+), 19 deletions(-) create mode 100644 docs/migrations.rst create mode 100644 sqlite_utils/migrations.py create mode 100644 tests/test_cli_migrate.py create mode 100644 tests/test_migrations.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 1406116..d5c41aa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _v_unreleased: + +Unreleased +---------- + +- New :ref:`database migrations system `, incorporating functionality that was previously provided by the separate `sqlite-migrate `__ plugin. Define migration sets using the new :class:`sqlite_utils.Migrations` class and apply them using the ``sqlite-utils migrate`` command or the :ref:`migrations Python API `. (:issue:`752`) + .. _v3_39: 3.39 (2025-11-24) @@ -182,7 +189,7 @@ This release introduces a new :ref:`plugin system `. Read more about th - Conversion functions passed to :ref:`table.convert(...) ` can now return lists or dictionaries, which will be inserted into the database as JSON strings. (:issue:`495`) - ``sqlite-utils install`` and ``sqlite-utils uninstall`` commands for installing packages into the same virtual environment as ``sqlite-utils``, :ref:`described here `. (:issue:`483`) - New :ref:`sqlite_utils.utils.flatten() ` utility function. (:issue:`500`) -- Documentation on :ref:`using Just ` to run tests, linters and build documentation. +- Documentation on :ref:`using Just ` to run tests, linters and build documentation. - Documentation now covers the :ref:`release_process` for this package. .. _v3_29: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c3bf7a0..64a27c2 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -20,7 +20,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. "query", "memory", "insert", "upsert", "bulk", "search", "transform", "extract", "schema", "insert-files", "analyze-tables", "convert", "tables", "views", "rows", "triggers", "indexes", "create-database", "create-table", "create-index", - "enable-fts", "populate-fts", "rebuild-fts", "disable-fts" + "migrate", "enable-fts", "populate-fts", "rebuild-fts", "disable-fts" ] refs = { "query": "cli_query", @@ -49,6 +49,7 @@ This page lists the ``--help`` for every ``sqlite-utils`` CLI sub-command. "enable-wal": "cli_wal", "enable-counts": "cli_enable_counts", "bulk": "cli_bulk", + "migrate": "cli_migrate", "create-database": "cli_create_database", "create-table": "cli_create_table", "drop-table": "cli_drop_table", @@ -965,6 +966,44 @@ See :ref:`cli_create_index`. -h, --help Show this message and exit. +.. _cli_ref_migrate: + +migrate +======= + +See :ref:`cli_migrate`. + +:: + + Usage: sqlite-utils migrate [OPTIONS] DB_PATH [MIGRATIONS]... + + Apply pending database migrations. + + Usage: + + sqlite-utils migrate database.db + + This will find the migrations.py file in the current directory or + subdirectories and apply any pending migrations. + + Or pass paths to one or more migrations.py files directly: + + sqlite-utils migrate database.db path/to/migrations.py + + Pass --list to see a list of applied and pending migrations without applying + them. + + Use --stop-before migration_set:name to stop before a migration. This option + can be used multiple times. + + Options: + --stop-before TEXT Stop before applying this migration. Use set:name to + target a migration set. + --list List migrations without running them + -v, --verbose Show verbose output + -h, --help Show this message and exit. + + .. _cli_ref_enable_fts: enable-fts diff --git a/docs/cli.rst b/docs/cli.rst index 415e4c2..c9389d8 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1058,6 +1058,37 @@ That will look for SpatiaLite in a set of predictable locations. To load it from sqlite-utils create-database empty.db --init-spatialite --load-extension /path/to/spatialite.so +.. _cli_migrate: + +Running migrations +================== + +The ``migrate`` command applies pending Python migrations to a database. For the full migration file format and Python API, see :ref:`migrations`. + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py + +If you omit the migration path it will search the current directory and subdirectories for files called ``migrations.py``: + +.. code-block:: bash + + sqlite-utils migrate creatures.db + +Use ``--list`` to list applied and pending migrations without running them: + +.. code-block:: bash + + sqlite-utils migrate creatures.db --list + +Use ``--stop-before`` to stop before a named migration. The option can be passed more than once, and can target a specific migration set using ``migration_set:migration_name``: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py \ + --stop-before creatures:add_weight \ + --stop-before sales:drop_index + .. _cli_inserting_data: Inserting JSON data diff --git a/docs/index.rst b/docs/index.rst index 27a22c3..052b790 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,6 +36,7 @@ Contents installation cli python-api + migrations plugins reference cli-reference diff --git a/docs/migrations.rst b/docs/migrations.rst new file mode 100644 index 0000000..948b468 --- /dev/null +++ b/docs/migrations.rst @@ -0,0 +1,171 @@ +.. _migrations: + +===================== + Database migrations +===================== + +``sqlite-utils`` includes a migration system for applying repeatable changes to SQLite database files. + +A migration is a Python function that receives a :class:`sqlite_utils.Database` instance and then executes Python code to modify that database - creating or transforming tables, adding indexes, inserting rows, or any other operation supported by SQLite. + +Migrations are grouped into named sets using the :class:`sqlite_utils.Migrations` class, and each applied migration is recorded in the ``_sqlite_migrations`` table in that database. + +This means you can run the migrate operation multiple times and it will only apply migrations that have not previously been recorded. + +.. _migrations_define: + +Defining migrations +=================== + +Ordered migration sets are defined by first creating a :class:`sqlite_utils.Migrations` object. + +Individual migrations are Python functions that are then registered with that migration set. Each migration function is passed a single argument that is a :ref:`sqlite_utils.Database ` instance. + +The name passed to ``Migrations("creatures")`` identifies that set of migrations. Use a name that is unique for your project, since multiple migration sets can be applied to the same database. + +Here is a simple example of a ``migrations.py`` file which creates a table, then adds an extra column to that table in a second migration: + +.. code-block:: python + + from sqlite_utils import Database, Migrations + + migrations = Migrations("creatures") + + @migrations() + def create_table(db): + db["creatures"].create( + {"id": int, "name": str, "species": str}, + pk="id", + ) + + @migrations() + def add_weight(db): + db["creatures"].add_column("weight", float) + +.. _migrations_python: + +Applying migrations in Python +============================= + +Once you have a ``Migrations(name)`` collection with one or more migrations registered to it, you can execute them in Python code like this: + +.. code-block:: python + + db = Database("creatures.db") + migrations.apply(db) + +Running ``migrations.apply(db)`` repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped. + +Migration functions are applied in the order that they were registered. The function name is used as the migration name unless you pass one explicitly: + +.. code-block:: python + + @migrations(name="001_create_table") + def create_table(db): + db["creatures"].create({"id": int, "name": str}, pk="id") + +When you apply a set of migrations you can stop part way through by specifying a ``stop_before=`` migration name: + +.. code-block:: python + + migrations.apply(db, stop_before="add_weight") + +Applying migrations using the CLI +================================= + +Run migrations using the ``sqlite-utils migrate`` command: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py + +The first argument is the database file. The remaining arguments can be paths to migration files or directories containing migration files. + +If you omit migration paths, ``sqlite-utils`` searches the current directory and subdirectories for files called ``migrations.py``: + +.. code-block:: bash + + sqlite-utils migrate creatures.db + +You can also pass a directory. Every ``migrations.py`` file in that directory tree will be considered: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/project/ + +Running the command repeatedly is safe. Migrations that already have a matching ``migration_set`` and ``name`` row in ``_sqlite_migrations`` will be skipped. + +Listing migrations +================== + +Use ``--list`` to show applied and pending migrations without running them: + +.. code-block:: bash + + sqlite-utils migrate creatures.db --list + +Example output: + +.. code-block:: output + + Migrations for: creatures + + Applied: + create_table - 2026-06-09 17:23:12.048092+00:00 + add_weight - 2026-06-09 17:23:12.051249+00:00 + + Pending: + add_age + +Stopping before a migration +=========================== + +When applying migrations using the CLI, you can stop before a named migration: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py --stop-before add_weight + +This applies any pending migrations before ``add_weight`` and leaves ``add_weight`` and later migrations pending. An unqualified migration name matches in any migration set. + +You can also target a specific migration set using ``migration_set:migration_name``. This is useful if a migrations file contains more than one migration set, or if multiple sets use the same migration name: + +.. code-block:: bash + + sqlite-utils migrate creatures.db path/to/migrations.py \ + --stop-before creatures:add_weight \ + --stop-before sales:drop_index + +The ``--stop-before`` option can be passed more than once. + +Verbose output +============== + +Use ``--verbose`` or ``-v`` to show the schema before and after migrations are applied, plus a unified diff when the schema changes: + +.. code-block:: bash + + sqlite-utils migrate creatures.db --verbose + +Migrating from sqlite-migrate +============================= + +This system uses the same migration table format as the older `sqlite-migrate `__ package. To use existing migration files directly with ``sqlite-utils``, update their import from ``sqlite_migrate`` to ``sqlite_utils``: + +.. code-block:: python + + from sqlite_utils import Migrations + + migration = Migrations("creatures") + + @migration() + def create_table(db): + db["creatures"].create({"id": int, "name": str}, pk="id") + +Python API +========== + +.. autoclass:: sqlite_utils.migrations.Migrations + :members: + :undoc-members: + :exclude-members: _Migration, _AppliedMigration diff --git a/sqlite_utils/__init__.py b/sqlite_utils/__init__.py index b8046f6..58ee7ab 100644 --- a/sqlite_utils/__init__.py +++ b/sqlite_utils/__init__.py @@ -2,5 +2,6 @@ from .utils import suggest_column_types from .hookspecs import hookimpl from .hookspecs import hookspec from .db import Database +from .migrations import Migrations -__all__ = ["Database", "suggest_column_types", "hookimpl", "hookspec"] +__all__ = ["Database", "Migrations", "suggest_column_types", "hookimpl", "hookspec"] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 5844dfc..1c3ce8d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,4 +1,5 @@ import base64 +import difflib from typing import Any import click from click_default_group import DefaultGroup @@ -10,6 +11,7 @@ import sqlite_utils from sqlite_utils.db import ( AlterError, BadMultiValues, + DEFAULT, DescIndex, NoTable, quote_identifier, @@ -2578,7 +2580,6 @@ def transform( _register_db_for_cleanup(db) _load_extensions(db, load_extension) types = {} - kwargs = {} for column, ctype in type: if ctype.upper() not in VALID_COLUMN_TYPES: raise click.ClickException( @@ -2598,29 +2599,46 @@ def transform( for column in default_none: default_dict[column] = None - kwargs["types"] = types - kwargs["drop"] = set(drop) - kwargs["rename"] = dict(rename) - kwargs["column_order"] = column_order or None - kwargs["not_null"] = not_null_dict + drop_set = set(drop) + rename_dict = dict(rename) + column_order_list = list(column_order) or None + drop_foreign_keys_value = drop_foreign_keys or None + add_foreign_keys_value = add_foreign_keys or None + pk_value = DEFAULT if pk: if len(pk) == 1: - kwargs["pk"] = pk[0] + pk_value = pk[0] else: - kwargs["pk"] = pk + pk_value = pk elif pk_none: - kwargs["pk"] = None - kwargs["defaults"] = default_dict - if drop_foreign_keys: - kwargs["drop_foreign_keys"] = drop_foreign_keys - if add_foreign_keys: - kwargs["add_foreign_keys"] = add_foreign_keys + pk_value = None + table_obj = db.table(table) if sql: - for line in db.table(table).transform_sql(**kwargs): + for line in table_obj.transform_sql( + types=types, + drop=drop_set, + rename=rename_dict, + column_order=column_order_list, + not_null=not_null_dict, + pk=pk_value, + defaults=default_dict, + drop_foreign_keys=drop_foreign_keys_value, + add_foreign_keys=add_foreign_keys_value, + ): click.echo(line) else: - db.table(table).transform(**kwargs) + table_obj.transform( + types=types, + drop=drop_set, + rename=rename_dict, + column_order=column_order_list, + not_null=not_null_dict, + pk=pk_value, + defaults=default_dict, + drop_foreign_keys=drop_foreign_keys_value, + add_foreign_keys=add_foreign_keys_value, + ) @cli.command() @@ -3252,6 +3270,142 @@ def create_spatial_index(db_path, table, column_name, load_extension): db.table(table).create_spatial_index(column_name) +def _find_migration_files(migrations): + if not migrations: + migrations = [pathlib.Path(".").resolve()] + files = set() + for path_str in migrations: + path = pathlib.Path(path_str) + if path.is_dir(): + files.update(path.rglob("migrations.py")) + else: + files.add(path) + return sorted(files) + + +def _compatible_migration_set(obj): + return isinstance(obj, sqlite_utils.Migrations) or all( + hasattr(obj, attr) for attr in ("name", "applied", "pending", "apply") + ) + + +def _load_migration_sets(files): + migration_sets = [] + for filepath in files: + code = filepath.read_text() + namespace = { + "__file__": str(filepath), + "__name__": "__sqlite_utils_migration__", + } + exec(code, namespace) + migration_sets.extend( + obj for obj in namespace.values() if _compatible_migration_set(obj) + ) + return migration_sets + + +def _display_migration_list(db, migration_sets): + for migration_set in migration_sets: + click.echo("Migrations for: {}".format(migration_set.name)) + click.echo() + click.echo(" Applied:") + for migration in migration_set.applied(db): + click.echo(" {} - {}".format(migration.name, migration.applied_at)) + click.echo() + click.echo(" Pending:") + output = False + for migration in migration_set.pending(db): + output = True + click.echo(" {}".format(migration.name)) + if not output: + click.echo(" (none)") + click.echo() + + +def _stop_before_for_migration_set(stop_before, migration_set_name): + matches = [] + for value in stop_before: + set_name, separator, migration_name = value.partition(":") + if separator: + if set_name == migration_set_name: + matches.append(migration_name) + else: + matches.append(value) + return matches + + +@click.command() +@click.argument( + "db_path", type=click.Path(dir_okay=False, readable=True, writable=True) +) +@click.argument("migrations", type=click.Path(dir_okay=True, exists=True), nargs=-1) +@click.option( + "--stop-before", + multiple=True, + help="Stop before applying this migration. Use set:name to target a migration set.", +) +@click.option( + "list_", "--list", is_flag=True, help="List migrations without running them" +) +@click.option("-v", "--verbose", is_flag=True, help="Show verbose output") +def migrate(db_path, migrations, stop_before, list_, verbose): + """ + Apply pending database migrations. + + Usage: + + sqlite-utils migrate database.db + + This will find the migrations.py file in the current directory + or subdirectories and apply any pending migrations. + + Or pass paths to one or more migrations.py files directly: + + sqlite-utils migrate database.db path/to/migrations.py + + Pass --list to see a list of applied and pending migrations + without applying them. + + Use --stop-before migration_set:name to stop before a + migration. This option can be used multiple times. + """ + files = _find_migration_files(migrations) + migration_sets = _load_migration_sets(files) + if not migration_sets: + raise click.ClickException("No migrations.py files found") + + db = sqlite_utils.Database(db_path) + _register_db_for_cleanup(db) + + if list_: + _display_migration_list(db, migration_sets) + return + + prev_schema = db.schema + if verbose: + click.echo("Migrating {}".format(db_path)) + click.echo("\nSchema before:\n") + click.echo(textwrap.indent(prev_schema, " ") or " (empty)") + click.echo() + for migration_set in migration_sets: + migration_set.apply( + db, + stop_before=_stop_before_for_migration_set(stop_before, migration_set.name), + ) + if verbose: + click.echo("Schema after:\n") + post_schema = db.schema + if post_schema == prev_schema: + click.echo(" (unchanged)") + else: + click.echo(textwrap.indent(post_schema, " ")) + click.echo("\nSchema diff:\n") + diff = list( + difflib.unified_diff(prev_schema.splitlines(), post_schema.splitlines()) + ) + click.echo("\n".join(diff[3:])) + + @cli.command(name="plugins") def plugins_list(): "List installed plugins" @@ -3259,6 +3413,7 @@ def plugins_list(): pm.hook.register_commands(cli=cli) +cli.add_command(migrate) def _render_common(title, values): diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py new file mode 100644 index 0000000..f349b17 --- /dev/null +++ b/sqlite_utils/migrations.py @@ -0,0 +1,126 @@ +from collections.abc import Iterable +from dataclasses import dataclass +import datetime +from typing import Callable, cast, TYPE_CHECKING + +if TYPE_CHECKING: + from sqlite_utils.db import Database, Table + + +class Migrations: + migrations_table = "_sqlite_migrations" + + @dataclass + class _Migration: + name: str + fn: Callable + + @dataclass + class _AppliedMigration: + name: str + applied_at: datetime.datetime + + def __init__(self, name: str): + """ + :param name: The name of the migration set. This should be unique. + """ + self.name = name + self._migrations: list[Migrations._Migration] = [] + + def __call__(self, *, name: str | None = None) -> Callable: + """ + :param name: The name to use for this migration - if not provided, + the name of the function will be used. + """ + + def inner(func: Callable) -> Callable: + self._migrations.append( + self._Migration(name or getattr(func, "__name__"), func) + ) + return func + + return inner + + def pending(self, db: "Database") -> list["Migrations._Migration"]: + """ + Return a list of pending migrations. + """ + self.ensure_migrations_table(db) + already_applied = { + r["name"] + for r in db[self.migrations_table].rows_where( + "migration_set = ?", [self.name] + ) + } + return [ + migration + for migration in self._migrations + if migration.name not in already_applied + ] + + def applied(self, db: "Database") -> list["Migrations._AppliedMigration"]: + """ + Return a list of applied migrations. + """ + self.ensure_migrations_table(db) + return [ + self._AppliedMigration(name=row["name"], applied_at=row["applied_at"]) + for row in db[self.migrations_table].rows_where( + "migration_set = ?", [self.name] + ) + ] + + def apply(self, db: "Database", *, stop_before: str | Iterable[str] | None = None): + """ + Apply any pending migrations to the database. + """ + self.ensure_migrations_table(db) + if stop_before is None: + stop_before_names = set() + elif isinstance(stop_before, str): + stop_before_names = {stop_before} + else: + stop_before_names = set(stop_before) + for migration in self.pending(db): + name = migration.name + if name in stop_before_names: + return + migration.fn(db) + _table(db, self.migrations_table).insert( + { + "migration_set": self.name, + "name": name, + "applied_at": str(datetime.datetime.now(datetime.timezone.utc)), + } + ) + + def ensure_migrations_table(self, db: "Database"): + """ + Ensure the _sqlite_migrations table exists and has the correct schema. + """ + table = _table(db, self.migrations_table) + if not table.exists(): + table.create( + { + "id": int, + "migration_set": str, + "name": str, + "applied_at": str, + }, + pk="id", + ) + table.create_index(["migration_set", "name"], unique=True) + elif table.pks != ["id"]: + table.transform(pk="id") + unique_indexes = {tuple(index.columns) for index in table.indexes} + if ("migration_set", "name") not in unique_indexes: + table.create_index(["migration_set", "name"], unique=True) + + def __repr__(self): + return "".format( + self.name, ", ".join(m.name for m in self._migrations) + ) + + +def _table(db: "Database", name: str) -> "Table": + return cast("Table", db[name]) diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py new file mode 100644 index 0000000..229bc9c --- /dev/null +++ b/tests/test_cli_migrate.py @@ -0,0 +1,303 @@ +import pathlib + +from click.testing import CliRunner +import pytest +import sqlite_utils +import sqlite_utils.cli + +TWO_MIGRATIONS = """ +from sqlite_utils import Migrations + +m = Migrations("hello") + +@m() +def foo(db): + db["foo"].insert({"hello": "world"}) + +@m() +def bar(db): + db["bar"].insert({"hello": "world"}) +""" + + +@pytest.fixture +def two_migrations(tmpdir): + path = pathlib.Path(tmpdir) + (path / "foo").mkdir() + migrations_py = path / "foo" / "migrations.py" + migrations_py.write_text(TWO_MIGRATIONS, "utf-8") + return path, migrations_py + + +@pytest.fixture +def two_sets_same_migration_name(tmpdir): + path = pathlib.Path(tmpdir) + migrations_py = path / "migrations.py" + migrations_py.write_text( + """ +from sqlite_utils import Migrations + +creatures = Migrations("creatures") + +@creatures() +def create_table(db): + db["creatures"].insert({"name": "Cleo"}) + +@creatures() +def add_weight(db): + db["creature_weights"].insert({"weight": 4.2}) + +sales = Migrations("sales") + +@sales() +def create_table(db): + db["sales"].insert({"id": 1}) + +@sales() +def add_weight(db): + db["sales_weights"].insert({"weight": 10}) +""", + "utf-8", + ) + return path, migrations_py + + +@pytest.mark.parametrize("arg", ("TMPDIR", "TMPDIR/foo/migrations.py", "TMPDIR/foo/")) +def test_basic(two_migrations, arg): + path, _ = two_migrations + db_path = str(path / "test.db") + + runner = CliRunner() + + def _list(): + list_result = runner.invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, "--list", arg.replace("TMPDIR", str(path))], + ) + assert list_result.exit_code == 0 + return list_result.output + + assert _list() == ( + "Migrations for: hello\n\n" + " Applied:\n\n" + " Pending:\n" + " foo\n" + " bar\n\n" + ) + + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, arg.replace("TMPDIR", str(path))] + ) + assert result.exit_code == 0, result.output + + list_output = _list() + assert "Migrations for: hello\n\n Applied:\n " in list_output + prior_to_pending = list_output.split(" Pending")[0] + assert " foo" in prior_to_pending + assert " bar" in prior_to_pending + assert " Pending:\n (none)" in list_output + + db = sqlite_utils.Database(db_path) + assert db["foo"].exists() + assert db["bar"].exists() + assert db["_sqlite_migrations"].exists() + rows = list(db["_sqlite_migrations"].rows) + assert len(rows) == 2 + assert rows[0]["name"] == "foo" + assert rows[1]["name"] == "bar" + + +def test_list_same_migration_names_in_different_sets(capsys): + applied = sqlite_utils.Migrations("applied") + + @applied(name="foo") + def applied_foo(db): + db["applied"].insert({"hello": "world"}) + + pending = sqlite_utils.Migrations("pending") + + @pending(name="foo") + def pending_foo(db): + db["pending"].insert({"hello": "world"}) + + db = sqlite_utils.Database(memory=True) + applied.apply(db) + + sqlite_utils.cli._display_migration_list(db, [applied, pending]) + + output = capsys.readouterr().out + assert ( + "Migrations for: pending\n\n" " Applied:\n\n" " Pending:\n" " foo\n\n" + ) in output + + +def test_verbose(tmpdir): + path = pathlib.Path(tmpdir) + (path / "foo").mkdir() + migrations_py = path / "foo" / "migrations.py" + migrations_py.write_text( + """ +from sqlite_utils import Migrations + +m = Migrations("hello") + +@m() +def foo(db): + db["dogs"].insert({"id": 1, "name": "Cleo"}) + """, + "utf-8", + ) + db_path = str(path / "test.db") + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py)] + ) + assert result.exit_code == 0 + + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"] + ) + assert result.exit_code == 0 + expected = """ +Schema before: + + CREATE TABLE "_sqlite_migrations" ( + "id" INTEGER PRIMARY KEY, + "migration_set" TEXT, + "name" TEXT, + "applied_at" TEXT + ); + CREATE UNIQUE INDEX "idx__sqlite_migrations_migration_set_name" + ON "_sqlite_migrations" ("migration_set", "name"); + CREATE TABLE "dogs" ( + "id" INTEGER, + "name" TEXT + ); + +Schema after: + + (unchanged) +""".strip() + assert expected in result.output + + new_migration = """ +@m() +def bar(db): + db["dogs"].add_column("age", int) + db["dogs"].add_column("weight", float) + db["dogs"].transform() +""" + migrations_py.write_text(migrations_py.read_text("utf-8") + new_migration) + + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(migrations_py), "--verbose"] + ) + assert result.exit_code == 0 + expected_diff = """ +Schema diff: + + ON "_sqlite_migrations" ("migration_set", "name"); + CREATE TABLE "dogs" ( + "id" INTEGER, +- "name" TEXT ++ "name" TEXT, ++ "age" INTEGER, ++ "weight" REAL + ); +""".strip() + assert expected_diff in result.output + + +def test_stop_before(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(path / "foo" / "migrations.py"), + "--stop-before", + "bar", + ], + ) + assert result.exit_code == 0 + db = sqlite_utils.Database(db_path) + assert db["foo"].exists() + assert not db["bar"].exists() + + +def test_stop_before_multiple_sets_unqualified(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + (path / "foo" / "migrations2.py").write_text( + """ +from sqlite_utils import Migrations + +m = Migrations("hello2") + +@m() +def foo(db): + db["foo"].insert({"hello": "world"}) + """, + "utf-8", + ) + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(path / "foo" / "migrations.py"), + str(path / "foo" / "migrations2.py"), + "--stop-before", + "foo", + ], + ) + assert result.exit_code == 0, result.output + db = sqlite_utils.Database(db_path) + assert db.table_names() == ["_sqlite_migrations"] + assert list(db["_sqlite_migrations"].rows) == [] + + +def test_stop_before_qualified_only_affects_named_set(two_sets_same_migration_name): + path, migrations_py = two_sets_same_migration_name + db_path = str(path / "test.db") + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(migrations_py), + "--stop-before", + "creatures:add_weight", + ], + ) + assert result.exit_code == 0, result.output + db = sqlite_utils.Database(db_path) + assert db["creatures"].exists() + assert not db["creature_weights"].exists() + assert db["sales"].exists() + assert db["sales_weights"].exists() + + +def test_stop_before_multiple_qualified(two_sets_same_migration_name): + path, migrations_py = two_sets_same_migration_name + db_path = str(path / "test.db") + result = CliRunner().invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(migrations_py), + "--stop-before", + "creatures:add_weight", + "--stop-before", + "sales:add_weight", + ], + ) + assert result.exit_code == 0, result.output + db = sqlite_utils.Database(db_path) + assert db["creatures"].exists() + assert not db["creature_weights"].exists() + assert db["sales"].exists() + assert not db["sales_weights"].exists() diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 0000000..6a0dd33 --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,110 @@ +import pytest +import sqlite_utils +from sqlite_utils import Migrations + + +@pytest.fixture +def migrations(): + migrations = Migrations("test") + + @migrations() + def m001(db): + db["dogs"].insert({"name": "Cleo"}) + + @migrations() + def m002(db): + db["cats"].create({"name": str}) + db.query("insert into dogs (name) values ('Pancakes')") + + return migrations + + +@pytest.fixture +def migrations_not_ordered_alphabetically(): + # Names order alphabetically in the wrong direction but this + # should still be applied correctly. + migrations = Migrations("test") + + @migrations() + def m002(db): + db["dogs"].insert({"name": "Cleo"}) + + @migrations() + def m001(db): + db["cats"].create({"name": str}) + db.query("insert into dogs (name) values ('Pancakes')") + + return migrations + + +@pytest.fixture +def migrations2(): + migrations = Migrations("test2") + + @migrations() + def m001(db): + db["dogs2"].insert({"name": "Cleo"}) + + return migrations + + +def test_basic(migrations): + db = sqlite_utils.Database(memory=True) + assert db.table_names() == [] + migrations.apply(db) + assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"} + + +def test_stop_before(migrations): + db = sqlite_utils.Database(memory=True) + assert db.table_names() == [] + migrations.apply(db, stop_before="m002") + assert set(db.table_names()) == {"_sqlite_migrations", "dogs"} + migrations.apply(db) + assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"} + + +def test_two_migration_sets(migrations, migrations2): + db = sqlite_utils.Database(memory=True) + assert db.table_names() == [] + migrations.apply(db) + migrations2.apply(db) + assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats", "dogs2"} + + +def test_order_does_not_matter(migrations, migrations_not_ordered_alphabetically): + db1 = sqlite_utils.Database(memory=True) + db2 = sqlite_utils.Database(memory=True) + migrations.apply(db1) + migrations_not_ordered_alphabetically.apply(db2) + assert db1.schema == db2.schema + + +@pytest.mark.parametrize( + "create_table,pk", + ( + ( + { + "migration_set": str, + "name": str, + "applied_at": str, + }, + "name", + ), + ( + { + "migration_set": str, + "name": str, + "applied_at": str, + }, + ("migration_set", "name"), + ), + ), +) +def test_upgrades_sqlite_migrations(migrations, create_table, pk): + db = sqlite_utils.Database(memory=True) + db["_sqlite_migrations"].create(create_table, pk=pk) + assert db.table_names() == ["_sqlite_migrations"] + assert db["_sqlite_migrations"].pks == ([pk] if isinstance(pk, str) else list(pk)) + migrations.apply(db) + assert db["_sqlite_migrations"].pks == ["id"] From 6729ea3f60fd10800143ab2e5d085d22b5956e83 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 10:43:19 -0700 Subject: [PATCH 004/120] db.atomic() method Closes #755 --- docs/changelog.rst | 1 + docs/python-api.rst | 31 ++++- sqlite_utils/db.py | 275 ++++++++++++++++++++++++++++--------------- tests/test_atomic.py | 174 +++++++++++++++++++++++++++ 4 files changed, 382 insertions(+), 99 deletions(-) create mode 100644 tests/test_atomic.py diff --git a/docs/changelog.rst b/docs/changelog.rst index d5c41aa..1fa3ac8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Unreleased ---------- +- New ``db.atomic()`` context manager for transactions, with nested transaction support using SQLite savepoints. Internal multi-step operations such as ``table.transform()`` now use this mechanism to avoid unexpectedly committing an existing transaction. - New :ref:`database migrations system `, incorporating functionality that was previously provided by the separate `sqlite-migrate `__ plugin. Define migration sets using the new :class:`sqlite_utils.Migrations` class and apply them using the ``sqlite-utils migrate`` command or the :ref:`migrations Python API `. (:issue:`752`) .. _v3_39: diff --git a/docs/python-api.rst b/docs/python-api.rst index 267591a..eab858e 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -239,6 +239,35 @@ The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around Other cursor methods such as ``.fetchone()`` and ``.fetchall()`` are also available, see the `standard library documentation `__. +.. _python_api_atomic: + +Transactions with db.atomic() +----------------------------- + +Use ``db.atomic()`` to group multiple operations in a transaction: + +.. code-block:: python + + with db.atomic(): + db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") + db.table("dogs").insert({"id": 2, "name": "Pancakes"}) + +If an exception is raised, changes made inside the block will be rolled back. + +``db.atomic()`` can be nested. Nested blocks use SQLite savepoints, so an exception in an inner block can roll back to that savepoint without rolling back the entire outer transaction: + +.. code-block:: python + + with db.atomic(): + db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") + try: + with db.atomic(): + db.table("dogs").insert({"id": 2, "name": "Pancakes"}) + raise ValueError("skip this one") + except ValueError: + pass + db.table("dogs").insert({"id": 3, "name": "Marnie"}) + .. _python_api_parameters: Passing parameters @@ -955,7 +984,7 @@ You can delete all records in a table that match a specific WHERE statement usin >>> db = sqlite_utils.Database("dogs.db") >>> # Delete every dog with age less than 3 - >>> with db.conn: + >>> with db.atomic(): >>> db.table("dogs").delete_where("age < ?", [3]) Calling ``table.delete_where()`` with no other arguments will delete every row in the table. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ed3fc7a..677a1f8 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -196,6 +196,19 @@ DEFAULT = Default() Tracer = Callable[[str, Optional[Union[Sequence[Any], Dict[str, Any]]]], None] +def _iter_complete_sql_statements(sql: str) -> Generator[str, None, None]: + statement = [] + for char in sql: + statement.append(char) + statement_sql = "".join(statement).strip() + if statement_sql and sqlite3.complete_statement(statement_sql): + yield statement_sql + statement = [] + statement_sql = "".join(statement).strip() + if statement_sql: + yield statement_sql + + COLUMN_TYPE_MAPPING: Dict[Any, str] = { float: "REAL", int: "INTEGER", @@ -406,6 +419,38 @@ class Database: "Close the SQLite connection, and the underlying database file" self.conn.close() + @contextlib.contextmanager + def atomic(self) -> Generator["Database", None, None]: + """ + Context manager for wrapping multiple database operations in a transaction. + + Nested blocks use SQLite savepoints. + """ + if self.conn.in_transaction: + savepoint = "sqlite_utils_{}".format(secrets.token_hex(16)) + self.conn.execute("SAVEPOINT {};".format(savepoint)) + try: + yield self + except BaseException: + self.conn.execute("ROLLBACK TO SAVEPOINT {};".format(savepoint)) + self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) + raise + else: + self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) + else: + self.conn.execute("BEGIN") + try: + yield self + except BaseException: + self.conn.rollback() + raise + else: + try: + self.conn.commit() + except BaseException: + self.conn.rollback() + raise + @contextlib.contextmanager def ensure_autocommit_off(self) -> Generator[None, None, None]: """ @@ -581,6 +626,15 @@ class Database: """ if self._tracer: self._tracer(sql, None) + return self._executescript(sql) + + def _executescript(self, sql: str) -> sqlite3.Cursor: + if self.conn.in_transaction: + cursor = self.conn.cursor() + # avoid sqlite3.executescript()'s implicit commit: + for statement in _iter_complete_sql_statements(sql): + cursor.execute(statement) + return cursor return self.conn.executescript(sql) def table(self, table_name: str, **kwargs: Any) -> "Table": @@ -729,7 +783,7 @@ class Database: if not hasattr(self, "_supports_strict"): try: table_name = "t{}".format(secrets.token_hex(16)) - with self.conn: + with self.atomic(): self.conn.execute( "create table {} (name text) strict".format(table_name) ) @@ -745,7 +799,7 @@ class Database: if not hasattr(self, "_supports_on_conflict"): table_name = "t{}".format(secrets.token_hex(16)) try: - with self.conn: + with self.atomic(): self.conn.execute( "create table {} (id integer primary key, name text)".format( table_name @@ -797,7 +851,7 @@ class Database: self.execute("PRAGMA journal_mode=delete;") def _ensure_counts_table(self) -> None: - with self.conn: + with self.atomic(): self.execute(_COUNTS_TABLE_CREATE_SQL.format(self._counts_table_name)) def enable_counts(self) -> None: @@ -833,7 +887,7 @@ class Database: def reset_counts(self) -> None: "Re-calculate cached counts for tables." tables = [table for table in self.tables if table.has_counts_triggers] - with self.conn: + with self.atomic(): self._ensure_counts_table() counts_table = self.table(self._counts_table_name) counts_table.delete_where() @@ -1288,7 +1342,8 @@ class Database: for table, fks in by_table.items(): self.table(table).transform(add_foreign_keys=fks) - self.vacuum() + if not self.conn.in_transaction: + self.vacuum() def index_foreign_keys(self) -> None: "Create indexes for every foreign key column on every table in the database." @@ -1820,7 +1875,7 @@ class Table(Queryable): self._defaults["strict"] = strict columns = {name: value for (name, value) in columns.items()} - with self.db.conn: + with self.db.atomic(): self.db.create_table( self.name, columns, @@ -1848,7 +1903,7 @@ class Table(Queryable): """ if not self.exists(): raise NoTable(f"Table {self.name} does not exist") - with self.db.conn: + with self.db.atomic(): sql = "CREATE TABLE {} AS SELECT * FROM {};".format( quote_identifier(new_name), quote_identifier(self.name), @@ -1905,20 +1960,40 @@ class Table(Queryable): column_order=column_order, keep_table=keep_table, ) - pragma_foreign_keys_was_on = self.db.execute("PRAGMA foreign_keys").fetchone()[ - 0 - ] + pragma_foreign_keys_was_on = bool( + self.db.execute("PRAGMA foreign_keys").fetchone()[0] + ) + already_in_transaction = self.db.conn.in_transaction + should_disable_foreign_keys = ( + pragma_foreign_keys_was_on and not already_in_transaction + ) + should_defer_foreign_keys = ( + pragma_foreign_keys_was_on and already_in_transaction + ) + defer_foreign_keys_was_on = False try: - if pragma_foreign_keys_was_on: + if should_disable_foreign_keys: self.db.execute("PRAGMA foreign_keys=0;") - with self.db.conn: + elif should_defer_foreign_keys: + defer_foreign_keys_was_on = bool( + self.db.execute("PRAGMA defer_foreign_keys").fetchone()[0] + ) + if not defer_foreign_keys_was_on: + self.db.execute("PRAGMA defer_foreign_keys=ON;") + with self.db.atomic(): for sql in sqls: self.db.execute(sql) # Run the foreign_key_check before we commit if pragma_foreign_keys_was_on: - self.db.execute("PRAGMA foreign_key_check;") + foreign_key_violations = self.db.execute( + "PRAGMA foreign_key_check;" + ).fetchall() + if foreign_key_violations: + raise sqlite3.IntegrityError("FOREIGN KEY constraint failed") finally: - if pragma_foreign_keys_was_on: + if should_defer_foreign_keys and not defer_foreign_keys_was_on: + self.db.execute("PRAGMA defer_foreign_keys=OFF;") + if should_disable_foreign_keys: self.db.execute("PRAGMA foreign_keys=1;") return self @@ -2158,86 +2233,89 @@ class Table(Queryable): columns, list(self.columns_dict.keys()) ) ) - table = table or "_".join(columns) - lookup_table = self.db.table(table) - fk_column = fk_column or "{}_id".format(table) - magic_lookup_column = "{}_{}".format(fk_column, os.urandom(6).hex()) + with self.db.atomic(): + table = table or "_".join(columns) + lookup_table = self.db.table(table) + fk_column = fk_column or "{}_id".format(table) + magic_lookup_column = "{}_{}".format(fk_column, os.urandom(6).hex()) - # Populate the lookup table with all of the extracted unique values - lookup_columns_definition = { - (rename.get(col) or col): typ - for col, typ in self.columns_dict.items() - if col in columns - } - if lookup_table.exists(): - if not set(lookup_columns_definition.items()).issubset( - lookup_table.columns_dict.items() - ): - raise InvalidColumns( - "Lookup table {} already exists but does not have columns {}".format( - table, lookup_columns_definition + # Populate the lookup table with all of the extracted unique values + lookup_columns_definition = { + (rename.get(col) or col): typ + for col, typ in self.columns_dict.items() + if col in columns + } + if lookup_table.exists(): + if not set(lookup_columns_definition.items()).issubset( + lookup_table.columns_dict.items() + ): + raise InvalidColumns( + "Lookup table {} already exists but does not have columns {}".format( + table, lookup_columns_definition + ) ) - ) - else: - lookup_table.create( - { - **{ - "id": int, - }, - **lookup_columns_definition, - }, - pk="id", - ) - lookup_columns = [(rename.get(col) or col) for col in columns] - lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True) - self.db.execute( - "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {}".format( - quote_identifier(table), - quote_identifier(self.name), - lookup_columns=", ".join(quote_identifier(c) for c in lookup_columns), - table_cols=", ".join(quote_identifier(c) for c in columns), - ) - ) - - # Now add the new fk_column - self.add_column(magic_lookup_column, int) - - # And populate it - self.db.execute( - "UPDATE {} SET {} = (SELECT id FROM {} WHERE {where})".format( - quote_identifier(self.name), - quote_identifier(magic_lookup_column), - quote_identifier(table), - where=" AND ".join( - "{}.{} IS {}.{}".format( - quote_identifier(self.name), - quote_identifier(column), - quote_identifier(table), - quote_identifier(rename.get(column) or column), - ) - for column in columns - ), - ) - ) - # Figure out the right column order - column_order = [] - for c in self.columns: - if c.name in columns and magic_lookup_column not in column_order: - column_order.append(magic_lookup_column) - elif c.name == magic_lookup_column: - continue else: - column_order.append(c.name) + lookup_table.create( + { + **{ + "id": int, + }, + **lookup_columns_definition, + }, + pk="id", + ) + lookup_columns = [(rename.get(col) or col) for col in columns] + lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True) + self.db.execute( + "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {}".format( + quote_identifier(table), + quote_identifier(self.name), + lookup_columns=", ".join( + quote_identifier(c) for c in lookup_columns + ), + table_cols=", ".join(quote_identifier(c) for c in columns), + ) + ) - # Drop the unnecessary columns and rename lookup column - self.transform( - drop=set(columns), - rename={magic_lookup_column: fk_column}, - column_order=column_order, - ) + # Now add the new fk_column + self.add_column(magic_lookup_column, int) - # And add the foreign key constraint - self.add_foreign_key(fk_column, table, "id") + # And populate it + self.db.execute( + "UPDATE {} SET {} = (SELECT id FROM {} WHERE {where})".format( + quote_identifier(self.name), + quote_identifier(magic_lookup_column), + quote_identifier(table), + where=" AND ".join( + "{}.{} IS {}.{}".format( + quote_identifier(self.name), + quote_identifier(column), + quote_identifier(table), + quote_identifier(rename.get(column) or column), + ) + for column in columns + ), + ) + ) + # Figure out the right column order + column_order = [] + for c in self.columns: + if c.name in columns and magic_lookup_column not in column_order: + column_order.append(magic_lookup_column) + elif c.name == magic_lookup_column: + continue + else: + column_order.append(c.name) + + # Drop the unnecessary columns and rename lookup column + self.transform( + drop=set(columns), + rename={magic_lookup_column: fk_column}, + column_order=column_order, + ) + + # And add the foreign key constraint + self.add_foreign_key(fk_column, table, "id") return self def create_index( @@ -2521,8 +2599,8 @@ class Table(Queryable): ), ) ) - with self.db.conn: - self.db.conn.executescript(sql) + with self.db.atomic(): + self.db._executescript(sql) self.db.use_counts_table = True @property @@ -2663,7 +2741,7 @@ class Table(Queryable): trigger_names = [] for row in self.db.execute(sql).fetchall(): trigger_names.append(row[0]) - with self.db.conn: + with self.db.atomic(): for trigger_name in trigger_names: self.db.execute( "DROP TRIGGER IF EXISTS {}".format(quote_identifier(trigger_name)) @@ -2862,7 +2940,7 @@ class Table(Queryable): sql = "delete from {} where {wheres}".format( quote_identifier(self.name), wheres=" and ".join(wheres) ) - with self.db.conn: + with self.db.atomic(): self.db.execute(sql, pk_values) return self @@ -2935,7 +3013,7 @@ class Table(Queryable): sets=", ".join(sets), wheres=" and ".join(wheres), ) - with self.db.conn: + with self.db.atomic(): try: rowcount = self.db.execute(sql, args).rowcount except OperationalError as e: @@ -3024,7 +3102,7 @@ class Table(Queryable): ), where=" where {}".format(where) if where is not None else "", ) - with self.db.conn: + with self.db.atomic(): self.db.execute(sql, where_args or []) if drop: self.transform(drop=columns) @@ -3072,7 +3150,7 @@ class Table(Queryable): with progressbar( length=self.count, silent=not show_progress, label="2: Updating" ) as bar: - with self.db.conn: + with self.db.atomic(): for pk, updates in pk_to_values.items(): self.update(pk, updates) bar.update(1) @@ -3291,7 +3369,7 @@ class Table(Queryable): list_mode, ) result = None - with self.db.conn: + with self.db.atomic(): for query, params in queries_and_params: try: result = self.db.execute(query, params) @@ -3528,7 +3606,8 @@ class Table(Queryable): self.last_rowid = None self.last_pk = None if truncate and self.exists(): - self.db.execute("DELETE FROM {};".format(quote_identifier(self.name))) + with self.db.atomic(): + self.db.execute("DELETE FROM {};".format(quote_identifier(self.name))) result = None for chunk in chunks(itertools.chain([first_record], records_iter), batch_size): chunk = list(chunk) diff --git a/tests/test_atomic.py b/tests/test_atomic.py new file mode 100644 index 0000000..acd5474 --- /dev/null +++ b/tests/test_atomic.py @@ -0,0 +1,174 @@ +import pytest + +from sqlite_utils.db import _iter_complete_sql_statements +from sqlite_utils.utils import sqlite3 + + +@pytest.mark.parametrize( + "sql,expected", + ( + ( + "CREATE TABLE t(id); INSERT INTO t VALUES (1)", + ["CREATE TABLE t(id);", "INSERT INTO t VALUES (1)"], + ), + ( + "INSERT INTO t VALUES ('a;b');", + ["INSERT INTO t VALUES ('a;b');"], + ), + ( + "-- comment;\nCREATE TABLE t(id);", + ["-- comment;\nCREATE TABLE t(id);"], + ), + ( + """ + CREATE TRIGGER t_ai AFTER INSERT ON t + BEGIN + UPDATE t SET value = 'a;b' WHERE id = new.id; + INSERT INTO log VALUES ('x;y'); + END; + """, + [ + "CREATE TRIGGER t_ai AFTER INSERT ON t\n" + " BEGIN\n" + " UPDATE t SET value = 'a;b' WHERE id = new.id;\n" + " INSERT INTO log VALUES ('x;y');\n" + " END;" + ], + ), + ), +) +def test_iter_complete_sql_statements(sql, expected): + assert list(_iter_complete_sql_statements(sql)) == expected + + +def test_atomic_commits(fresh_db): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + + assert list(fresh_db["dogs"].rows) == [{"id": 1, "name": "Cleo"}] + + +def test_atomic_rolls_back(fresh_db): + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + raise RuntimeError("boom") + + assert not fresh_db["dogs"].exists() + + +def test_nested_atomic_rolls_back_to_savepoint(fresh_db): + fresh_db["dogs"].create({"id": int, "name": str}, pk="id") + + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}) + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + raise RuntimeError("boom") + fresh_db["dogs"].insert({"id": 3, "name": "Marnie"}) + + assert list(fresh_db["dogs"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 3, "name": "Marnie"}, + ] + + +def test_outer_atomic_rolls_back_released_savepoint(fresh_db): + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes"}) + raise RuntimeError("boom") + + assert not fresh_db["dogs"].exists() + + +def test_executescript_does_not_commit_open_atomic_block(fresh_db): + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db.executescript(""" + CREATE TABLE dogs(id INTEGER PRIMARY KEY, name TEXT); + CREATE TRIGGER dogs_ai AFTER INSERT ON dogs + BEGIN + UPDATE dogs SET name = upper(new.name) || '; updated' WHERE id = new.id; + END; + -- This comment has a semicolon; + INSERT INTO dogs VALUES (1, 'Cleo; the first'); + """) + raise RuntimeError("boom") + + assert not fresh_db["dogs"].exists() + + +def test_transform_does_not_commit_open_atomic_block(fresh_db): + fresh_db["dogs"].insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") + + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["dogs"].insert({"id": 2, "name": "Pancakes", "age": "6"}) + fresh_db["dogs"].transform(rename={"age": "dog_age"}) + raise RuntimeError("boom") + + assert ( + fresh_db["dogs"].schema + == 'CREATE TABLE "dogs" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" TEXT\n)' + ) + assert list(fresh_db["dogs"].rows) == [ + {"id": 1, "name": "Cleo", "age": "5"}, + ] + + +def test_transform_parent_table_with_foreign_keys_in_atomic(fresh_db): + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db["books"].insert( + {"id": 1, "title": "Book", "author_id": 1}, + pk="id", + foreign_keys={"author_id"}, + ) + + with fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "full_name"}) + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + assert ( + fresh_db["authors"].schema + == 'CREATE TABLE "authors" (\n "id" INTEGER PRIMARY KEY,\n "full_name" TEXT\n)' + ) + assert fresh_db.execute("PRAGMA foreign_key_check").fetchall() == [] + + +def test_transform_parent_table_with_foreign_keys_rolls_back(fresh_db): + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db["books"].insert( + {"id": 1, "title": "Book", "author_id": 1}, + pk="id", + foreign_keys={"author_id"}, + ) + + with pytest.raises(RuntimeError): + with fresh_db.atomic(): + fresh_db["authors"].transform(rename={"name": "full_name"}) + raise RuntimeError("boom") + + assert ( + fresh_db["authors"].schema + == 'CREATE TABLE "authors" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT\n)' + ) + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + assert fresh_db.execute("PRAGMA foreign_key_check").fetchall() == [] + + +def test_transform_detects_foreign_key_check_violations(fresh_db): + fresh_db.conn.execute("PRAGMA foreign_keys=ON") + fresh_db["authors"].insert({"id": 1, "name": "Tina"}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 2}, pk="id") + + with pytest.raises(sqlite3.IntegrityError): + fresh_db["books"].transform(add_foreign_keys=(("author_id", "authors", "id"),)) + + assert fresh_db["books"].foreign_keys == [] + assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] From 1a28416e1095f2b8015195db7f8273b6f7953ab6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 15:54:19 -0700 Subject: [PATCH 005/120] Fix for detect_fts failing on [], refs #694 --- sqlite_utils/db.py | 2 +- tests/test_fts.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 677a1f8..d447cd2 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2776,7 +2776,7 @@ class Table(Queryable): ) """).strip() args = { - "like": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), + "like": "%VIRTUAL TABLE%USING FTS%content=[{}]%".format(self.name), "like2": '%VIRTUAL TABLE%USING FTS%content="{}"%'.format(self.name), "table": self.name, } diff --git a/tests/test_fts.py b/tests/test_fts.py index 9c635ff..ba4b32c 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -420,6 +420,29 @@ def test_enable_fts_replace_does_nothing_if_args_the_same(): assert all(q[0].startswith("select ") for q in queries) +def test_enable_fts_replace_handles_legacy_bracket_quoted_content_table(): + db = Database(memory=True) + db["books"].insert( + { + "id": 1, + "title": "Habits of Australian Marsupials", + "author": "Marlee Hawkins", + }, + pk="id", + ) + db.executescript(""" + CREATE VIRTUAL TABLE [books_fts] USING FTS5 ( + [title], + content=[books] + ); + """) + + db["books"].enable_fts(["title", "author"], replace=True) + + assert db["books_fts"].columns_dict.keys() == {"title", "author"} + assert 'content="books"' in db["books_fts"].schema + + def test_enable_fts_error_message_on_views(): db = Database(memory=True) db.create_view("hello", "select 1 + 1") From 2b0cc04c8def113d176cbc95630b665bdb05d6d9 Mon Sep 17 00:00:00 2001 From: Rami Abdelrazzaq Date: Sun, 21 Jun 2026 17:57:06 -0500 Subject: [PATCH 006/120] Fix issue #702: Handle CSV with only header row in --detect-types (#707) When inserting a CSV file that contains only a header row (no data rows), the --detect-types option would crash with an AssertionError because it tried to transform a table that didn't exist. This fix adds a check to ensure the table exists before attempting to apply type transformations. The fix is applied in two places: 1. insert_upsert_implementation() for the insert command 2. memory() command for CSV/TSV files Added test case: test_insert_csv_headers_only Co-authored-by: Test User --- sqlite_utils/cli.py | 4 ++-- tests/test_cli_insert.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 1c3ce8d..2f6268f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1177,7 +1177,7 @@ def insert_upsert_implementation( ) else: raise - if tracker is not None: + if tracker is not None and db.table(table).exists(): db.table(table).transform(types=tracker.types) # Clean up open file-like objects @@ -2034,7 +2034,7 @@ def memory( rows = (_flatten(row) for row in rows) db.table(file_table).insert_all(rows, alter=True) - if tracker is not None: + if tracker is not None and db.table(file_table).exists(): db.table(file_table).transform(types=tracker.types) # Add convenient t / t1 / t2 views view_names = ["t{}".format(i + 1)] diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 9f21b00..5812851 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -597,3 +597,21 @@ def test_insert_streaming_batch_size_1(db_path): proc.stdin.close() proc.wait() assert proc.returncode == 0 + + +def test_insert_csv_headers_only(tmpdir): + """Test that CSV with only header row (no data) works with --detect-types (issue #702)""" + db_path = str(tmpdir / "test.db") + csv_path = str(tmpdir / "headers_only.csv") + with open(csv_path, "w") as fp: + fp.write("id,name,age\n") + # Should not crash with --detect-types (which is now the default) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "data", csv_path, "--csv"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + # Table should not exist since there were no data rows + db = Database(db_path) + assert not db["data"].exists() From 733a67490feb99ea256ae97ff5e9ec23fa872ed0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 16:04:45 -0700 Subject: [PATCH 007/120] Test for CSV headers with memory command, refs #702, #707 --- tests/test_cli_memory.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index cf61985..2ed4aaa 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -156,6 +156,24 @@ def test_memory_csv_encoding(tmpdir, use_stdin): } +def test_memory_csv_headers_only(tmpdir): + csv_path = str(tmpdir / "headers_only.csv") + with open(csv_path, "w") as fp: + fp.write("id,name,age\n") + + result = CliRunner().invoke( + cli.cli, + ["memory", csv_path, "", "--schema"], + catch_exceptions=False, + ) + + assert result.exit_code == 0 + assert result.output.strip() == ( + 'CREATE VIEW "t1" AS select * from "headers_only";\n' + 'CREATE VIEW "t" AS select * from "headers_only";' + ) + + @pytest.mark.parametrize("extra_args", ([], ["select 1"])) def test_memory_dump(extra_args): result = CliRunner().invoke( From f448b61f5ec3c28758eb6b3801bb705b921f763a Mon Sep 17 00:00:00 2001 From: Rami Abdelrazzaq Date: Sun, 21 Jun 2026 18:14:45 -0500 Subject: [PATCH 008/120] fix: Plugins are still loaded when running tests (#719) * fix: defer plugin entrypoint loading until runtime Fixes simonw/sqlite-utils#713 * Lazy-load plugins when listing them Keep plugin entrypoints from loading at module import time, but preserve the existing behavior of get_plugins() by loading entrypoints the first time plugins are listed outside tests. --------- Co-authored-by: Simon Willison --- sqlite_utils/cli.py | 3 ++- sqlite_utils/db.py | 3 ++- sqlite_utils/plugins.py | 10 ++++++++-- tests/test_plugins.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 2f6268f..f15850d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -16,7 +16,7 @@ from sqlite_utils.db import ( NoTable, quote_identifier, ) -from sqlite_utils.plugins import pm, get_plugins +from sqlite_utils.plugins import ensure_plugins_loaded, pm, get_plugins from sqlite_utils.utils import maximize_csv_field_size_limit from sqlite_utils import recipes import textwrap @@ -3412,6 +3412,7 @@ def plugins_list(): click.echo(json.dumps(get_plugins(), indent=2)) +ensure_plugins_loaded() pm.hook.register_commands(cli=cli) cli.add_command(migrate) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d447cd2..2005210 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -41,7 +41,7 @@ from typing import ( Tuple, ) import uuid -from sqlite_utils.plugins import pm +from sqlite_utils.plugins import ensure_plugins_loaded, pm try: iterdump = importlib.import_module("sqlite_dump").iterdump @@ -401,6 +401,7 @@ class Database: self._registered_functions: set = set() self.use_counts_table = use_counts_table if execute_plugins: + ensure_plugins_loaded() pm.hook.prepare_connection(conn=self.conn) self.strict = strict diff --git a/sqlite_utils/plugins.py b/sqlite_utils/plugins.py index 457a907..0aff7ff 100644 --- a/sqlite_utils/plugins.py +++ b/sqlite_utils/plugins.py @@ -6,13 +6,19 @@ from . import hookspecs pm: pluggy.PluginManager = pluggy.PluginManager("sqlite_utils") pm.add_hookspecs(hookspecs) +_plugins_loaded = False -if not getattr(sys, "_called_from_test", False): - # Only load plugins if not running tests + +def ensure_plugins_loaded() -> None: + global _plugins_loaded + if _plugins_loaded or getattr(sys, "_called_from_test", False): + return pm.load_setuptools_entrypoints("sqlite_utils") + _plugins_loaded = True def get_plugins() -> List[Dict[str, Union[str, List[str]]]]: + ensure_plugins_loaded() plugins: List[Dict[str, Union[str, List[str]]]] = [] plugin_to_distinfo = dict(pm.list_plugin_distinfo()) for plugin in pm.get_plugins(): diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 1d459c9..c793e32 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -2,6 +2,7 @@ from click.testing import CliRunner import click import importlib import pytest +import sys from sqlite_utils import cli, Database, hookimpl, plugins @@ -16,6 +17,36 @@ def _supports_pragma_function_list(): db.close() +def test_get_plugins_loads_setuptools_entrypoints_once(monkeypatch): + calls = [] + monkeypatch.delattr(sys, "_called_from_test", raising=False) + monkeypatch.setattr(plugins, "_plugins_loaded", False) + monkeypatch.setattr( + plugins.pm, + "load_setuptools_entrypoints", + lambda group: calls.append(group) or 0, + ) + + plugins.get_plugins() + plugins.get_plugins() + + assert calls == ["sqlite_utils"] + + +def test_get_plugins_does_not_load_setuptools_entrypoints_in_tests(monkeypatch): + calls = [] + monkeypatch.setattr(sys, "_called_from_test", True, raising=False) + monkeypatch.setattr(plugins, "_plugins_loaded", False) + monkeypatch.setattr( + plugins.pm, + "load_setuptools_entrypoints", + lambda group: calls.append(group) or 0, + ) + + assert plugins.get_plugins() == [] + assert calls == [] + + def test_register_commands(): importlib.reload(cli) assert plugins.get_plugins() == [] From c754a0ebafff00aa4ac6c80e482c914eb1544265 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 16:26:41 -0700 Subject: [PATCH 009/120] Release 4.0rc1 --- docs/changelog.rst | 23 ++++++++++++++++------- pyproject.toml | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1fa3ac8..765b0e0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,13 +4,22 @@ Changelog =========== -.. _v_unreleased: +.. _v4_0rc1: -Unreleased ----------- +4.0rc1 (2026-06-21) +------------------- -- New ``db.atomic()`` context manager for transactions, with nested transaction support using SQLite savepoints. Internal multi-step operations such as ``table.transform()`` now use this mechanism to avoid unexpectedly committing an existing transaction. -- New :ref:`database migrations system `, incorporating functionality that was previously provided by the separate `sqlite-migrate `__ plugin. Define migration sets using the new :class:`sqlite_utils.Migrations` class and apply them using the ``sqlite-utils migrate`` command or the :ref:`migrations Python API `. (:issue:`752`) +- New :ref:`database migrations system `, incorporating functionality that was previously provided by the separate `sqlite-migrate `__ plugin. Define migration sets using the new :class:`sqlite_utils.Migrations` class and apply them using the ``sqlite-utils migrate`` command or the :ref:`migrations Python API `. (:issue:`752`) +- New ``db.atomic()`` :ref:`context manager providing nested transaction support ` using SQLite transactions and savepoints. Internal multi-step operations such as ``table.transform()`` now use this mechanism to avoid unexpectedly committing an existing transaction. (:issue:`755`) +- ``Database`` objects can now be :ref:`used as context managers `, automatically closing the connection when the ``with`` block exits. The CLI also now closes database and file handles more reliably, resolving a number of ``ResourceWarning`` warnings. (:issue:`692`) +- The ``sqlite-utils convert`` command can now accept a direct callable reference such as ``r.parsedate`` or ``json.loads --import json`` as the conversion code, as an alternative to calling it explicitly with ``r.parsedate(value)``. (:issue:`686`) +- Fixed a bug where CSV or TSV files with only a header row could crash ``sqlite-utils insert`` and ``sqlite-utils memory`` when type detection was enabled. Thanks, `Rami Abdelrazzaq `__. (:issue:`702`, `#707 `__) +- Fixed a bug where installed plugins could be loaded while running the test suite, despite the test-mode safeguard that disables plugin loading. Thanks, `Rami Abdelrazzaq `__. (:issue:`713`, `#719 `__) +- ``table.detect_fts()`` now recognizes legacy FTS virtual tables that quote the ``content=`` table name using square brackets, allowing ``table.enable_fts(..., replace=True)`` to replace them correctly. (:issue:`694`) +- Now depends on Click 8.3.1 or later, removing compatibility workarounds for Click's ``Sentinel`` default values. (:issue:`666`) +- Improved type annotations throughout the package, with ``ty`` now run in CI. (:issue:`697`) +- Development tooling now uses ``uv`` dependency groups, with separate ``dev`` and ``docs`` groups. (:issue:`691`) +- The test suite now runs against Python 3.15-dev. (:issue:`738`) .. _v3_39: @@ -18,8 +27,8 @@ Unreleased ----------------- - Fixed a bug with ``sqlite-utils install`` when the tool had been installed using ``uv``. (:issue:`687`) -- The ```--functions``` argument now optionally accepts a path to a Python file as an alternative to a string full of code, and can be specified multiple times - see :ref:`cli_query_functions`. (:issue:`659`) -- ``sqlite-utils`` now requires on Python 3.10 or higher. +- The ``--functions`` argument now optionally accepts a path to a Python file as an alternative to a string full of code, and can be specified multiple times - see :ref:`cli_query_functions`. (:issue:`659`) +- ``sqlite-utils`` now requires Python 3.10 or higher. .. _v4_0a1: diff --git a/pyproject.toml b/pyproject.toml index 5bfa492..6f11585 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.0a1" +version = "4.0rc1" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 401fb6949c5cd7bddd0b17fedc03455a30a7414c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 21 Jun 2026 16:29:32 -0700 Subject: [PATCH 010/120] Fix broken tracer test --- tests/test_tracer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_tracer.py b/tests/test_tracer.py index ac490c5..d14697d 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -64,7 +64,7 @@ def test_with_tracer(): " )\n" " )", { - "like": '%VIRTUAL TABLE%USING FTS%content="dogs"%', + "like": "%VIRTUAL TABLE%USING FTS%content=[dogs]%", "like2": '%VIRTUAL TABLE%USING FTS%content="dogs"%', "table": "dogs", }, From b5d0080cd16d1584bbffa3fcd2f5ccfb6cd27150 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jun 2026 12:00:05 -0700 Subject: [PATCH 011/120] Handle db.table(table_name).insert({}), closes #759 --- sqlite_utils/db.py | 20 +++++++++++++++++++- tests/test_default_value.py | 21 +++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 2005210..38e95c7 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3179,6 +3179,20 @@ class Table(Queryable): return a list of ``(sql, parameters)`` 2-tuples which, when executed in order, perform the desired INSERT / UPSERT / REPLACE operation. """ + # Dict-mode insert({}) has no explicit columns; SQLite spells that as + # DEFAULT VALUES. List mode with no columns is a different input shape. + if not list_mode and not all_columns: + or_clause = "" + if replace: + or_clause = " OR REPLACE" + elif ignore: + or_clause = " OR IGNORE" + sql = ( + f"INSERT{or_clause} INTO {quote_identifier(self.name)} " + "DEFAULT VALUES" + ) + return [(sql, []) for _ in chunk] + if hash_id_columns and hash_id is None: hash_id = "id" @@ -3603,7 +3617,11 @@ class Table(Queryable): assert ( num_columns <= SQLITE_MAX_VARS ), "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) - batch_size = max(1, min(batch_size, SQLITE_MAX_VARS // num_columns)) + batch_size = ( + 1 + if num_columns == 0 + else max(1, min(batch_size, SQLITE_MAX_VARS // num_columns)) + ) self.last_rowid = None self.last_pk = None if truncate and self.exists(): diff --git a/tests/test_default_value.py b/tests/test_default_value.py index c594c9f..c5e4b17 100644 --- a/tests/test_default_value.py +++ b/tests/test_default_value.py @@ -31,3 +31,24 @@ def test_quote_default_value(fresh_db, column_def, initial_value, expected_value assert expected_value == fresh_db.quote_default_value( fresh_db["foo"].columns[0].default_value ) + + +def test_insert_empty_record_uses_default_values(fresh_db): + fresh_db.execute(""" + CREATE TABLE has_defaults ( + id INTEGER PRIMARY KEY, + name TEXT, + timestamp TEXT DEFAULT CURRENT_TIMESTAMP, + is_active INTEGER NOT NULL DEFAULT 1 + ) + """) + + table = fresh_db["has_defaults"] + table.insert({}) + + rows = list(table.rows) + assert len(rows) == 1 + assert rows[0]["id"] == 1 + assert rows[0]["name"] is None + assert rows[0]["timestamp"] is not None + assert rows[0]["is_active"] == 1 From bfd74a35bb055a2dd28da8eabd149e4e03f30fed Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 24 Jun 2026 10:43:53 -0700 Subject: [PATCH 012/120] upsert() detects existing compound primary keys Closes #629 --- sqlite_utils/db.py | 5 +++++ tests/test_upsert.py | 47 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 38e95c7..ae99322 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3554,6 +3554,11 @@ class Table(Queryable): if hash_id_columns and hash_id is None: hash_id = "id" + if upsert and not pk and not hash_id and self.exists(): + existing_pks = [column.name for column in self.columns if column.is_pk] + if existing_pks: + pk = existing_pks[0] if len(existing_pks) == 1 else tuple(existing_pks) + if upsert and (not pk and not hash_id): raise PrimaryKeyRequired("upsert() requires a pk") diff --git a/tests/test_upsert.py b/tests/test_upsert.py index de0e362..09783bb 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -49,6 +49,53 @@ def test_upsert_error_if_no_pk(fresh_db): table.upsert({"id": 1, "name": "Cleo"}) +def test_upsert_error_if_existing_table_has_no_pk(fresh_db): + table = fresh_db.create_table("table", {"id": int, "name": str}) + with pytest.raises(PrimaryKeyRequired): + table.upsert({"id": 1, "name": "Cleo"}) + + +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_uses_compound_pk_from_existing_table(use_old_upsert): + # https://github.com/simonw/sqlite-utils/issues/629 + db = Database(memory=True, use_old_upsert=use_old_upsert) + db.execute(""" + create table summary ( + Source text, + Object text, + Category text, + Count integer, + primary key (Source, Object, Category) + ) + """) + table = db["summary"] + table.upsert( + { + "Source": "Client A", + "Object": "Accounts", + "Category": "All", + "Count": 3, + } + ) + assert table.last_pk == ("Client A", "Accounts", "All") + table.upsert( + { + "Source": "Client A", + "Object": "Accounts", + "Category": "All", + "Count": 4, + } + ) + assert list(table.rows) == [ + { + "Source": "Client A", + "Object": "Accounts", + "Category": "All", + "Count": 4, + } + ] + + def test_upsert_with_hash_id(fresh_db): table = fresh_db["table"] table.upsert({"foo": "bar"}, hash_id="pk") From e71b4ca3ee1af796f8e86fe3be36ae073c0e4ee8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 24 Jun 2026 13:11:29 -0700 Subject: [PATCH 013/120] Ran cog --- docs/cli-reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 64a27c2..48d0145 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -1191,7 +1191,7 @@ See :ref:`cli_add_column`. :: Usage: sqlite-utils add-column [OPTIONS] PATH TABLE COL_NAME - [[integer|int|float|real|text|str|blob|bytes]] + [integer|int|float|real|text|str|blob|bytes] Add a column to the specified table From 79117b9d110d72f46dab5fe2cda412ff4789ab55 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 24 Jun 2026 13:31:52 -0700 Subject: [PATCH 014/120] Pin click>=8.4.2 in dev dependencies to fix Cog Closes #761 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 6f11585..2c86f85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ [dependency-groups] dev = [ "black>=26.3.1", + "click>=8.4.2", "cogapp", "hypothesis", "pytest", From 0c369a447eeaf39084f0d14a45b3eeb7eacb631b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 17:33:23 +0000 Subject: [PATCH 015/120] Add 4.0rc1 pre-release review notes Consolidated findings from a final review before the stable 4.0 release: five release blockers, semantic decisions to lock in now, polish items, and verified-sound areas. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- fable-review-4.0rc1.md | 375 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 fable-review-4.0rc1.md diff --git a/fable-review-4.0rc1.md b/fable-review-4.0rc1.md new file mode 100644 index 0000000..c590b10 --- /dev/null +++ b/fable-review-4.0rc1.md @@ -0,0 +1,375 @@ +# sqlite-utils 4.0rc1 pre-release review + +Final review before the stable 4.0 release, focused on issues that would be +**breaking changes if fixed after 4.0 ships**. Reviewed at commit `79117b9` +(4.0rc1 plus four commits). Every finding below was verified against the +actual code; the two most serious were reproduced end-to-end. + +Test suite at review time: **1080 passed, 16 skipped**. + +**Verdict: do not tag stable yet.** There are five release blockers, all of +which are small fixes, plus a handful of semantic decisions that are cheap to +make now and breaking to change later. Recommended path: fix the blockers, +make the documented decisions, cut an **rc2**. + +--- + +## Release blockers + +Data loss or wrong-by-default behavior that 4.0 would lock in. + +### 1. `delete_where()` never commits and poisons the connection (data loss) + +`Table.delete_where()` (`sqlite_utils/db.py:2948`) runs its DELETE via a bare +`self.db.execute()` with no `atomic()` wrapper — compare `Table.delete()` at +`db.py:2944`, which wraps correctly. The connection is left +`in_transaction=True`, so every *subsequent* `atomic()` call takes the +savepoint branch (`db.py:430-440`) and never commits either. + +Reproduced end-to-end: + +```python +db = sqlite_utils.Database("dw.db") +db["t"].insert_all([{"id": i} for i in range(3)], pk="id") +db["t"].delete_where("id = ?", [0]) # conn.in_transaction is now True +db["t"].insert({"id": 50}) +db["u"].insert({"a": 1}) +db.close() +# Reopen: rows are [0, 1, 2] — the delete, row 50, AND table u are all gone. +``` + +In 3.x this leak was latent because later operations used `with db.conn:` +which committed the pending work (verified against 3.38, where the same +sequence persists everything). The 4.0 `atomic()` design — "don't commit an +existing transaction" — converts the old latent leak into permanent silent +data loss. + +**`optimize()` (`db.py:2790`) and `rebuild_fts()` (`db.py:2752`) have the +identical leak** — both run `INSERT INTO fts(fts) VALUES(...)` via bare +`self.db.execute()`. The CLI escapes only because `cli.py:341` and +`cli.py:369` wrap the calls in `with db.conn:`. + +Tellingly, the only `delete_where` example in the docs +(`docs/python-api.rst:983-990`) already wraps it in `with db.atomic():`, +papering over the bug, while every other single-op example needs no wrapper. + +**Fix:** wrap all three in `with self.db.atomic():`; update the docs example. +These were the only leaky public write ops found — every other op +(insert/upsert/update/delete/lookup/transform/extract/create/add_column/ +create_index/enable_fts/enable_counts/m2m/convert/duplicate) was verified to +leave `in_transaction=False`. + +### 2. `drop-view` drops tables and `drop-table` drops views, silently + +`cli.py:1728` (`drop-table`) and `cli.py:1800` (`drop-view`) both use +`db[name].drop()`. `Database.__getitem__` dispatches on the actual object +type, and `Table.drop()`/`View.drop()` each issue their matching `DROP` +statement. + +Reproduced: `sqlite-utils drop-view t.db t` deleted **table `t`** with exit +code 0. The mirror case (`drop-table` on a view) also succeeds silently. + +This is a data-loss footgun that contradicts the issue #657 table/view split +that is a headline 4.0 breaking change. Fixing it later converts a silent +"success" into an error — a semver problem, so it must land in 4.0. + +**Fix:** one line each — use `db.table(name)` / `db.view(name)` and catch +`NoTable` / `NoView` for a clean `ClickException`. + +### 3. Post-rc1 `insert({})` change lets `upsert` silently insert rows + +The new `DEFAULT VALUES` branch (commit `b5d0080`, +`db.py:3184-3194`) checks `not list_mode and not all_columns` but never +checks the `upsert` flag: + +- `db["t"].upsert({}, pk="id")` executes `INSERT INTO t DEFAULT VALUES` — a + brand-new row is written — and *then* raises an unrelated `KeyError: 'id'` + from the `last_pk` computation (`db.py:3742-3746`). +- `upsert_all([{}, {}], pk="id")` (2+ records skips the `last_pk` path) + **silently inserts two new rows on every call, no error at all**. + +At rc1 both failed fast with `ZeroDivisionError` — ugly, but zero rows +written. Going from crash-without-mutation to silent insertion under +"upsert" semantics is a regression 4.0 would lock in. + +Relatedly, the compound-pk auto-detection (commit `bfd74a3`, +`db.py:3557-3560`) makes records that *omit* the pk value reachable where +they previously raised `PrimaryKeyRequired` before touching the database: +`upsert_all([{"v": "a"}, {"v": "b"}])` on an existing pk table now inserts +with `id=NULL` (never conflicts) and appends new rows on every call +(verified: two calls → 4 rows). Same for compound pks with a missing +component. Not covered by any test. + +**Fix:** raise a clean error in the empty-record and missing-pk-value paths +when `upsert=True` (an empty record has no pk value, so it can never be an +upsert). + +### 4. `Migrations.apply()` transaction semantics are accidental — decide now + +`sqlite_utils/migrations.py:84-95` runs each migration function and its +tracking-row insert with no transaction wrapper. Verified: a migration that +fails halfway leaves its partial side effects committed, records nothing, +and re-running re-executes the *entire* function including already-applied +statements — the classic double-apply hazard. + +This matches sqlite-migrate, so either behavior is defensible — but 4.0 +freezes the contract: adding per-migration transactions in 4.1 would break +migrations that manage their own transactions or run statements illegal +inside one (`VACUUM`, some `PRAGMA`s). + +**Decide now:** wrap each migration in `db.atomic()`, or explicitly document +"migrations are not transactional; write idempotent steps" so the current +behavior is the contract rather than an accident. + +Also decide now: **`_AppliedMigration.applied_at` is annotated +`datetime.datetime` (`migrations.py:21`) but is a `str` at runtime** +(`migrations.py:67` passes the TEXT column value straight through), and the +type is published via autodoc (`docs/migrations.rst:168-171`). Changing the +annotation to `str` now is a free one-liner (and the sqlite-migrate- +compatible choice); "fixing" it to a real datetime later breaks every +consumer doing string operations. + +### 5. `enable_wal()` / `disable_wal()` commit open transactions + +`ensure_autocommit_off()` (`db.py:456-472`) assigns `conn.isolation_level`, +and CPython's setter **commits any pending transaction** as a side effect. +Verified consequences: + +- `with db.atomic(): insert(2); db.enable_wal(); insert(3); raise` → **all + rows persist despite the exception**, directly contradicting the + documented rollback guarantee (`docs/python-api.rst:255`), which is *the* + headline 4.0 semantic. +- A user's own `BEGIN` + insert + `enable_wal()` → their subsequent + `rollback()` is a no-op; the insert persists. This is exactly the + "unexpectedly committing an existing transaction" bug class `atomic()` was + introduced to eliminate (changelog, issue #755). + +Pre-existing in 3.x, but 4.0 is the moment to fix. + +**Fix:** make `enable_wal`/`disable_wal` raise (or skip the isolation dance) +when `conn.in_transaction`. + +--- + +## Decisions to make now — cheap today, breaking after 4.0 + +### `Database.__enter__` / `__exit__` contract is unpinned + +`__exit__` only calls `self.close()` (`db.py:408-417`), which silently rolls +back uncommitted changes. Verified: a raw `db.execute("insert ...")` inside +`with Database(p) as db:` is discarded on exit. This diverges from +`sqlite3.Connection`'s own context manager (commits on success, does *not* +close). The docs (`docs/python-api.rst:139-145`) say only "automatically +close the connection" — no mention of rollback — and **no test anywhere uses +`with Database(...)`**, so the contract is completely unpinned. Whichever way +this might later be "improved" is a breaking behavior change. Decide now, +add one docs sentence ("any uncommitted changes are rolled back") and a +pinning test. Note blockers 1–2 make this worse: after a `delete_where`, +exiting the `with` block discards everything. + +### `atomic()` is broken on Python 3.12+ `autocommit` connections + +`db.py:441-453` uses `conn.commit()`/`conn.rollback()`, documented no-ops +when `Connection.autocommit=True`. Verified on 3.13: + +- `autocommit=True` connection → a successful `atomic()` block leaves the + transaction permanently open; rollback on exception is also a no-op. +- `autocommit=False` connection → the connection is always in a transaction, + so `atomic()` always takes the savepoint branch and never commits + anything. + +`Database(filename_or_conn)` accepts arbitrary connections (`db.py:397`). +Either handle these modes (issue literal `COMMIT`/`ROLLBACK` statements, or +reject such connections) or document supported connection modes in +`python_api_atomic` now — changing commit timing later is breaking. +(Legacy `isolation_level=None` connections were verified to work correctly.) + +### `atomic()` savepoint-branch semantics are intended but undocumented + +If the connection is already in a transaction (user ran `BEGIN` or raw DML), +`atomic()` becomes a savepoint and its successful exit does **not** persist +anything until the user commits (verified: a user `rollback()` discards the +atomic block's writes). Reasonable semantics, but the `conn.in_transaction` +sniffing must be documented before it is locked — the docs currently +describe nesting only in terms of `atomic()`-inside-`atomic()`. + +### `atomic()` issues plain deferred `BEGIN` + +`db.py:442` — no way to request `BEGIN IMMEDIATE`. A future switch of the +default would change locking/`SQLITE_BUSY` behavior (breaking); adding an +`immediate=` parameter later is fine. Documenting "deferred" now costs one +sentence. + +### No-op `-d/--detect-types` flag: keep or kill + +`cli.py:942-947` still defines `-d/--detect-types` on `insert`/`upsert` +(help says "(default)"), deliberately tested as a no-op +(`tests/test_cli.py:2299,2322`). The `detect_types` parameter of +`insert_upsert_implementation` (`cli.py:1002`) is dead code — the body only +reads `no_detect_types`. If the flag is ever to be removed, 4.0 is the only +chance; keeping it as back-compat is defensible, but note +`--detect-types --no-detect-types` together silently favors the latter with +no conflict error. + +### `--stop-before` is silently ignored for old `sqlite_migrate` objects + +The duck-typing in `_compatible_migration_set` (`cli.py:3286-3289`) exists +precisely so migration files still doing `from sqlite_migrate import +Migrations` keep working — but `cli.py:3393` always passes `stop_before=` as +a **list**, and the old plugin's `apply()` does `if name == stop_before:` +against a string. Verified with the released 0.1b0: `--stop-before step2` +applied `step2` anyway. The CLI applies the exact migration the user asked +it not to, on the one upgrade path the duck typing exists to support. + +Related: a **typo'd `--stop-before` name silently applies everything**, +including the migration you meant to stop before (unknown names simply never +match; no error from the CLI). Validation has to live in the CLI ("each +`--stop-before` value must match at least one set") because unqualified +names legitimately fan out across sets. Adding that error post-4.0 turns +currently-succeeding invocations into failures — decide now. + +### Public-API validation via bare `assert` + +User-facing errors raised via `assert` throughout `db.py` (370, 396, 1016, +1950, 2829, 3028, 3565, 3571, 3967, …). They vanish under `python -O`, and +`db.py:3028` carries `# TODO: Test this works (rolls back) - use better +exception:`. Converting `AssertionError` to `ValueError` later changes +exception types callers may catch — best done in a major, if ever. + +--- + +## Should-fix polish (non-breaking later, but ugly for a stable release) + +- **`insert`/`upsert` into a view name prints a raw traceback** — + `NoTable("Table v is actually a view")` from `cli.py:1154` is not + converted to `ClickException` (contrast `duplicate` at `cli.py:1676`). +- **Misleading `NoView` message** — `db.view("t")` where `t` is a table says + "View t does not exist" (`db.py:661`); the mirror case helpfully says + "Table v is actually a view" (`db.py:650`). +- **Stale `__getitem__` docstring** (`db.py:502`) says it "returns a Table + object"; it returns a `View` for views. Feeds autodoc. +- **Wrong `hash_id` doc in the `Table` docstring** (`db.py:1588`) — says + bool; it's a column-name string (`Optional[str]`). +- **`sqlite-utils migrate --list` writes to the database** — + `pending()`/`applied()` call `ensure_migrations_table` + (`migrations.py:48,65`), so `--list` creates the tracking table, performs + the one-way legacy sqlite-migrate schema upgrade, and — since `db_path` + has no `exists=True` (`cli.py:3338-3340`) — creates the database file + itself. A read-looking operation should not do any of that. +- **`applied()` has no `ORDER BY`** (`migrations.py:66-71`) — relies on + rowid order; `tests/test_cli_migrate.py:104-107` asserts that order. + Add `order_by="id"` to make `--list` deterministic by contract. +- **Duplicate migration names within a set**: both functions execute (side + effects committed) before an opaque `IntegrityError`; only the first is + recorded (`migrations.py:36-42`). A `ValueError` at registration would be + cheap and is effectively non-breaking to add now. +- **`pending()`/`applied()` return underscore-private dataclasses** + (`_Migration`/`_AppliedMigration`) that are excluded from autodoc + (`docs/migrations.rst:171`), so the `.name`/`.applied_at`/`.fn` fields + users must access are undocumented API. +- **`set:name` colon syntax is unvalidated** (`cli.py:3328` uses + `partition(":")`) — a set or migration name containing `:` can never be + targeted. Document/validate "no colons in names" now. +- **Bare `@migrations` decorator** gives an opaque `TypeError` + (`migrations.py:30`); a helpful message would be purely additive. +- **Tracer never sees transaction statements** — `atomic()` uses + `self.conn.execute()`/`conn.commit()` directly (`db.py:432-450`), + bypassing the tracer. Routing through `self.execute()` later would change + tracer output that downstream tests may assert on — cheap to decide now. +- **Order-dependent empty-dict semantics** (from `b5d0080`): + `insert_all([{}, {"v": "hi"}])` gives the empty record its column DEFAULTs, + but `insert_all([{"v": "hi"}, {}])` inserts explicit `NULL` for it — a + `NOT NULL ... DEFAULT` column succeeds in one ordering and raises + `IntegrityError` in the other. +- **Batch-size cliff when the first record is `{}`** (`db.py:3625-3629`) — + `num_columns` comes from the first record only, forcing `batch_size=1` + for the entire stream. Performance-only, degenerate input. +- **`pyproject.toml` build-system underpinned** — `requires = ["setuptools"]` + while the PEP 639 `license = "Apache-2.0"` string needs setuptools ≥ + 77.0.3; non-isolated sdist builds on older setuptools will fail. Pin + `setuptools>=77`. +- **Undocumented post-rc1 behavior changes** — neither upsert pk + auto-detection (`bfd74a3`) nor `insert({})`-uses-DEFAULT-VALUES + (`b5d0080`) is mentioned in `docs/` or the changelog. Note also that + `db.table(name).insert({})` only works for *existing* tables — on a + missing table it still raises `AssertionError: Tables must have at least + one column`, so the issue #759 title scenario is only partially covered. + +--- + +## Verified sound — no action needed + +- **sqlite-migrate on-disk compatibility** (the scariest area): tested + against the actual 0.1b0 sdist from PyPI. Same `_sqlite_migrations` table; + the legacy compound-pk `(migration_set, name)` schema is detected via + `table.pks != ["id"]` and upgraded in place (`migrations.py:97-117`); rows + are preserved and previously-applied migrations are **not** re-applied. + Both timestamp formats are identical. Tests cover both legacy layouts + (`tests/test_migrations.py:83-110`). No data-corruption risk for upgrading + users. The built-in `migrate` command is registered *after* plugin hooks + (`cli.py:3415-3417`), so a still-installed old plugin's command is + correctly overridden. +- Migration ordering is definition order (documented and tested); identity + is `(set name, migration name)` with a unique index; same name across sets + is fine. The new CLI is a strict superset of the old plugin's; file + discovery is now `sorted()` where the plugin iterated an unordered set. +- Nested savepoint rollback/release logic including exception paths + (`tests/test_atomic.py:44-120`); commit-time deferred-FK failure rolls + back cleanly; `transform()` inside an open transaction correctly uses + `PRAGMA defer_foreign_keys`; `_executescript` statement-splitting avoids + sqlite3's implicit commit. Savepoint naming via `secrets.token_hex(16)` is + collision-free. +- Compound-pk upsert detection happy path works for single and compound pks + on both the `ON CONFLICT` and `use_old_upsert` implementations; explicit + `pk=` still wins over detection; rowid/missing/hash_id tables still raise + `PrimaryKeyRequired` (`tests/test_upsert.py:52-97`). +- The table/view split (#657) is otherwise complete: `db.table()` on a view + raises `NoTable`, never returns a `View`; `db.view()` accepts no + table-only kwargs. The tracer-test fix (`401fb69`) and click pin + (`79117b9`, dev-group only) are correct. +- `__all__` exports are coherent (`Database`, `Migrations`, + `suggest_column_types`, `hookimpl`, `hookspec`); every `sqlite_utils.X` + docs reference resolves. The `pip` runtime dependency is genuinely + required (`cli.py:2967-2983` uses `run_module("pip")` for + `install`/`uninstall`). Classifiers match `requires-python`. No + deprecation debt: zero `DeprecationWarning` markers; clean under + `-W error::DeprecationWarning`. + +--- + +## The transform() incoming-FK branch: hold for 4.1 + +The `update_incoming_fks` work on `claude/investigate-transform-fk-KMTZ7` is +right to hold: + +- It is purely additive (`update_incoming_fks: bool = False` keyword + + `--update-incoming-fks` flag; default behavior unchanged), so 4.1 is safe + semver-wise. +- The branch predates the `atomic()` refactor — it still uses + `with self.db.conn:` directly in `transform()`, the exact pattern 4.0 + eliminated (#755) — and has real issues: `_skip_fk_validation` is mutable + state on the whole `Database` object, and the incoming-FK rebuilds execute + `transform_sql()` output raw for referencing tables, silently dropping + their indexes (the 3.38 index-recreation logic only runs for the primary + table). +- One decision belongs to 4.0: whether `transform()` should eventually + refuse/warn *by default* when a rename would leave dangling incoming FKs. + Adding a warning later is fine; making it an error by default later is + breaking. If strict-by-default is the desired end state, add just the + guard in 4.0 and ship the auto-update machinery in 4.1. + +--- + +## Suggested path to stable + +1. Fix the five blockers — three `atomic()` wrappers (`delete_where`, + `optimize`, `rebuild_fts`), two CLI lookups (`drop-table`/`drop-view`), + the `upsert` guard in the DEFAULT VALUES / missing-pk paths, the + `applied_at: str` annotation, and the `in_transaction` guard in + `enable_wal`/`disable_wal`. +2. Make and document the decisions: `apply()` transactionality, + `Database.__exit__` semantics (+ pinning test), `atomic()` supported + connection modes and deferred-`BEGIN` note, `--detect-types` keep/kill, + `--stop-before` validation and old-plugin compat. +3. Cut **rc2** — the transaction-semantics fixes deserve one more candidate + round before the semantics freeze. From f3c012774a609999e65d49f032f354315dda6cb9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 17:39:12 +0000 Subject: [PATCH 016/120] Add review feedback for the create-table-parser branch Assessment of the CREATE TABLE parser as the foundation for 4.1 transform() improvements: fit for 4.1, two tokenizer bugs to fix first (comments, numeric literal defaults), test wiring, and design decisions around round-trip fidelity and API visibility. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- create-table-parser-feedback.md | 167 ++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 create-table-parser-feedback.md diff --git a/create-table-parser-feedback.md b/create-table-parser-feedback.md new file mode 100644 index 0000000..2b6f727 --- /dev/null +++ b/create-table-parser-feedback.md @@ -0,0 +1,167 @@ +# Review: `create-table-parser` branch + +Feedback on the `create-table-parser` branch (commits `ed1de15`, `95d6ff4`, +`407db59` — 819-line `create_table_parser.py` plus JSON test corpora), +reviewed as the proposed foundation for 4.1 `transform()` improvements that +make UNIQUE and CHECK constraints supported instead of silently dropped. + +**Verdict: good fit for 4.1 — right design, right release — but it has two +silent-corruption bugs that must be fixed before `transform()` can trust it, +and a few design decisions to settle.** + +## Why 4.1 is the right home + +- The parser is purely additive: a new module and a new capability. Nothing + about it needs to land in 4.0, so it does not constrain or delay the + stable release. +- The `transform()` improvement it enables — preserving CHECK and UNIQUE + constraints instead of silently dropping them — is a behavior change in + the bug-fix direction, appropriate for a minor release with a prominent + changelog note. +- SQLite exposes no introspection for CHECK constraints (and only partial + detail elsewhere), so parsing the `sqlite_master` DDL is the correct — and + only — approach. The module docstring says exactly this. + +## What holds up well (verified empirically) + +- **Corpus results:** all 272 statements in `tests/valid_create_table.json` + parse without crashing; the 230 that are independently executable were + cross-checked against SQLite's own `PRAGMA table_xinfo` — **0 column-name + or primary-key mismatches**. The 29 invalid statements also do not crash + the parser. +- Survived adversarial probing: identifier quoting in all four styles + including embedded escaped quotes (`"col ""x"""`, `[square]`, backticks), + strings containing `,)` and keywords inside CHECK expressions, + `col IN (...)` option extraction with escaped quotes, generated columns + (`GENERATED ALWAYS AS ... STORED` and bare `AS`), FK action clauses + (`ON DELETE SET NULL ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED`), + `WITHOUT ROWID, STRICT` trailers, keyword-named columns (`"unique"`), + multi-word types with parenthesized arguments, and `CREATE TABLE AS + SELECT`. +- The design is genuinely good: recursive-descent functions mirroring the + SQLite grammar are readable; the constraint list as source of truth with + derived accessor properties (`table.checks`, `col.not_null`, + `table.primary_key`) is the right model — adding a constraint kind means + adding a dataclass and a property. The group-capturing tokenizer + (parenthesized groups taken whole so nested commas never reach the parser) + is a clean way to sidestep expression parsing. + +## Must-fix before `transform()` builds on it + +### 1. Comments produce phantom columns (silent corruption) + +The tokenizer does not handle `--` line comments or `/* */` block comments, +and `sqlite_master` stores DDL verbatim. Reproduced: + +``` +CREATE TABLE t ( + a TEXT, -- user's name, (important) + b INTEGER +) +→ columns parsed as ['a', '-'] (real: ['a', 'b']) + +CREATE TABLE t (a TEXT /* legacy, do not use */, b INTEGER) +→ columns parsed as ['a', 'do', 'b'] (real: ['a', 'b']) +``` + +Hand-written schemas — exactly the ones that have CHECK constraints — are +exactly the ones with comments. If `transform()` rebuilds a table from a +model with wrong columns, that is data loss. Comment handling belongs in +`_tokenize` / `_locate_body` / `_split_top_level` (all three scan raw text). + +### 2. Numeric literal defaults are truncated (silent corruption) + +`_tokenize` splits `1.5` into `1` / `.` / `5` and `_default_value` consumes +one token. Reproduced: + +| DDL | parsed default | correct | +|----------------------|----------------|---------| +| `DEFAULT 1.5` | `'1'` | `1.5` | +| `DEFAULT -1.5` | `'-1'` | `-1.5` | +| `DEFAULT 1e-3` | `'1e'` | `1e-3` | +| `DEFAULT x'0102'` | `'x'` | `x'0102'` | + +Re-emitting these in a transform rewrites `DEFAULT 1.5` as `DEFAULT 1` — +silent schema corruption. Notably the corpus already contains +`DEFAULT -45.8e22`, but nothing catches this because the fixtures are +input-only (see next point). Fix by lexing numeric literals (including +sign, decimal point, exponent) and blob literals (`x'...'`) as single +tokens, or by making `_default_value` consume the full literal. + +### 3. No tests are actually executed + +The JSON corpora are not wired into pytest — there is no `test_*.py` on the +branch, so the parser currently has zero executed tests. Beyond wiring the +corpus in, add **expected-output snapshots** (parse each valid statement, +assert the full structured result), not just "doesn't crash": the +`DEFAULT -45.8e22` bug sat undetected in the corpus precisely because only +inputs are recorded. A cheap high-value addition: property test that for +every executable statement, parsed column names / pk / not-null match +`PRAGMA table_xinfo`. + +## Design decisions to settle + +### Discard-vs-error policy for grammar the model does not capture + +The plan is presumably parse → modify model → serialize, which matches how +`transform()` already rebuilds tables. That makes "what the model discards" +the critical list, because anything discarded is silently stripped from the +user's schema on transform: + +- `ON CONFLICT` clauses on PRIMARY KEY / UNIQUE / NOT NULL (parsed by + `_conflict_clause`, thrown away) +- FK `MATCH` and `DEFERRABLE INITIALLY DEFERRED` (parsed, thrown away — + deferred FKs are real in the wild) +- `ASC` / `DESC` in table-level `PRIMARY KEY (a DESC)` / `UNIQUE (...)` + column lists (`_column_list_group` keeps only the leading identifier; + inline single-column PK order *is* captured) +- Unknown column constraints (`_column_constraint` consumes one token and + returns `None`) + +Either capture these in the model, or have `transform()` raise +`TransformError` when the source DDL contains grammar it cannot round-trip — +the honest-failure pattern 3.38 established for un-recreatable indexes. +Silence is the one wrong answer. + +Same question for column renames: a renamed column referenced inside a CHECK +expression (stored as a raw string) cannot be mechanically rewritten, so +`transform(rename=...)` intersecting a check should raise `TransformError` +rather than emit a stale constraint. Likewise dropping a column referenced +by a table-level CHECK or multi-column UNIQUE. + +### Ship it private in 4.1 + +The module defines `Table`, `Column`, and `ForeignKey` — the package already +has `db.Table` and a `db.ForeignKey` namedtuple with different, incompatible +fields (`(table, column, other_table, other_column)` vs +`(columns, table, references, ...)`). Two public `ForeignKey` types in one +package is a confusion trap, and 4.0 is a fresh reminder that public surface +is forever. Recommendation: land it as a private module +(`sqlite_utils._parser` or similar) powering `transform()` in 4.1, promote +to documented public API in a later minor once battle-tested. Public-later +is additive; public-now-retract-later is breaking. + +Related naming nit: the parser's `Table.schema` is the attached-database +name (`main`/`temp`), while `db.Table.schema` is the DDL string. Rename one +(e.g. `database` or `schema_name`) before anything goes public. + +### Housekeeping + +- `create_table_parser.py` lives at the repo root; it needs to move inside + the `sqlite_utils/` package. +- `parse_checks` is documented as a "backwards-compatible helper" but + nothing in sqlite-utils exists for it to be compatible with — stale + comment from an earlier draft. +- The valid corpus is strong on grammar coverage but light on real-world + mess: add fixtures with comments, tabs/newlines in odd places, numeric + and blob defaults, and a couple of schemas generated by sqlite-utils + itself (double-quoted everything) and by common tools. + +## Bottom line + +Hold it for 4.1 exactly as planned — it has no 4.0 dependency and should not +delay the stable release. Before `transform()` builds on it: fix comment +handling and numeric/blob literal lexing, wire the fixtures into pytest with +expected-output snapshots, decide the discard-vs-error policy, and keep the +module private for one release. The bones are solid; both bugs are +tokenizer-level and small. From b17c37144eded6af1a326699e8fa732b039ff4be Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 17:57:35 +0000 Subject: [PATCH 017/120] Fix delete_where() leaving the connection in an open transaction Table.delete_where() ran its DELETE via a bare execute() with no commit, unlike Table.delete() which wraps in db.atomic(). The connection was left with an open implicit transaction, so the deletion (and all subsequent writes, including later atomic() blocks which switched to savepoint mode) was silently rolled back when the connection closed. Wrap the DELETE in db.atomic() and remove the now-unnecessary atomic() wrapper from the delete_where documentation example. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/python-api.rst | 3 +-- sqlite_utils/db.py | 3 ++- tests/test_delete.py | 18 ++++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index eab858e..b756605 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -984,8 +984,7 @@ You can delete all records in a table that match a specific WHERE statement usin >>> db = sqlite_utils.Database("dogs.db") >>> # Delete every dog with age less than 3 - >>> with db.atomic(): - >>> db.table("dogs").delete_where("age < ?", [3]) + >>> db.table("dogs").delete_where("age < ?", [3]) Calling ``table.delete_where()`` with no other arguments will delete every row in the table. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ae99322..5acab69 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2966,7 +2966,8 @@ class Table(Queryable): sql = "delete from {}".format(quote_identifier(self.name)) if where is not None: sql += " where " + where - self.db.execute(sql, where_args or []) + with self.db.atomic(): + self.db.execute(sql, where_args or []) if analyze: self.analyze() return self diff --git a/tests/test_delete.py b/tests/test_delete.py index 652096c..a2d93aa 100644 --- a/tests/test_delete.py +++ b/tests/test_delete.py @@ -1,3 +1,6 @@ +import sqlite_utils + + def test_delete_rowid_table(fresh_db): table = fresh_db["table"] table.insert({"foo": 1}).last_pk @@ -32,6 +35,21 @@ def test_delete_where_all(fresh_db): assert table.count == 0 +def test_delete_where_commits(tmpdir): + path = str(tmpdir / "test.db") + db = sqlite_utils.Database(path) + db["table"].insert_all([{"id": i} for i in range(5)], pk="id") + db["table"].delete_where("id > ?", [2]) + # The connection must not be left inside an open transaction, + # otherwise subsequent atomic() blocks never commit either + assert not db.conn.in_transaction + db["table"].insert({"id": 100}) + db.close() + db2 = sqlite_utils.Database(path) + assert [r["id"] for r in db2["table"].rows] == [0, 1, 2, 100] + db2.close() + + def test_delete_where_analyze(fresh_db): table = fresh_db["table"] table.insert_all(({"id": i, "i": i} for i in range(10)), pk="id") From d36639ed73a1e05e425dd3cfa666a389c9fa26d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 18:06:06 +0000 Subject: [PATCH 018/120] Run each migration inside a transaction Migrations.apply() now wraps each migration function and its _sqlite_migrations tracking row in db.atomic(), so they commit together: a failing migration rolls back cleanly, is not recorded, and stays pending, eliminating the double-apply hazard where a partially-failed migration left committed side effects behind. Migrations that cannot run inside a transaction (VACUUM, journal mode changes, manual transaction management) can opt out by registering with @migrations(transactional=False). Also fixed the test fixtures which used db.query() for INSERT statements - query() is a lazy generator, so those inserts never actually executed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/migrations.rst | 19 ++++++++++ sqlite_utils/migrations.py | 42 ++++++++++++++++------ tests/test_migrations.py | 74 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 123 insertions(+), 12 deletions(-) diff --git a/docs/migrations.rst b/docs/migrations.rst index 948b468..5628ded 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -70,6 +70,25 @@ When you apply a set of migrations you can stop part way through by specifying a migrations.apply(db, stop_before="add_weight") +.. _migrations_transactions: + +Migrations and transactions +=========================== + +Each migration runs inside a transaction, together with the ``_sqlite_migrations`` record of it having been applied. If a migration function raises an exception, everything it did is rolled back, no record is written and the migration stays pending - so fixing the error and re-applying will run that migration again from a clean state. Migrations that completed earlier in the same ``apply()`` run stay applied. + +Some operations cannot run inside a transaction, for example ``VACUUM`` or changing the journal mode with ``db.enable_wal()``. Register migrations like these with ``transactional=False``: + +.. code-block:: python + + @migrations(transactional=False) + def compact(db): + db.execute("VACUUM") + +A migration registered with ``transactional=False`` runs without a wrapping transaction, so if it fails part way through any changes it already made will not be rolled back, and re-applying will run the whole function again. + +Avoid calling ``db.conn.commit()`` or otherwise managing transactions manually inside a transactional migration - register the migration with ``transactional=False`` if it needs to control its own transactions. + Applying migrations using the CLI ================================= diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py index f349b17..8af0784 100644 --- a/sqlite_utils/migrations.py +++ b/sqlite_utils/migrations.py @@ -14,6 +14,7 @@ class Migrations: class _Migration: name: str fn: Callable + transactional: bool = True @dataclass class _AppliedMigration: @@ -27,15 +28,22 @@ class Migrations: self.name = name self._migrations: list[Migrations._Migration] = [] - def __call__(self, *, name: str | None = None) -> Callable: + def __call__( + self, *, name: str | None = None, transactional: bool = True + ) -> Callable: """ :param name: The name to use for this migration - if not provided, the name of the function will be used. + :param transactional: If ``True`` (the default) the migration and the + record of it having been applied are wrapped in a transaction, which + will be rolled back if the migration raises an exception. Pass + ``False`` for migrations that cannot run inside a transaction, for + example those that execute ``VACUUM``. """ def inner(func: Callable) -> Callable: self._migrations.append( - self._Migration(name or getattr(func, "__name__"), func) + self._Migration(name or getattr(func, "__name__"), func, transactional) ) return func @@ -73,6 +81,12 @@ class Migrations: def apply(self, db: "Database", *, stop_before: str | Iterable[str] | None = None): """ Apply any pending migrations to the database. + + Each migration runs inside a transaction, together with the record of + it having been applied - if the migration raises an exception its + changes are rolled back, no record is written and the migration stays + pending. Migrations registered with ``transactional=False`` run + outside of a transaction. """ self.ensure_migrations_table(db) if stop_before is None: @@ -85,14 +99,22 @@ class Migrations: name = migration.name if name in stop_before_names: return - migration.fn(db) - _table(db, self.migrations_table).insert( - { - "migration_set": self.name, - "name": name, - "applied_at": str(datetime.datetime.now(datetime.timezone.utc)), - } - ) + if migration.transactional: + with db.atomic(): + migration.fn(db) + self._record_applied(db, name) + else: + migration.fn(db) + self._record_applied(db, name) + + def _record_applied(self, db: "Database", name: str): + _table(db, self.migrations_table).insert( + { + "migration_set": self.name, + "name": name, + "applied_at": str(datetime.datetime.now(datetime.timezone.utc)), + } + ) def ensure_migrations_table(self, db: "Database"): """ diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 6a0dd33..34741a5 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -14,7 +14,7 @@ def migrations(): @migrations() def m002(db): db["cats"].create({"name": str}) - db.query("insert into dogs (name) values ('Pancakes')") + db.execute("insert into dogs (name) values ('Pancakes')") return migrations @@ -32,7 +32,7 @@ def migrations_not_ordered_alphabetically(): @migrations() def m001(db): db["cats"].create({"name": str}) - db.query("insert into dogs (name) values ('Pancakes')") + db.execute("insert into dogs (name) values ('Pancakes')") return migrations @@ -80,6 +80,76 @@ def test_order_does_not_matter(migrations, migrations_not_ordered_alphabetically assert db1.schema == db2.schema +def test_failing_migration_rolls_back(migrations): + @migrations() + def m003(db): + db["birds"].create({"name": str}) + db.execute("insert into dogs (name) values ('Dozer')") + raise ValueError("boom") + + db = sqlite_utils.Database(memory=True) + with pytest.raises(ValueError): + migrations.apply(db) + # m001 and m002 committed before the failure and stay applied + assert set(db.table_names()) == {"_sqlite_migrations", "dogs", "cats"} + assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"] + assert [m.name for m in migrations.applied(db)] == ["m001", "m002"] + # Everything m003 did was rolled back and it is still pending + assert [m.name for m in migrations.pending(db)] == ["m003"] + + +def test_rerun_after_failure_applies_each_migration_once(): + state = {"fail": True} + migrations = Migrations("test") + + @migrations() + def m001(db): + db["dogs"].insert({"name": "Cleo"}) + + @migrations() + def m002(db): + db["dogs"].insert({"name": "Pancakes"}) + if state["fail"]: + raise ValueError("boom") + + db = sqlite_utils.Database(memory=True) + with pytest.raises(ValueError): + migrations.apply(db) + state["fail"] = False + migrations.apply(db) + # m001 must not have been re-applied, m002 applied exactly once + assert [r["name"] for r in db["dogs"].rows] == ["Cleo", "Pancakes"] + assert [m.name for m in migrations.applied(db)] == ["m001", "m002"] + + +def test_non_transactional_migration_allows_vacuum(tmpdir): + path = str(tmpdir / "test.db") + db = sqlite_utils.Database(path) + migrations = Migrations("test") + + @migrations() + def m001(db): + db["dogs"].insert({"name": "Cleo"}) + + @migrations(transactional=False) + def m002(db): + db.execute("VACUUM") + + migrations.apply(db) + assert [m.name for m in migrations.applied(db)] == ["m001", "m002"] + db.close() + + +def test_apply_composes_inside_outer_transaction(migrations): + db = sqlite_utils.Database(memory=True) + with pytest.raises(ZeroDivisionError): + with db.atomic(): + migrations.apply(db) + raise ZeroDivisionError + # The outer transaction rolled back, taking the migrations with it + assert db.table_names() == [] + + @pytest.mark.parametrize( "create_table,pk", ( From 862944bc38381ea4cb23fe050e0481c4a7495447 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 18:13:50 +0000 Subject: [PATCH 019/120] drop-table and drop-view now refuse to drop the wrong object type drop-table on a view name silently dropped the view, and drop-view on a table name silently dropped the table, because both used db[name].drop() which dispatches on the actual object type. They now use db.table() / db.view() and exit with an explanatory error pointing at the correct command. --ignore still exits cleanly but no longer drops anything. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- sqlite_utils/cli.py | 19 ++++++++++++++++--- tests/test_cli.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index f15850d..d51ba7b 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -14,6 +14,7 @@ from sqlite_utils.db import ( DEFAULT, DescIndex, NoTable, + NoView, quote_identifier, ) from sqlite_utils.plugins import ensure_plugins_loaded, pm, get_plugins @@ -1725,7 +1726,13 @@ def drop_table(path, table, ignore, load_extension): _register_db_for_cleanup(db) _load_extensions(db, load_extension) try: - db[table].drop(ignore=ignore) + db.table(table).drop(ignore=ignore) + except NoTable: + # A view exists with this name + if not ignore: + raise click.ClickException( + '"{}" is a view, not a table - use drop-view to drop it'.format(table) + ) except OperationalError: raise click.ClickException('Table "{}" does not exist'.format(table)) @@ -1797,8 +1804,14 @@ def drop_view(path, view, ignore, load_extension): _register_db_for_cleanup(db) _load_extensions(db, load_extension) try: - db[view].drop(ignore=ignore) - except OperationalError: + db.view(view).drop(ignore=ignore) + except NoView: + if ignore: + return + if view in db.table_names(): + raise click.ClickException( + '"{}" is a table, not a view - use drop-table to drop it'.format(view) + ) raise click.ClickException('View "{}" does not exist'.format(view)) diff --git a/tests/test_cli.py b/tests/test_cli.py index 565fbc2..02c00ab 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1467,6 +1467,24 @@ def test_drop_table_error(): assert result.exit_code == 0 +def test_drop_table_on_view_errors(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db["t"].insert({"id": 1}) + db.create_view("v", "select * from t") + result = runner.invoke(cli.cli, ["drop-table", "test.db", "v"]) + assert result.exit_code == 1 + assert 'Error: "v" is a view, not a table - use drop-view to drop it' == ( + result.output.strip() + ) + assert "v" in db.view_names() + # --ignore exits cleanly but must still not drop the view + result = runner.invoke(cli.cli, ["drop-table", "test.db", "v", "--ignore"]) + assert result.exit_code == 0 + assert "v" in db.view_names() + + def test_drop_view(): runner = CliRunner() with runner.isolated_filesystem(): @@ -1485,6 +1503,23 @@ def test_drop_view(): assert "hello" not in db.view_names() +def test_drop_view_on_table_errors(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + db["t"].insert({"id": 1}) + result = runner.invoke(cli.cli, ["drop-view", "test.db", "t"]) + assert result.exit_code == 1 + assert 'Error: "t" is a table, not a view - use drop-table to drop it' == ( + result.output.strip() + ) + assert "t" in db.table_names() + # --ignore exits cleanly but must still not drop the table + result = runner.invoke(cli.cli, ["drop-view", "test.db", "t", "--ignore"]) + assert result.exit_code == 0 + assert "t" in db.table_names() + + def test_drop_view_error(): runner = CliRunner() with runner.isolated_filesystem(): From 45af93d46314db8af0b9621462d20f0840227b42 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 18:24:52 +0000 Subject: [PATCH 020/120] db.query() now executes its SQL immediately Previously query() was a generator function, so nothing - including the SQL itself - ran until the result was first iterated. A write statement passed to query() silently did nothing, and SQL errors surfaced far from the call site. The SQL now executes as soon as query() is called, while rows are still fetched lazily during iteration. Statements that return no rows now raise a ValueError directing callers to execute() instead. As a side effect, statements like INSERT ... RETURNING now work naturally with query(). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/python-api.rst | 4 ++++ sqlite_utils/db.py | 18 ++++++++++++++++-- tests/test_query.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index b756605..39361c9 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -219,6 +219,10 @@ The ``db.query(sql)`` function executes a SQL query and returns an iterator over # {'name': 'Cleo'} # {'name': 'Pancakes'} +The SQL query is executed as soon as ``db.query()`` is called. The resulting rows are fetched lazily as you iterate, so large result sets are not loaded into memory all at once. Because execution is immediate, an error in your SQL will raise an exception straight away, and a statement such as ``INSERT ... RETURNING`` will take effect even if you do not iterate over its results. + +``db.query()`` can only be used with SQL that returns rows. Passing a statement that returns no rows - an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause, for example - will raise a ``ValueError``. Use :ref:`db.execute() ` for those statements instead. + .. _python_api_execute: db.execute(sql, params) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 5acab69..63c798f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -593,14 +593,28 @@ class Database: """ Execute ``sql`` and return an iterable of dictionaries representing each row. + The SQL is executed as soon as this method is called - the resulting rows + are then fetched lazily as the returned iterable is iterated over. + :param sql: SQL query to execute :param params: Parameters to use in that query - an iterable for ``where id = ?`` parameters, or a dictionary for ``where id = :id`` + :raises ValueError: if the SQL statement does not return rows - use + :meth:`execute` for those statements instead """ cursor = self.execute(sql, params or tuple()) + if cursor.description is None: + raise ValueError( + "query() can only be used with SQL that returns rows - " + "use execute() for other statements" + ) keys = [d[0] for d in cursor.description] - for row in cursor: - yield dict(zip(keys, row)) + + def rows() -> Generator[dict, None, None]: + for row in cursor: + yield dict(zip(keys, row)) + + return rows() def execute( self, sql: str, parameters: Optional[Union[Sequence, Dict[str, Any]]] = None diff --git a/tests/test_query.py b/tests/test_query.py index fe79cc0..0b0b9c5 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1,3 +1,5 @@ +import pytest +import sqlite3 import types @@ -8,6 +10,33 @@ def test_query(fresh_db): assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}] +def test_query_executes_eagerly(fresh_db): + # The SQL runs when query() is called, not when the result is iterated, + # so errors are raised at the call site + with pytest.raises(sqlite3.OperationalError): + fresh_db.query("select * from missing_table") + + +def test_query_rejects_statements_that_return_no_rows(fresh_db): + fresh_db["dogs"].insert({"name": "Cleo"}) + with pytest.raises(ValueError) as ex: + fresh_db.query("update dogs set name = 'Cleopaws'") + assert "execute()" in str(ex.value) + + +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 35, 0), + reason="RETURNING requires SQLite 3.35.0 or higher", +) +def test_query_insert_returning(fresh_db): + fresh_db["dogs"].insert({"name": "Cleo"}) + rows = list( + fresh_db.query("insert into dogs (name) values ('Pancakes') returning name") + ) + assert rows == [{"name": "Pancakes"}] + assert fresh_db["dogs"].count == 2 + + def test_execute_returning_dicts(fresh_db): # Like db.query() but returns a list, included for backwards compatibility # see https://github.com/simonw/sqlite-utils/issues/290 From e762656d0687d86607c6c2acdefc086584f31d70 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 18:35:45 +0000 Subject: [PATCH 021/120] optimize() and rebuild_fts() now commit their changes Both ran their INSERT INTO fts(fts) statements via a bare execute() with no commit, leaving the connection inside an open implicit transaction - the FTS operation and all subsequent writes were then silently rolled back when the connection closed. Both are now wrapped in db.atomic(), matching delete_where() and the other write operations. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- sqlite_utils/db.py | 16 +++++++++------- tests/test_fts.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 63c798f..6f947e1 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2769,11 +2769,12 @@ class Table(Queryable): if fts_table is None: # Assume this is itself an FTS table fts_table = self.name - self.db.execute( - "INSERT INTO {table}({table}) VALUES('rebuild');".format( - table=quote_identifier(fts_table) + with self.db.atomic(): + self.db.execute( + "INSERT INTO {table}({table}) VALUES('rebuild');".format( + table=quote_identifier(fts_table) + ) ) - ) return self def detect_fts(self) -> Optional[str]: @@ -2805,9 +2806,10 @@ class Table(Queryable): "Run the ``optimize`` operation against the associated full-text search index table." fts_table = self.detect_fts() if fts_table is not None: - self.db.execute(""" - INSERT INTO {table} ({table}) VALUES ("optimize"); - """.strip().format(table=quote_identifier(fts_table))) + with self.db.atomic(): + self.db.execute(""" + INSERT INTO {table} ({table}) VALUES ("optimize"); + """.strip().format(table=quote_identifier(fts_table))) return self def search_sql( diff --git a/tests/test_fts.py b/tests/test_fts.py index ba4b32c..11df898 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -336,6 +336,24 @@ def test_rebuild_fts(fresh_db): assert len(rows2) == 2 +@pytest.mark.parametrize("method", ["optimize", "rebuild_fts"]) +def test_optimize_and_rebuild_fts_commit(tmpdir, method): + path = str(tmpdir / "test.db") + db = Database(path) + table = db["searchable"] + table.insert(search_records[0]) + table.enable_fts(["text", "country"]) + getattr(table, method)() + # The connection must not be left inside an open transaction, + # otherwise this and all subsequent writes are lost on close + assert not db.conn.in_transaction + table.insert(search_records[1]) + db.close() + db2 = Database(path) + assert db2["searchable"].count == 2 + db2.close() + + @pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"]) def test_rebuild_fts_invalid(fresh_db, invalid_table): fresh_db["not_searchable"].insert({"foo": "bar"}) From 286a87dba4b93c034614a1fe931fcf503b5e643d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 18:37:20 +0000 Subject: [PATCH 022/120] Add Unreleased changelog section covering changes since 4.0rc1 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 765b0e0..2c7e2f4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,18 @@ Changelog =========== +.. _unreleased: + +Unreleased +---------- + +- **Breaking change**: ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` take effect without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. +- Fixed a bug where ``table.delete_where()``, ``table.optimize()`` and ``table.rebuild_fts()`` did not commit their changes, leaving the connection inside an open transaction. Their work - and any subsequent writes - could then be silently rolled back when the connection was closed. All three now use ``db.atomic()``, consistent with the other write methods. +- The ``sqlite-utils drop-table`` command now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. Both now exit with an error suggesting the correct command to use. +- Migrations applied by the new :ref:`migrations system ` now run inside a transaction, together with the record of the migration having been applied. If a migration raises an exception its changes are rolled back and it stays pending, so it can be safely re-applied after the error is fixed. Migrations that cannot run inside a transaction, such as those executing ``VACUUM``, can opt out using ``@migrations(transactional=False)`` - see :ref:`migrations_transactions`. +- ``table.upsert()`` and ``table.upsert_all()`` now detect the primary key or compound primary key of an existing table, so the ``pk=`` argument is no longer required when upserting into a table that already has a primary key. +- ``db.table(table_name).insert({})`` can now be used to insert a row consisting entirely of default values into an existing table, using ``INSERT INTO ... DEFAULT VALUES``. (:issue:`759`) + .. _v4_0rc1: 4.0rc1 (2026-06-21) From d6753a9f57f3d58b89c40bbbebbbbea465a33327 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 18:43:37 +0000 Subject: [PATCH 023/120] upsert() now requires a value for every primary key column A record missing a primary key value - or with None for one - can never match an existing row, because a NULL primary key never satisfies ON CONFLICT. Such records were previously inserted as brand new rows (silently, for multi-record upsert_all calls) or triggered a KeyError after the insert had already happened. Both cases, including upserting an empty record, now raise PrimaryKeyRequired before any SQL is executed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 1 + docs/python-api.rst | 2 ++ sqlite_utils/db.py | 25 +++++++++++++++++++++++++ tests/test_upsert.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2c7e2f4..3a36722 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,6 +14,7 @@ Unreleased - The ``sqlite-utils drop-table`` command now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. Both now exit with an error suggesting the correct command to use. - Migrations applied by the new :ref:`migrations system ` now run inside a transaction, together with the record of the migration having been applied. If a migration raises an exception its changes are rolled back and it stays pending, so it can be safely re-applied after the error is fixed. Migrations that cannot run inside a transaction, such as those executing ``VACUUM``, can opt out using ``@migrations(transactional=False)`` - see :ref:`migrations_transactions`. - ``table.upsert()`` and ``table.upsert_all()`` now detect the primary key or compound primary key of an existing table, so the ``pk=`` argument is no longer required when upserting into a table that already has a primary key. +- ``table.upsert()`` and ``table.upsert_all()`` now raise ``PrimaryKeyRequired`` if a record is missing a value for any primary key column, or has a value of ``None`` for one. Previously such records - which can never match an existing row - were quietly inserted as brand new rows, or triggered a confusing ``KeyError`` after the insert had already taken place. - ``db.table(table_name).insert({})`` can now be used to insert a row consisting entirely of default values into an existing table, using ``INSERT INTO ... DEFAULT VALUES``. (:issue:`759`) .. _v4_0rc1: diff --git a/docs/python-api.rst b/docs/python-api.rst index 39361c9..ae2c0cb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1021,6 +1021,8 @@ Note that the ``pk`` and ``column_order`` parameters here are optional if you ar An ``upsert_all()`` method is also available, which behaves like ``insert_all()`` but performs upserts instead. +Every record passed to ``upsert()`` or ``upsert_all()`` must include a value for each primary key column - a record without one could never match an existing row, so a ``sqlite_utils.db.PrimaryKeyRequired`` exception is raised instead of quietly inserting a new row. + .. note:: ``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 `__ for details of this change. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6f947e1..b4f5dcc 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -3199,6 +3199,11 @@ class Table(Queryable): # Dict-mode insert({}) has no explicit columns; SQLite spells that as # DEFAULT VALUES. List mode with no columns is a different input shape. if not list_mode and not all_columns: + if upsert: + raise PrimaryKeyRequired( + "upsert() requires a value for the primary key - " + "an empty record cannot be upserted" + ) or_clause = "" if replace: or_clause = " OR REPLACE" @@ -3286,6 +3291,26 @@ class Table(Queryable): # Everything from here on is for upsert=True pk_cols = [pk] if isinstance(pk, str) else list(pk) + # Every record must provide a value for every primary key column - a + # NULL primary key never matches ON CONFLICT, so the record would be + # inserted as a brand new row instead of upserted + missing_pk_cols = [c for c in pk_cols if c not in all_columns] + if missing_pk_cols: + raise PrimaryKeyRequired( + "upsert() requires a value for the primary key column{}: {}".format( + "s" if len(missing_pk_cols) > 1 else "", + ", ".join(missing_pk_cols), + ) + ) + pk_indexes = [all_columns.index(c) for c in pk_cols] + for record_values in values: + if any(record_values[i] is None for i in pk_indexes): + raise PrimaryKeyRequired( + "upsert() requires a value for the primary key column{}: {}".format( + "s" if len(pk_cols) > 1 else "", + ", ".join(pk_cols), + ) + ) non_pk_cols = [c for c in all_columns if c not in pk_cols] conflict_sql = ", ".join(quote_identifier(c) for c in pk_cols) diff --git a/tests/test_upsert.py b/tests/test_upsert.py index 09783bb..a782b26 100644 --- a/tests/test_upsert.py +++ b/tests/test_upsert.py @@ -49,6 +49,42 @@ def test_upsert_error_if_no_pk(fresh_db): table.upsert({"id": 1, "name": "Cleo"}) +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_empty_record_errors(use_old_upsert): + db = Database(memory=True, use_old_upsert=use_old_upsert) + table = db["table"] + table.insert({"id": 1, "name": "Cleo"}, pk="id") + with pytest.raises(PrimaryKeyRequired): + table.upsert({}, pk="id") + with pytest.raises(PrimaryKeyRequired): + table.upsert_all([{}, {}], pk="id") + # No rows can have been inserted + assert table.count == 1 + + +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_missing_pk_value_errors(use_old_upsert): + db = Database(memory=True, use_old_upsert=use_old_upsert) + table = db["table"] + table.insert({"id": 1, "name": "Cleo"}, pk="id") + # Records that omit the pk column entirely + with pytest.raises(PrimaryKeyRequired): + table.upsert_all([{"name": "Pancakes"}, {"name": "Marnie"}], pk="id") + # A record with an explicit None pk value can never conflict + with pytest.raises(PrimaryKeyRequired): + table.upsert({"id": None, "name": "Pancakes"}, pk="id") + assert list(table.rows) == [{"id": 1, "name": "Cleo"}] + + +def test_upsert_missing_compound_pk_value_errors(fresh_db): + table = fresh_db["table"] + table.insert({"a": "x", "b": "y", "v": 1}, pk=("a", "b")) + # Missing one component of the detected compound primary key + with pytest.raises(PrimaryKeyRequired): + table.upsert({"a": "x", "v": 2}) + assert list(table.rows) == [{"a": "x", "b": "y", "v": 1}] + + def test_upsert_error_if_existing_table_has_no_pk(fresh_db): table = fresh_db.create_table("table", {"id": int, "name": str}) with pytest.raises(PrimaryKeyRequired): From ffec11cfb7875120e34d52d89dc5a06e79de0f9d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 18:46:09 +0000 Subject: [PATCH 024/120] enable_wal() and disable_wal() refuse to run inside a transaction Changing the journal mode assigns conn.isolation_level, which commits any open transaction as a side effect - silently breaking the rollback guarantee of atomic() blocks and of user-managed transactions. Both methods now raise RuntimeError if a transaction is open. Calling them when the database is already in the requested mode remains a no-op. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 1 + docs/python-api.rst | 2 ++ sqlite_utils/db.py | 21 ++++++++++++++++++++- tests/test_wal.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3a36722..0c409a1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,7 @@ Unreleased - ``table.upsert()`` and ``table.upsert_all()`` now detect the primary key or compound primary key of an existing table, so the ``pk=`` argument is no longer required when upserting into a table that already has a primary key. - ``table.upsert()`` and ``table.upsert_all()`` now raise ``PrimaryKeyRequired`` if a record is missing a value for any primary key column, or has a value of ``None`` for one. Previously such records - which can never match an existing row - were quietly inserted as brand new rows, or triggered a confusing ``KeyError`` after the insert had already taken place. - ``db.table(table_name).insert({})`` can now be used to insert a row consisting entirely of default values into an existing table, using ``INSERT INTO ... DEFAULT VALUES``. (:issue:`759`) +- ``db.enable_wal()`` and ``db.disable_wal()`` now raise a ``RuntimeError`` if called while a transaction is open. Previously they would silently commit the open transaction as a side effect of changing the journal mode, breaking the rollback guarantee of ``db.atomic()`` and of user-managed transactions. .. _v4_0rc1: diff --git a/docs/python-api.rst b/docs/python-api.rst index ae2c0cb..f03f963 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2702,6 +2702,8 @@ You can disable WAL mode using ``.disable_wal()``: Database("my_database.db").disable_wal() +The journal mode can only be changed outside of a transaction. Calling either method while a transaction is open - inside a ``db.atomic()`` block, for example - raises a ``RuntimeError``, unless the database is already in the requested mode in which case the call is a no-op. + You can check the current journal mode for a database using the ``journal_mode`` property: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b4f5dcc..97e8920 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -854,17 +854,36 @@ class Database: def enable_wal(self) -> None: """ Sets ``journal_mode`` to ``'wal'`` to enable Write-Ahead Log mode. + + :raises RuntimeError: if called while a transaction is open - the + journal mode can only be changed outside of a transaction """ if self.journal_mode != "wal": + self._ensure_no_open_transaction("enable_wal()") with self.ensure_autocommit_off(): self.execute("PRAGMA journal_mode=wal;") def disable_wal(self) -> None: - "Sets ``journal_mode`` back to ``'delete'`` to disable Write-Ahead Log mode." + """ + Sets ``journal_mode`` back to ``'delete'`` to disable Write-Ahead Log mode. + + :raises RuntimeError: if called while a transaction is open - the + journal mode can only be changed outside of a transaction + """ if self.journal_mode != "delete": + self._ensure_no_open_transaction("disable_wal()") with self.ensure_autocommit_off(): self.execute("PRAGMA journal_mode=delete;") + def _ensure_no_open_transaction(self, operation: str) -> None: + # Changing journal mode assigns conn.isolation_level, which commits + # any open transaction as a side effect - breaking the rollback + # guarantee of atomic() and of user-managed transactions + if self.conn.in_transaction: + raise RuntimeError( + "{} cannot be used while a transaction is open".format(operation) + ) + def _ensure_counts_table(self) -> None: with self.atomic(): self.execute(_COUNTS_TABLE_CREATE_SQL.format(self._counts_table_name)) diff --git a/tests/test_wal.py b/tests/test_wal.py index 23ca144..fa3c855 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -21,3 +21,39 @@ def test_enable_disable_wal(db_path_tmpdir): db.disable_wal() assert "delete" == db.journal_mode assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()] + + +def test_enable_wal_inside_transaction_raises(db_path_tmpdir): + db, path, tmpdir = db_path_tmpdir + db["test"].insert({"id": 1}, pk="id") + with pytest.raises(RuntimeError): + with db.atomic(): + db["test"].insert({"id": 2}, pk="id") + db.enable_wal() + # The atomic() block must have rolled back cleanly and the + # journal mode must be unchanged + assert db.journal_mode == "delete" + assert [r["id"] for r in db["test"].rows] == [1] + + +def test_disable_wal_inside_transaction_raises(db_path_tmpdir): + db, path, tmpdir = db_path_tmpdir + db.enable_wal() + db["test"].insert({"id": 1}, pk="id") + with pytest.raises(RuntimeError): + with db.atomic(): + db["test"].insert({"id": 2}, pk="id") + db.disable_wal() + assert db.journal_mode == "wal" + assert [r["id"] for r in db["test"].rows] == [1] + + +def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): + # Calling enable_wal() when WAL is already enabled is a no-op, + # so it is fine inside a transaction + db, path, tmpdir = db_path_tmpdir + db.enable_wal() + with db.atomic(): + db["test"].insert({"id": 1}, pk="id") + db.enable_wal() + assert [r["id"] for r in db["test"].rows] == [1] From c76aad50aeec0bc78a7847bdcdb87f3afa616c5b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 18:47:08 +0000 Subject: [PATCH 025/120] Correct applied_at type annotation to str _AppliedMigration.applied_at was annotated datetime.datetime but the value is the TEXT timestamp read straight from the _sqlite_migrations table - always a string, matching the format written by both this module and the older sqlite-migrate package. Added a test pinning the runtime type. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- sqlite_utils/migrations.py | 4 +++- tests/test_migrations.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py index 8af0784..20572af 100644 --- a/sqlite_utils/migrations.py +++ b/sqlite_utils/migrations.py @@ -19,7 +19,9 @@ class Migrations: @dataclass class _AppliedMigration: name: str - applied_at: datetime.datetime + # A string timestamp such as "2026-07-04 12:00:00.000000+00:00" - + # stored as TEXT in the _sqlite_migrations table + applied_at: str def __init__(self, name: str): """ diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 34741a5..c190fd5 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -80,6 +80,18 @@ def test_order_does_not_matter(migrations, migrations_not_ordered_alphabetically assert db1.schema == db2.schema +def test_applied_at_is_a_string(migrations): + db = sqlite_utils.Database(memory=True) + migrations.apply(db) + applied = migrations.applied(db) + assert len(applied) == 2 + for migration in applied: + # applied_at is the TEXT timestamp straight from the + # _sqlite_migrations table, e.g. "2026-07-04 12:00:00.000000+00:00" + assert isinstance(migration.applied_at, str) + assert migration.applied_at.endswith("+00:00") + + def test_failing_migration_rolls_back(migrations): @migrations() def m003(db): From 70b48a3b16669d2b02711f2c95714665902c3450 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 18:51:55 +0000 Subject: [PATCH 026/120] Use the sqlite3/sqlean shim in test_query.py test_query_executes_eagerly failed on the sqlean CI matrix jobs because sqlean.dbapi2.OperationalError is not a subclass of the stdlib sqlite3.OperationalError. Import sqlite3 via sqlite_utils.utils like the other test modules do. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- tests/test_query.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_query.py b/tests/test_query.py index 0b0b9c5..91bf1ac 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1,7 +1,8 @@ import pytest -import sqlite3 import types +from sqlite_utils.utils import sqlite3 + def test_query(fresh_db): fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) From a6fcb5af624ed667d7c2b3a6ede3ad41ae0f067b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 19:05:59 +0000 Subject: [PATCH 027/120] Pin the Database context manager contract with Database(...) closes the connection on exit without committing, so uncommitted changes are rolled back. This was the existing behavior but was undocumented and untested - it is now stated in the docs and pinned by a test. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/python-api.rst | 2 ++ tests/test_constructor.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index f03f963..d575ba6 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -144,6 +144,8 @@ The ``Database`` object also works as a context manager, which will automaticall db["my_table"].insert({"name": "Example"}) # Connection is automatically closed here +Exiting the block closes the connection without committing, so any changes that are still inside an open transaction at that point - for example writes made with raw ``db.execute()`` calls - will be rolled back. Commit explicitly, or use methods such as ``.insert()`` and the ``db.atomic()`` context manager which commit their own transactions. + .. _python_api_attach: Attaching additional databases diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 8743911..0798996 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -29,6 +29,21 @@ def test_sqlite_version(): assert actual == as_string +def test_database_context_manager(tmpdir): + path = str(tmpdir / "test.db") + with Database(path) as db: + db["t"].insert({"id": 1}) + # A raw write left uncommitted on purpose: + db.execute("insert into t (id) values (2)") + # The connection is closed... + with pytest.raises(sqlite3.ProgrammingError): + db.execute("select 1") + # ... and the uncommitted change was rolled back, not committed + db2 = Database(path) + assert [r["id"] for r in db2["t"].rows] == [1] + db2.close() + + @pytest.mark.parametrize("memory", [True, False]) def test_database_close(tmpdir, memory): if memory: From fd867282b3ae8ba05e60eee39471181008102c47 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 19:06:44 +0000 Subject: [PATCH 028/120] Document atomic() deferred BEGIN, savepoint behavior and connection modes Documents that atomic() opens a deferred BEGIN, that calling it while the connection is already inside a transaction produces a savepoint whose changes only persist when the outer transaction commits, and that Python 3.12+ autocommit=True/False connections are not supported. Adds a test pinning the savepoint-inside-manual- BEGIN behavior. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/python-api.rst | 6 ++++++ tests/test_atomic.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index d575ba6..e888b7d 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -274,6 +274,12 @@ If an exception is raised, changes made inside the block will be rolled back. pass db.table("dogs").insert({"id": 3, "name": "Marnie"}) +The transaction is opened with a deferred ``BEGIN`` - SQLite takes the necessary locks when the first statement inside the block runs. + +The same savepoint mechanism used for nesting applies if the connection is already inside a transaction when ``db.atomic()`` is called - for example if you executed ``BEGIN`` yourself, or made writes using raw ``db.execute()`` calls. In that case the block becomes a savepoint within your transaction: exiting the block releases the savepoint, but nothing is committed to disk until your outer transaction commits. + +``db.atomic()`` is designed for connections in Python's default transaction handling mode. Connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options are not supported, because ``commit()`` and ``rollback()`` behave differently on those connections. + .. _python_api_parameters: Passing parameters diff --git a/tests/test_atomic.py b/tests/test_atomic.py index acd5474..8c4d18d 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -172,3 +172,20 @@ def test_transform_detects_foreign_key_check_violations(fresh_db): assert fresh_db["books"].foreign_keys == [] assert fresh_db.conn.execute("PRAGMA foreign_keys").fetchone()[0] + + +def test_atomic_inside_manual_transaction_uses_savepoint(fresh_db): + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.execute("begin") + with fresh_db.atomic(): + fresh_db["t"].insert({"id": 2}, pk="id") + # Nothing is committed until the user's own transaction commits + assert fresh_db.conn.in_transaction + fresh_db.conn.rollback() + assert [r["id"] for r in fresh_db["t"].rows] == [1] + # And with a commit instead, the atomic block's writes persist + fresh_db.execute("begin") + with fresh_db.atomic(): + fresh_db["t"].insert({"id": 3}, pk="id") + fresh_db.conn.commit() + assert [r["id"] for r in fresh_db["t"].rows] == [1, 3] From 0bad21280fc29e42a58358a68f702cd270bb9094 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 19:08:53 +0000 Subject: [PATCH 029/120] migrate --stop-before: validate names and fix legacy compatibility An unknown --stop-before value previously matched nothing and every migration was silently applied, including the one the user meant to stop before. The CLI now errors unless each value matches a known migration. The CLI also always passed stop_before as a list, but older duck-typed sqlite-migrate Migrations objects compare it against a single string name - so the flag was silently ignored for exactly the migration files the compatibility shim exists to support. Legacy sets now receive a single string, with an error if multiple values target one legacy set. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/migrations.rst | 2 + sqlite_utils/cli.py | 35 +++++++++-- tests/test_cli_migrate.py | 125 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 4 deletions(-) diff --git a/docs/migrations.rst b/docs/migrations.rst index 5628ded..eddac5f 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -157,6 +157,8 @@ You can also target a specific migration set using ``migration_set:migration_nam The ``--stop-before`` option can be passed more than once. +If a ``--stop-before`` value does not match any known migration the command exits with an error, rather than silently applying everything. + Verbose output ============== diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index d51ba7b..23f04a7 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3400,11 +3400,38 @@ def migrate(db_path, migrations, stop_before, list_, verbose): click.echo("\nSchema before:\n") click.echo(textwrap.indent(prev_schema, " ") or " (empty)") click.echo() + if stop_before: + # Every --stop-before value must match at least one known migration + known_names = set() + for migration_set in migration_sets: + names = {m.name for m in migration_set.pending(db)} + names.update(m.name for m in migration_set.applied(db)) + known_names.update(names) + known_names.update( + "{}:{}".format(migration_set.name, name) for name in names + ) + unknown = [value for value in stop_before if value not in known_names] + if unknown: + raise click.ClickException( + "--stop-before did not match any migrations: {}".format( + ", ".join(unknown) + ) + ) for migration_set in migration_sets: - migration_set.apply( - db, - stop_before=_stop_before_for_migration_set(stop_before, migration_set.name), - ) + matches = _stop_before_for_migration_set(stop_before, migration_set.name) + if isinstance(migration_set, sqlite_utils.Migrations): + migration_set.apply(db, stop_before=matches) + else: + # Legacy sqlite-migrate Migrations objects take a single string + # for stop_before, not a list + distinct = list(dict.fromkeys(matches)) + if len(distinct) > 1: + raise click.ClickException( + "Migration set '{}' uses the older sqlite-migrate class, " + "which only supports a single --stop-before value - " + "got: {}".format(migration_set.name, ", ".join(distinct)) + ) + migration_set.apply(db, stop_before=distinct[0] if distinct else None) if verbose: click.echo("Schema after:\n") post_schema = db.schema diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index 229bc9c..4a2ca9c 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -301,3 +301,128 @@ def test_stop_before_multiple_qualified(two_sets_same_migration_name): assert not db["creature_weights"].exists() assert db["sales"].exists() assert not db["sales_weights"].exists() + + +LEGACY_MIGRATIONS = """ +import datetime + +class _Migration: + def __init__(self, name, fn): + self.name = name + self.fn = fn + +class _Applied: + def __init__(self, name, applied_at): + self.name = name + self.applied_at = applied_at + +class LegacyMigrations: + # Mimics the sqlite-migrate 0.x Migrations class, in particular + # apply(db, stop_before=None) taking a single string + migrations_table = "_sqlite_migrations" + + def __init__(self, name): + self.name = name + self._migrations = [] + + def __call__(self, fn): + self._migrations.append(_Migration(fn.__name__, fn)) + return fn + + def ensure_migrations_table(self, db): + db[self.migrations_table].create( + {"migration_set": str, "name": str, "applied_at": str}, + pk=("migration_set", "name"), + if_not_exists=True, + ) + + def applied(self, db): + self.ensure_migrations_table(db) + return [ + _Applied(row["name"], row["applied_at"]) + for row in db[self.migrations_table].rows_where( + "migration_set = ?", [self.name] + ) + ] + + def pending(self, db): + applied = {m.name for m in self.applied(db)} + return [m for m in self._migrations if m.name not in applied] + + def apply(self, db, stop_before=None): + for migration in self.pending(db): + if migration.name == stop_before: + return + migration.fn(db) + db[self.migrations_table].insert( + { + "migration_set": self.name, + "name": migration.name, + "applied_at": str( + datetime.datetime.now(datetime.timezone.utc) + ), + } + ) + +legacy = LegacyMigrations("legacy_set") + +@legacy +def first(db): + db["first"].insert({"hello": "world"}) + +@legacy +def second(db): + db["second"].insert({"hello": "world"}) +""" + + +def test_stop_before_unknown_name_errors(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, str(path), "--stop-before", "fooo"], + ) + assert result.exit_code == 1 + assert "--stop-before did not match any migrations: fooo" in result.output + # Nothing should have been applied + db = sqlite_utils.Database(db_path) + assert "foo" not in db.table_names() + assert "bar" not in db.table_names() + + +def test_stop_before_with_legacy_migrations_class(tmpdir): + path = pathlib.Path(tmpdir) + (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8") + db_path = str(path / "test.db") + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, str(path), "--stop-before", "second"], + ) + assert result.exit_code == 0, result.output + db = sqlite_utils.Database(db_path) + assert "first" in db.table_names() + assert "second" not in db.table_names() + + +def test_stop_before_multiple_values_for_legacy_set_errors(tmpdir): + path = pathlib.Path(tmpdir) + (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8") + db_path = str(path / "test.db") + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, + [ + "migrate", + db_path, + str(path), + "--stop-before", + "legacy_set:first", + "--stop-before", + "legacy_set:second", + ], + ) + assert result.exit_code == 1 + assert "single --stop-before" in result.output From 30cc95c0a6c4fc1947f43feb1dc9798305a1b9e8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 19:10:56 +0000 Subject: [PATCH 030/120] migrate --list is now read-only pending() and applied() no longer create the _sqlite_migrations table or perform the one-way legacy schema upgrade - that now only happens in apply(). The CLI --list path no longer creates the database file when it does not exist. applied() results are now explicitly ordered by insertion. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 1 + docs/migrations.rst | 2 +- sqlite_utils/cli.py | 12 +++++++++--- sqlite_utils/migrations.py | 22 +++++++++++----------- tests/test_cli_migrate.py | 37 +++++++++++++++++++++++++++++++++++++ tests/test_migrations.py | 8 ++++++++ 6 files changed, 67 insertions(+), 15 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0c409a1..1e268ea 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,7 @@ Unreleased - ``table.upsert()`` and ``table.upsert_all()`` now raise ``PrimaryKeyRequired`` if a record is missing a value for any primary key column, or has a value of ``None`` for one. Previously such records - which can never match an existing row - were quietly inserted as brand new rows, or triggered a confusing ``KeyError`` after the insert had already taken place. - ``db.table(table_name).insert({})`` can now be used to insert a row consisting entirely of default values into an existing table, using ``INSERT INTO ... DEFAULT VALUES``. (:issue:`759`) - ``db.enable_wal()`` and ``db.disable_wal()`` now raise a ``RuntimeError`` if called while a transaction is open. Previously they would silently commit the open transaction as a side effect of changing the journal mode, breaking the rollback guarantee of ``db.atomic()`` and of user-managed transactions. +- Improvements to the ``sqlite-utils migrate`` command: ``--stop-before`` values that do not match any known migration are now an error instead of being silently ignored, ``--stop-before`` now works correctly with migration files that still use the older ``sqlite_migrate.Migrations`` class, and ``--list`` is now a read-only operation that no longer creates the database file or the migrations tracking table. ``migrations.applied()`` now returns migrations in the order they were applied. .. _v4_0rc1: diff --git a/docs/migrations.rst b/docs/migrations.rst index eddac5f..61709c4 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -117,7 +117,7 @@ Running the command repeatedly is safe. Migrations that already have a matching Listing migrations ================== -Use ``--list`` to show applied and pending migrations without running them: +Use ``--list`` to show applied and pending migrations without running them. This is a read-only operation - it will not create the database file or the ``_sqlite_migrations`` table: .. code-block:: bash diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 23f04a7..1c94806 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3387,13 +3387,19 @@ def migrate(db_path, migrations, stop_before, list_, verbose): if not migration_sets: raise click.ClickException("No migrations.py files found") - db = sqlite_utils.Database(db_path) - _register_db_for_cleanup(db) - if list_: + if pathlib.Path(db_path).exists(): + db = sqlite_utils.Database(db_path) + else: + # Listing is read-only - don't create the database file + db = sqlite_utils.Database(memory=True) + _register_db_for_cleanup(db) _display_migration_list(db, migration_sets) return + db = sqlite_utils.Database(db_path) + _register_db_for_cleanup(db) + prev_schema = db.schema if verbose: click.echo("Migrating {}".format(db_path)) diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py index 20572af..7ae72e6 100644 --- a/sqlite_utils/migrations.py +++ b/sqlite_utils/migrations.py @@ -54,14 +54,10 @@ class Migrations: def pending(self, db: "Database") -> list["Migrations._Migration"]: """ Return a list of pending migrations. + + This is a read-only operation - it does not write to the database. """ - self.ensure_migrations_table(db) - already_applied = { - r["name"] - for r in db[self.migrations_table].rows_where( - "migration_set = ?", [self.name] - ) - } + already_applied = {migration.name for migration in self.applied(db)} return [ migration for migration in self._migrations @@ -70,13 +66,17 @@ class Migrations: def applied(self, db: "Database") -> list["Migrations._AppliedMigration"]: """ - Return a list of applied migrations. + Return a list of applied migrations, in the order they were applied. + + This is a read-only operation - it does not write to the database. """ - self.ensure_migrations_table(db) + table = _table(db, self.migrations_table) + if not table.exists(): + return [] return [ self._AppliedMigration(name=row["name"], applied_at=row["applied_at"]) - for row in db[self.migrations_table].rows_where( - "migration_set = ?", [self.name] + for row in table.rows_where( + "migration_set = ?", [self.name], order_by="rowid" ) ] diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index 4a2ca9c..1cdf8e7 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -426,3 +426,40 @@ def test_stop_before_multiple_values_for_legacy_set_errors(tmpdir): ) assert result.exit_code == 1 assert "single --stop-before" in result.output + + +def test_list_does_not_create_database_file(two_migrations): + path, _ = two_migrations + db_path = path / "test.db" + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", str(db_path), str(path), "--list"] + ) + assert result.exit_code == 0, result.output + assert "Pending:\n foo\n bar" in result.output + # Listing migrations must not create the database file + assert not db_path.exists() + + +def test_list_does_not_upgrade_legacy_migrations_table(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + db = sqlite_utils.Database(db_path) + db["_sqlite_migrations"].create( + {"migration_set": str, "name": str, "applied_at": str}, + pk=("migration_set", "name"), + ) + db["_sqlite_migrations"].insert( + {"migration_set": "hello", "name": "foo", "applied_at": "x"} + ) + db.close() + runner = CliRunner() + result = runner.invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"] + ) + assert result.exit_code == 0, result.output + assert "foo - x" in result.output + # --list must not perform the one-way legacy schema upgrade + db2 = sqlite_utils.Database(db_path) + assert db2["_sqlite_migrations"].pks == ["migration_set", "name"] + db2.close() diff --git a/tests/test_migrations.py b/tests/test_migrations.py index c190fd5..1733aa4 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -190,3 +190,11 @@ def test_upgrades_sqlite_migrations(migrations, create_table, pk): assert db["_sqlite_migrations"].pks == ([pk] if isinstance(pk, str) else list(pk)) migrations.apply(db) assert db["_sqlite_migrations"].pks == ["id"] + + +def test_pending_and_applied_are_read_only(migrations): + db = sqlite_utils.Database(memory=True) + assert [m.name for m in migrations.pending(db)] == ["m001", "m002"] + assert migrations.applied(db) == [] + # Neither call should have created the tracking table + assert db.table_names() == [] From 397cdcc491342825e460cb9a08ef06da7552a409 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 19:13:15 +0000 Subject: [PATCH 031/120] Polish: clearer errors, corrected docstrings, setuptools pin - insert/upsert into a name that is actually a view now shows a clean error instead of a traceback - db.view() on a name that is a table now says so in its error - Database.__getitem__ docstring documents that views are returned for view names; hash_id docstring corrected (it is a column name, not a boolean) - Registering two migrations with the same name in one set now raises ValueError at registration time instead of executing both and failing with IntegrityError at apply time - build-system requires setuptools>=77, needed for the PEP 639 license expression Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- pyproject.toml | 3 ++- sqlite_utils/cli.py | 2 ++ sqlite_utils/db.py | 17 +++++++++++++---- sqlite_utils/migrations.py | 9 ++++++++- tests/test_cli_insert.py | 13 +++++++++++++ tests/test_create.py | 5 ++++- tests/test_migrations.py | 16 ++++++++++++++++ 7 files changed, 58 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2c86f85..4ad1bd9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,8 @@ CI = "https://github.com/simonw/sqlite-utils/actions" sqlite-utils = "sqlite_utils.cli:cli" [build-system] -requires = ["setuptools"] +# setuptools 77+ is needed for the PEP 639 license = "Apache-2.0" expression +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [tool.flake8] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 1c94806..f2e1ba6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1155,6 +1155,8 @@ def insert_upsert_implementation( db.table(table).insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs ) + except NoTable as e: + raise click.ClickException(str(e)) except Exception as e: if ( isinstance(e, OperationalError) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 97e8920..b53df51 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -499,10 +499,12 @@ class Database: def __getitem__(self, table_name: str) -> Union["Table", "View"]: """ - ``db[table_name]`` returns a :class:`.Table` object for the table with the specified name. - If the table does not exist yet it will be created the first time data is inserted into it. + ``db[name]`` returns a :class:`.Table` object for the table with the specified name, + or a :class:`.View` object if the name matches an existing SQL view. + If neither exists yet, a table is assumed - it will be created the first + time data is inserted into it. - :param table_name: The name of the table + :param table_name: The name of the table or view """ if table_name in self.view_names(): return self.view(table_name) @@ -672,6 +674,12 @@ class Database: :param view_name: Name of the view """ if view_name not in self.view_names(): + if view_name in self.table_names(): + raise NoView( + "View {name} does not exist - {name} is a table".format( + name=view_name + ) + ) raise NoView("View {} does not exist".format(view_name)) return View(self, view_name) @@ -1618,7 +1626,8 @@ class Table(Queryable): :param not_null: List of columns that cannot be null :param defaults: Dictionary of column names and default values :param batch_size: Integer number of rows to insert at a time - :param hash_id: If True, use a hash of the row values as the primary key + :param hash_id: Name of a column to create and use as a primary key, where the + value of that primary key is derived from a hash of the row values :param hash_id_columns: List of columns to use for the hash_id :param alter: If True, automatically alter the table if it doesn't match the schema :param ignore: If True, ignore rows that already exist when inserting diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py index 7ae72e6..36e053d 100644 --- a/sqlite_utils/migrations.py +++ b/sqlite_utils/migrations.py @@ -44,8 +44,15 @@ class Migrations: """ def inner(func: Callable) -> Callable: + migration_name = name or getattr(func, "__name__") + if any(m.name == migration_name for m in self._migrations): + raise ValueError( + "Migration '{}' is already registered in set '{}'".format( + migration_name, self.name + ) + ) self._migrations.append( - self._Migration(name or getattr(func, "__name__"), func, transactional) + self._Migration(migration_name, func, transactional) ) return func diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 5812851..2df1e0c 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -615,3 +615,16 @@ def test_insert_csv_headers_only(tmpdir): # Table should not exist since there were no data rows db = Database(db_path) assert not db["data"].exists() + + +def test_insert_into_view_errors(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["t"].insert({"id": 1}) + db.create_view("v", "select * from t") + db.close() + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "v", "-"], input='{"id": 2}' + ) + assert result.exit_code == 1 + assert result.output.strip() == "Error: Table v is actually a view" diff --git a/tests/test_create.py b/tests/test_create.py index b1a6ad1..afa7a65 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1382,7 +1382,10 @@ def test_bad_table_and_view_exceptions(fresh_db): assert ex.value.args[0] == "Table v is actually a view" with pytest.raises(NoView) as ex2: fresh_db.view("t") - assert ex2.value.args[0] == "View t does not exist" + assert ex2.value.args[0] == "View t does not exist - t is a table" + with pytest.raises(NoView) as ex3: + fresh_db.view("missing") + assert ex3.value.args[0] == "View missing does not exist" # Tests for issue #655: Table configuration should be stored in _defaults diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 1733aa4..5634d79 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -198,3 +198,19 @@ def test_pending_and_applied_are_read_only(migrations): assert migrations.applied(db) == [] # Neither call should have created the tracking table assert db.table_names() == [] + + +def test_duplicate_migration_name_errors(): + migrations = Migrations("test") + + @migrations() + def m001(db): + pass + + with pytest.raises(ValueError) as ex: + + @migrations(name="m001") + def m001_again(db): + pass + + assert "m001" in str(ex.value) From f1cdceaca9fb6e46eef12e16e06a6aad959160dc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 19:15:31 +0000 Subject: [PATCH 032/120] Remove the no-op -d/--detect-types flag from insert and upsert Type detection has been the default for CSV/TSV data since 4.0a1, making this flag a no-op kept only for backwards compatibility. 4.0 is the release where it can be removed - the flag now errors, prompting scripts to drop it. --no-detect-types is unchanged. The dead detect_types parameter has been removed from insert_upsert_implementation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 1 + docs/cli-reference.rst | 2 -- sqlite_utils/cli.py | 14 +------------- tests/test_cli.py | 36 +++++++++++++++++++++--------------- 4 files changed, 23 insertions(+), 30 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1e268ea..ce1397f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Unreleased ---------- +- **Breaking change**: The no-op ``-d/--detect-types`` flag has been removed from the ``insert`` and ``upsert`` commands. Type detection has been the default for CSV/TSV data since 4.0a1, so the flag did nothing - invocations using it should simply drop it. ``--no-detect-types`` remains available to disable detection. - **Breaking change**: ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` take effect without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. - Fixed a bug where ``table.delete_where()``, ``table.optimize()`` and ``table.rebuild_fts()`` did not commit their changes, leaving the connection inside an open transaction. Their work - and any subsequent writes - could then be silently rolled back when the connection was closed. All three now use ``db.atomic()``, consistent with the other write methods. - The ``sqlite-utils drop-table`` command now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. Both now exit with an error suggesting the correct command to use. diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 48d0145..cbac48b 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -286,7 +286,6 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr --alter Alter existing table to add any missing columns --not-null TEXT Columns that should be created as NOT NULL --default ... Default value that should be set for a column - -d, --detect-types Detect types for columns in CSV/TSV data (default) --no-detect-types Treat all CSV/TSV columns as TEXT --analyze Run ANALYZE at the end of this operation --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -344,7 +343,6 @@ See :ref:`cli_upsert`. --alter Alter existing table to add any missing columns --not-null TEXT Columns that should be created as NOT NULL --default ... Default value that should be set for a column - -d, --detect-types Detect types for columns in CSV/TSV data (default) --no-detect-types Treat all CSV/TSV columns as TEXT --analyze Run ANALYZE at the end of this operation --load-extension TEXT Path to SQLite extension, with optional :entrypoint diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index f2e1ba6..a2f881a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -940,12 +940,6 @@ def insert_upsert_options(*, require_pk=False): type=(str, str), help="Default value that should be set for a column", ), - click.option( - "-d", - "--detect-types", - is_flag=True, - help="Detect types for columns in CSV/TSV data (default)", - ), click.option( "--no-detect-types", is_flag=True, @@ -1000,7 +994,6 @@ def insert_upsert_implementation( truncate=False, not_null=None, default=None, - detect_types=None, no_detect_types=False, analyze=False, load_extension=None, @@ -1067,7 +1060,7 @@ def insert_upsert_implementation( ) else: docs = (dict(zip(headers, row)) for row in reader) - # detect_types is now the default, unless --no-detect-types is passed + # Type detection is the default, unless --no-detect-types is passed if not no_detect_types: tracker = TypeTracker() docs = tracker.wrap(docs) @@ -1241,7 +1234,6 @@ def insert( batch_size, stop_after, alter, - detect_types, no_detect_types, analyze, load_extension, @@ -1324,7 +1316,6 @@ def insert( ignore=ignore, replace=replace, truncate=truncate, - detect_types=detect_types, no_detect_types=no_detect_types, analyze=analyze, load_extension=load_extension, @@ -1363,7 +1354,6 @@ def upsert( alter, not_null, default, - detect_types, no_detect_types, analyze, load_extension, @@ -1409,7 +1399,6 @@ def upsert( upsert=True, not_null=not_null, default=default, - detect_types=detect_types, no_detect_types=no_detect_types, analyze=analyze, load_extension=load_extension, @@ -1497,7 +1486,6 @@ def bulk( upsert=False, not_null=set(), default={}, - detect_types=False, no_detect_types=True, load_extension=load_extension, silent=False, diff --git a/tests/test_cli.py b/tests/test_cli.py index 02c00ab..d26e4dd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2331,18 +2331,14 @@ def test_csv_insert_bom(tmpdir): ] -@pytest.mark.parametrize("option", (None, "-d", "--detect-types")) -def test_insert_detect_types(tmpdir, option): - """Test that type detection is now the default behavior""" +def test_insert_detect_types(tmpdir): + """Test that type detection is the default behavior""" db_path = str(tmpdir / "test.db") data = "name,age,weight\nCleo,6,45.5\nDori,1,3.5" - extra = [] - if option: - extra = [option] result = CliRunner().invoke( cli.cli, - ["insert", db_path, "creatures", "-", "--csv"] + extra, + ["insert", db_path, "creatures", "-", "--csv"], catch_exceptions=False, input=data, ) @@ -2354,17 +2350,27 @@ def test_insert_detect_types(tmpdir, option): ] -@pytest.mark.parametrize("option", (None, "-d", "--detect-types")) -def test_upsert_detect_types(tmpdir, option): - """Test that type detection is now the default behavior for upsert""" +@pytest.mark.parametrize("command", ("insert", "upsert")) +@pytest.mark.parametrize("option", ("-d", "--detect-types")) +def test_detect_types_flag_removed(tmpdir, command, option): + # The old no-op flag was removed in 4.0 - it should now error db_path = str(tmpdir / "test.db") - data = "id,name,age,weight\n1,Cleo,6,45.5\n2,Dori,1,3.5" - extra = [] - if option: - extra = [option] result = CliRunner().invoke( cli.cli, - ["upsert", db_path, "creatures", "-", "--csv", "--pk", "id"] + extra, + [command, db_path, "creatures", "-", "--csv", "--pk", "id", option], + input="id,name\n1,Cleo", + ) + assert result.exit_code == 2 + assert "No such option" in result.output + + +def test_upsert_detect_types(tmpdir): + """Test that type detection is the default behavior for upsert""" + db_path = str(tmpdir / "test.db") + data = "id,name,age,weight\n1,Cleo,6,45.5\n2,Dori,1,3.5" + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "creatures", "-", "--csv", "--pk", "id"], catch_exceptions=False, input=data, ) From a2e6ea2eca5f2a49c116c9af1a37aba4b1a7eaac Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 19:19:24 +0000 Subject: [PATCH 033/120] Raise ValueError instead of AssertionError for invalid arguments User-facing argument validation in db.py previously used bare assert statements, which vanish entirely under python -O and raise AssertionError - an exception type callers should not have to catch for input validation. Fifteen validation sites now raise ValueError with the same messages. Internal invariants (an unreachable branch and a post-update rowcount check) remain asserts. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 1 + sqlite_utils/db.py | 123 ++++++++++++++++++++++---------------- tests/test_convert.py | 2 +- tests/test_create.py | 12 ++-- tests/test_create_view.py | 2 +- tests/test_m2m.py | 4 +- tests/test_recreate.py | 2 +- 7 files changed, 83 insertions(+), 63 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ce1397f..0b41875 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Unreleased ---------- +- **Breaking change**: Python API validation errors now raise ``ValueError`` instead of ``AssertionError``. Previously invalid arguments - such as ``create_table()`` with no columns, ``transform()`` on a table that does not exist, or passing both ``ignore=True`` and ``replace=True`` - were rejected using bare ``assert`` statements, which are silently skipped when Python runs with the ``-O`` flag. Code that caught ``AssertionError`` for these cases should catch ``ValueError`` instead. - **Breaking change**: The no-op ``-d/--detect-types`` flag has been removed from the ``insert`` and ``upsert`` commands. Type detection has been the default for CSV/TSV data since 4.0a1, so the flag did nothing - invocations using it should simply drop it. ``--no-detect-types`` remains available to disable detection. - **Breaking change**: ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` take effect without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. - Fixed a bug where ``table.delete_where()``, ``table.optimize()`` and ``table.rebuild_fts()`` did not commit their changes, leaving the connection inside an open transaction. Their work - and any subsequent writes - could then be silently rolled back when the connection was closed. All three now use ``db.atomic()``, consistent with the other write methods. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b53df51..a0ca6ed 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -367,9 +367,11 @@ class Database: self.memory_name = None self.memory = False self.use_old_upsert = use_old_upsert - assert (filename_or_conn is not None and (not memory and not memory_name)) or ( - filename_or_conn is None and (memory or memory_name) - ), "Either specify a filename_or_conn or pass memory=True" + if not ( + (filename_or_conn is not None and (not memory and not memory_name)) + or (filename_or_conn is None and (memory or memory_name)) + ): + raise ValueError("Either specify a filename_or_conn or pass memory=True") if memory_name: uri = "file:{}?mode=memory&cache=shared".format(memory_name) self.conn = sqlite3.connect( @@ -393,7 +395,8 @@ class Database: raise self.conn = sqlite3.connect(str(filename_or_conn)) else: - assert not recreate, "recreate cannot be used with connections, only paths" + if recreate: + raise ValueError("recreate cannot be used with connections, only paths") self.conn = cast(sqlite3.Connection, filename_or_conn) self._tracer: Optional[Tracer] = tracer if recursive_triggers: @@ -968,20 +971,21 @@ class Database: other_column = table.guess_foreign_column(other_table) fks.append(ForeignKey(name, column, other_table, other_column)) return fks - assert all( - isinstance(fk, (tuple, list)) for fk in foreign_keys - ), "foreign_keys= should be a list of tuples" + if not all(isinstance(fk, (tuple, list)) for fk in foreign_keys): + raise ValueError("foreign_keys= should be a list of tuples") fks = [] for tuple_or_list in foreign_keys: if len(tuple_or_list) == 4: - assert ( - tuple_or_list[0] == name - ), "First item in {} should have been {}".format(tuple_or_list, name) - assert len(tuple_or_list) in ( - 2, - 3, - 4, - ), "foreign_keys= should be a list of tuple pairs or triples" + if tuple_or_list[0] != name: + raise ValueError( + "First item in {} should have been {}".format( + tuple_or_list, name + ) + ) + if len(tuple_or_list) not in (2, 3, 4): + raise ValueError( + "foreign_keys= should be a list of tuple pairs or triples" + ) if len(tuple_or_list) in (3, 4): if len(tuple_or_list) == 4: tuple_or_list = cast(Tuple[str, str, str], tuple_or_list[1:]) @@ -1054,17 +1058,20 @@ class Database: # Soundness check not_null, and defaults if provided not_null = not_null or set() defaults = defaults or {} - assert columns, "Tables must have at least one column" - assert all( - n in columns for n in not_null - ), "not_null set {} includes items not in columns {}".format( - repr(not_null), repr(set(columns.keys())) - ) - assert all( - n in columns for n in defaults - ), "defaults set {} includes items not in columns {}".format( - repr(set(defaults)), repr(set(columns.keys())) - ) + if not columns: + raise ValueError("Tables must have at least one column") + if not all(n in columns for n in not_null): + raise ValueError( + "not_null set {} includes items not in columns {}".format( + repr(not_null), repr(set(columns.keys())) + ) + ) + if not all(n in columns for n in defaults): + raise ValueError( + "defaults set {} includes items not in columns {}".format( + repr(set(defaults)), repr(set(columns.keys())) + ) + ) column_items = list(columns.items()) if column_order is not None: @@ -1295,9 +1302,8 @@ class Database: :param ignore: Set to ``True`` to do nothing if a view with this name already exists :param replace: Set to ``True`` to replace the view if one with this name already exists """ - assert not ( - ignore and replace - ), "Use one or the other of ignore/replace, not both" + if ignore and replace: + raise ValueError("Use one or the other of ignore/replace, not both") create_sql = "CREATE VIEW {name} AS {sql}".format( name=quote_identifier(name), sql=sql ) @@ -1342,9 +1348,13 @@ class Database: tuples """ # foreign_keys is a list of explicit 4-tuples - assert all( + if not all( len(fk) == 4 and isinstance(fk, (list, tuple)) for fk in foreign_keys - ), "foreign_keys must be a list of 4-tuples, (table, column, other_table, other_column)" + ): + raise ValueError( + "foreign_keys must be a list of 4-tuples, " + "(table, column, other_table, other_column)" + ) foreign_keys_to_create = [] @@ -1989,7 +1999,8 @@ class Table(Queryable): :param keep_table: If specified, the existing table will be renamed to this and will not be dropped """ - assert self.exists(), "Cannot transform a table that doesn't exist yet" + if not self.exists(): + raise ValueError("Cannot transform a table that doesn't exist yet") sqls = self.transform_sql( types=types, rename=rename, @@ -2161,8 +2172,10 @@ class Table(Queryable): elif not not_null: pass else: - assert False, "not_null must be a dict or a set or None, it was {}".format( - repr(not_null) + raise ValueError( + "not_null must be a dict or a set or None, it was {}".format( + repr(not_null) + ) ) # defaults= create_table_defaults = { @@ -2870,9 +2883,10 @@ class Table(Queryable): "{}.{}".format(original_quoted, quote_identifier(c)) for c in columns ) fts_table = self.detect_fts() - assert fts_table, "Full-text search is not configured for table '{}'".format( - self.name - ) + if not fts_table: + raise ValueError( + "Full-text search is not configured for table '{}'".format(self.name) + ) fts_table_quoted = quote_identifier(fts_table) virtual_table_using = self.db.table(fts_table).virtual_table_using sql = textwrap.dedent(""" @@ -3119,7 +3133,8 @@ class Table(Queryable): ) if output is not None: - assert len(columns) == 1, "output= can only be used with a single column" + if len(columns) != 1: + raise ValueError("output= can only be used with a single column") if output not in self.columns_dict: self.add_column(output, output_type or "text") @@ -3632,15 +3647,15 @@ class Table(Queryable): if upsert and (not pk and not hash_id): raise PrimaryKeyRequired("upsert() requires a pk") - assert not (hash_id and pk), "Use either pk= or hash_id=" + if hash_id and pk: + raise ValueError("Use either pk= or hash_id=") if hash_id_columns and (hash_id is None): hash_id = "id" if hash_id: pk = hash_id - assert not ( - ignore and replace - ), "Use either ignore=True or replace=True, not both" + if ignore and replace: + raise ValueError("Use either ignore=True or replace=True, not both") all_columns = [] first = True num_records_processed = 0 @@ -3689,9 +3704,10 @@ class Table(Queryable): first_record = cast(Dict[str, Any], first_record) num_columns = len(first_record.keys()) - assert ( - num_columns <= SQLITE_MAX_VARS - ), "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) + if num_columns > SQLITE_MAX_VARS: + raise ValueError( + "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) + ) batch_size = ( 1 if num_columns == 0 @@ -3946,10 +3962,12 @@ class Table(Queryable): :param extra_values: Additional column values to be used only if creating a new record :param strict: Boolean, apply STRICT mode if creating the table. """ - assert isinstance(lookup_values, dict) - assert pk is not None - if extra_values is not None: - assert isinstance(extra_values, dict) + if not isinstance(lookup_values, dict): + raise ValueError("lookup_values must be a dictionary") + if pk is None: + raise ValueError("pk cannot be None") + if extra_values is not None and not isinstance(extra_values, dict): + raise ValueError("extra_values must be a dictionary") combined_values = dict(lookup_values) if extra_values is not None: combined_values.update(extra_values) @@ -4034,9 +4052,10 @@ class Table(Queryable): other_table = self.db.table(other_table, pk=pk) our_id = self.last_pk if lookup is not None: - assert record_or_iterable is None, "Provide lookup= or record, not both" - else: - assert record_or_iterable is not None, "Provide lookup= or record, not both" + if record_or_iterable is not None: + raise ValueError("Provide lookup= or record, not both") + elif record_or_iterable is None: + raise ValueError("Provide lookup= or record, not both") tables = list(sorted([self.name, other_table.name])) columns = ["{}_id".format(t) for t in tables] if m2m_table is not None: diff --git a/tests/test_convert.py b/tests/test_convert.py index 848b39a..ea3fd96 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -77,7 +77,7 @@ def test_convert_output(fresh_db, drop, expected): def test_convert_output_multiple_column_error(fresh_db): table = fresh_db["table"] - with pytest.raises(AssertionError) as excinfo: + with pytest.raises(ValueError) as excinfo: table.convert(["title", "other"], lambda v: v, output="out") assert "output= can only be used with a single column" in str(excinfo.value) diff --git a/tests/test_create.py b/tests/test_create.py index afa7a65..decefcf 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -108,7 +108,7 @@ def test_create_table_with_defaults(fresh_db): def test_create_table_with_bad_not_null(fresh_db): - with pytest.raises(AssertionError): + with pytest.raises(ValueError): fresh_db.create_table( "players", {"name": str, "score": int}, not_null={"mouse"} ) @@ -243,11 +243,11 @@ def test_create_table_column_order(fresh_db, use_table_factory): # If you specify a column that doesn't point to a table, you get an error: (("one_id", "two_id", "three_id"), NoObviousTable), # Tuples of the wrong length get an error: - ((("one_id", "one", "id", "five"), ("two_id", "two", "id")), AssertionError), + ((("one_id", "one", "id", "five"), ("two_id", "two", "id")), ValueError), # Likewise a bad column: ((("one_id", "one", "id2"),), AlterError), # Or a list of dicts - (({"one_id": "one"},), AssertionError), + (({"one_id": "one"},), ValueError), ), ) @pytest.mark.parametrize("use_table_factory", [True, False]) @@ -700,7 +700,7 @@ def test_bulk_insert_more_than_999_values(fresh_db): def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error): record = dict([("c{}".format(i), i) for i in range(num_columns)]) if should_error: - with pytest.raises(AssertionError): + with pytest.raises(ValueError): fresh_db["big"].insert(record) else: fresh_db["big"].insert(record) @@ -1061,7 +1061,7 @@ def test_create_table_numpy(fresh_db): def test_cannot_provide_both_filename_and_memory(): with pytest.raises( - AssertionError, match="Either specify a filename_or_conn or pass memory=True" + ValueError, match="Either specify a filename_or_conn or pass memory=True" ): Database("/tmp/foo.db", memory=True) @@ -1214,7 +1214,7 @@ def test_create_if_not_exists(fresh_db): def test_create_if_no_columns(fresh_db): - with pytest.raises(AssertionError) as error: + with pytest.raises(ValueError) as error: fresh_db["t"].create({}) assert error.value.args[0] == "Tables must have at least one column" diff --git a/tests/test_create_view.py b/tests/test_create_view.py index f0e2855..056e246 100644 --- a/tests/test_create_view.py +++ b/tests/test_create_view.py @@ -15,7 +15,7 @@ def test_create_view_error(fresh_db): def test_create_view_only_arrow_one_param(fresh_db): - with pytest.raises(AssertionError): + with pytest.raises(ValueError): fresh_db.create_view("bar", "select 1 + 2", ignore=True, replace=True) diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 2cb2ca3..d613bb9 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -139,9 +139,9 @@ def test_m2m_lookup(fresh_db): def test_m2m_requires_either_records_or_lookup(fresh_db): people = fresh_db.table("people", pk="id").insert({"name": "Wahyu"}) - with pytest.raises(AssertionError): + with pytest.raises(ValueError): people.m2m("tags") - with pytest.raises(AssertionError): + with pytest.raises(ValueError): people.m2m("tags", {"tag": "hello"}, lookup={"foo": "bar"}) diff --git a/tests/test_recreate.py b/tests/test_recreate.py index 224d182..bce53d5 100644 --- a/tests/test_recreate.py +++ b/tests/test_recreate.py @@ -15,7 +15,7 @@ def test_recreate_ignored_for_in_memory(): def test_recreate_not_allowed_for_connection(): conn = sqlite3.connect(":memory:") try: - with pytest.raises(AssertionError): + with pytest.raises(ValueError): Database(conn, recreate=True) finally: conn.close() From 910a90970dc41748fd92ae5026baa0ac0315c824 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 19:21:34 +0000 Subject: [PATCH 034/120] Add upgrading guide to the documentation New docs page describing the changes needed to upgrade between major versions, with a detailed 3.x to 4.0 section covering CLI, Python API and packaging changes, plus brief sections for the 2.x to 3.0 and 1.x to 2.0 upgrades. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/index.rst | 1 + docs/upgrading.rst | 113 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 docs/upgrading.rst diff --git a/docs/index.rst b/docs/index.rst index 052b790..f190d47 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,5 +40,6 @@ Contents plugins reference cli-reference + upgrading contributing changelog diff --git a/docs/upgrading.rst b/docs/upgrading.rst new file mode 100644 index 0000000..36d720b --- /dev/null +++ b/docs/upgrading.rst @@ -0,0 +1,113 @@ +.. _upgrading: + +=========== + Upgrading +=========== + +This page describes the changes you may need to make to your own code or scripts when upgrading between major versions of ``sqlite-utils``. + +For the full list of changes in every release see the :ref:`changelog`. + +.. _upgrading_3_to_4: + +Upgrading from 3.x to 4.0 +========================= + +Requirements +------------ + +- Python 3.10 or higher is required. +- The ``click`` dependency must be version 8.3.1 or later. + +Command-line changes +-------------------- + +**Type detection is now the default for CSV and TSV imports.** ``sqlite-utils insert`` and ``sqlite-utils upsert`` now detect column types when importing CSV or TSV data - previously every column was created as ``TEXT`` unless you passed ``--detect-types``. To restore the old behavior pass the new ``--no-detect-types`` flag: + +.. code-block:: bash + + sqlite-utils insert data.db rows data.csv --csv --no-detect-types + +Two related things have been removed: + +- The ``SQLITE_UTILS_DETECT_TYPES`` environment variable. +- The old ``-d/--detect-types`` flag itself. Since detection is now the default the flag did nothing - remove it from any scripts that used it. + +**The convert command no longer skips falsey values.** ``sqlite-utils convert`` previously skipped values that evaluated to ``False`` (empty strings, ``0``) unless you passed ``--no-skip-false``. All values are now converted and the ``--no-skip-false`` flag has been removed. + +**drop-table and drop-view check the object type.** ``sqlite-utils drop-table`` now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. If you relied on that (unlikely), use the matching command instead. + +**sqlite-utils tui has moved to a plugin.** The optional terminal interface is now provided by the `sqlite-utils-tui `__ plugin: + +.. code-block:: bash + + sqlite-utils install sqlite-utils-tui + +Python API changes +------------------ + +**db.table() no longer returns views.** ``db.table(name)`` now raises a ``sqlite_utils.db.NoTable`` exception if ``name`` is a SQL view. Use the new ``db.view(name)`` method for views: + +.. code-block:: python + + table = db.table("my_table") + view = db.view("my_view") + +``db["name"]`` still returns either a ``Table`` or a ``View`` depending on what exists in the database. + +**db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily. Two consequences: + +- Errors in your SQL now raise at the ``db.query()`` call site rather than on first iteration. +- Passing a statement that returns no rows - such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause - previously did nothing at all, silently. It now raises a ``ValueError``. Use ``db.execute()`` for statements that do not return rows. + +**Upserts use INSERT ... ON CONFLICT.** Upsert operations now use SQLite's ``INSERT ... ON CONFLICT SET`` syntax rather than the previous ``INSERT OR IGNORE`` followed by ``UPDATE``. If your code depends on the old behavior, pass ``use_old_upsert=True`` to the ``Database()`` constructor - see :ref:`python_api_old_upsert`. + +**Upsert records must include their primary keys.** ``table.upsert()`` and ``table.upsert_all()`` now raise ``sqlite_utils.db.PrimaryKeyRequired`` if a record is missing a value for any primary key column (or has ``None`` for one). Previously such records were quietly inserted as new rows. Relatedly, ``pk=`` is now optional when the table already exists with a primary key - it is detected automatically. + +**Floating point columns are now REAL.** Auto-detected floating point columns are created with the correct SQLite type ``REAL`` instead of ``FLOAT``. Code that inspects column types should expect ``REAL``. + +**Generated schemas use double quotes.** Tables created by this library now wrap table and column names in ``"double-quotes"`` where they previously used ``[square-braces]``. If you compare ``table.schema`` strings against expected values you will need to update them. + +**table.convert() no longer skips falsey values.** Matching the CLI change above, ``table.convert()`` now converts every value. The ``skip_false`` parameter has been removed - previously it defaulted to ``True``, skipping empty strings and other falsey values. + +**Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``. + +**Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation. Changes you may notice: + +- Multi-step operations such as ``table.transform()`` no longer commit an existing transaction you have open - they use savepoints inside it instead. +- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``RuntimeError`` if called while a transaction is open, instead of silently committing it. +- Using ``Database`` as a context manager (``with Database(path) as db:``) closes the connection on exit *without* committing - uncommitted changes from raw ``db.execute()`` calls are rolled back. + +Packaging changes +----------------- + +- ``sqlite-utils`` now uses ``pyproject.toml`` in place of ``setup.py``. +- ``pip`` is now a runtime dependency, used by the ``sqlite-utils install`` and ``uninstall`` commands. + +New features to be aware of +--------------------------- + +Not breaking changes, but new in 4.0 and worth knowing about when you upgrade: + +- A :ref:`database migrations system `, incorporating the functionality of the ``sqlite-migrate`` plugin. If you used that plugin, the built-in system reads the same ``_sqlite_migrations`` table - your applied migrations will not run again. Update your migration files to use ``from sqlite_utils import Migrations``. +- :ref:`db.atomic() ` for nested transaction support. +- ``table.insert_all()`` and ``table.upsert_all()`` accept an iterator of lists or tuples as an alternative to dictionaries - see :ref:`python_api_insert_lists`. + +.. _upgrading_2_to_3: + +Upgrading from 2.x to 3.0 +========================= + +The 3.0 release redesigned search. The breaking changes were minor: + +- ``table.search()`` returns a generator of dictionaries, sorted by relevance. It previously returned a list of tuples sorted by ``rowid``. +- The ``-c`` shortcut for ``--csv`` and the ``-f`` shortcut for ``--fmt`` were removed from the CLI - use the full option names. + +.. _upgrading_1_to_2: + +Upgrading from 1.x to 2.0 +========================= + +The 2.0 release changed the meaning of *upsert*. In 1.x, ``table.upsert()`` and ``table.upsert_all()`` actually performed ``INSERT OR REPLACE`` operations - entirely replacing the existing row. Since 2.0 an upsert updates only the columns you provide, leaving other columns untouched. + +If you want the 1.x behavior, use ``table.insert(..., replace=True)`` or ``table.insert_all(..., replace=True)`` instead. From 509d87b292407ad7350b8fd0e1c57c27b39b7ca8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 19:56:42 +0000 Subject: [PATCH 035/120] Remove review notes files These were working documents for the 4.0 pre-release review - the actionable findings have all been fixed on this branch and the changelog and upgrading guide carry the user-facing record. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- create-table-parser-feedback.md | 167 -------------- fable-review-4.0rc1.md | 375 -------------------------------- 2 files changed, 542 deletions(-) delete mode 100644 create-table-parser-feedback.md delete mode 100644 fable-review-4.0rc1.md diff --git a/create-table-parser-feedback.md b/create-table-parser-feedback.md deleted file mode 100644 index 2b6f727..0000000 --- a/create-table-parser-feedback.md +++ /dev/null @@ -1,167 +0,0 @@ -# Review: `create-table-parser` branch - -Feedback on the `create-table-parser` branch (commits `ed1de15`, `95d6ff4`, -`407db59` — 819-line `create_table_parser.py` plus JSON test corpora), -reviewed as the proposed foundation for 4.1 `transform()` improvements that -make UNIQUE and CHECK constraints supported instead of silently dropped. - -**Verdict: good fit for 4.1 — right design, right release — but it has two -silent-corruption bugs that must be fixed before `transform()` can trust it, -and a few design decisions to settle.** - -## Why 4.1 is the right home - -- The parser is purely additive: a new module and a new capability. Nothing - about it needs to land in 4.0, so it does not constrain or delay the - stable release. -- The `transform()` improvement it enables — preserving CHECK and UNIQUE - constraints instead of silently dropping them — is a behavior change in - the bug-fix direction, appropriate for a minor release with a prominent - changelog note. -- SQLite exposes no introspection for CHECK constraints (and only partial - detail elsewhere), so parsing the `sqlite_master` DDL is the correct — and - only — approach. The module docstring says exactly this. - -## What holds up well (verified empirically) - -- **Corpus results:** all 272 statements in `tests/valid_create_table.json` - parse without crashing; the 230 that are independently executable were - cross-checked against SQLite's own `PRAGMA table_xinfo` — **0 column-name - or primary-key mismatches**. The 29 invalid statements also do not crash - the parser. -- Survived adversarial probing: identifier quoting in all four styles - including embedded escaped quotes (`"col ""x"""`, `[square]`, backticks), - strings containing `,)` and keywords inside CHECK expressions, - `col IN (...)` option extraction with escaped quotes, generated columns - (`GENERATED ALWAYS AS ... STORED` and bare `AS`), FK action clauses - (`ON DELETE SET NULL ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED`), - `WITHOUT ROWID, STRICT` trailers, keyword-named columns (`"unique"`), - multi-word types with parenthesized arguments, and `CREATE TABLE AS - SELECT`. -- The design is genuinely good: recursive-descent functions mirroring the - SQLite grammar are readable; the constraint list as source of truth with - derived accessor properties (`table.checks`, `col.not_null`, - `table.primary_key`) is the right model — adding a constraint kind means - adding a dataclass and a property. The group-capturing tokenizer - (parenthesized groups taken whole so nested commas never reach the parser) - is a clean way to sidestep expression parsing. - -## Must-fix before `transform()` builds on it - -### 1. Comments produce phantom columns (silent corruption) - -The tokenizer does not handle `--` line comments or `/* */` block comments, -and `sqlite_master` stores DDL verbatim. Reproduced: - -``` -CREATE TABLE t ( - a TEXT, -- user's name, (important) - b INTEGER -) -→ columns parsed as ['a', '-'] (real: ['a', 'b']) - -CREATE TABLE t (a TEXT /* legacy, do not use */, b INTEGER) -→ columns parsed as ['a', 'do', 'b'] (real: ['a', 'b']) -``` - -Hand-written schemas — exactly the ones that have CHECK constraints — are -exactly the ones with comments. If `transform()` rebuilds a table from a -model with wrong columns, that is data loss. Comment handling belongs in -`_tokenize` / `_locate_body` / `_split_top_level` (all three scan raw text). - -### 2. Numeric literal defaults are truncated (silent corruption) - -`_tokenize` splits `1.5` into `1` / `.` / `5` and `_default_value` consumes -one token. Reproduced: - -| DDL | parsed default | correct | -|----------------------|----------------|---------| -| `DEFAULT 1.5` | `'1'` | `1.5` | -| `DEFAULT -1.5` | `'-1'` | `-1.5` | -| `DEFAULT 1e-3` | `'1e'` | `1e-3` | -| `DEFAULT x'0102'` | `'x'` | `x'0102'` | - -Re-emitting these in a transform rewrites `DEFAULT 1.5` as `DEFAULT 1` — -silent schema corruption. Notably the corpus already contains -`DEFAULT -45.8e22`, but nothing catches this because the fixtures are -input-only (see next point). Fix by lexing numeric literals (including -sign, decimal point, exponent) and blob literals (`x'...'`) as single -tokens, or by making `_default_value` consume the full literal. - -### 3. No tests are actually executed - -The JSON corpora are not wired into pytest — there is no `test_*.py` on the -branch, so the parser currently has zero executed tests. Beyond wiring the -corpus in, add **expected-output snapshots** (parse each valid statement, -assert the full structured result), not just "doesn't crash": the -`DEFAULT -45.8e22` bug sat undetected in the corpus precisely because only -inputs are recorded. A cheap high-value addition: property test that for -every executable statement, parsed column names / pk / not-null match -`PRAGMA table_xinfo`. - -## Design decisions to settle - -### Discard-vs-error policy for grammar the model does not capture - -The plan is presumably parse → modify model → serialize, which matches how -`transform()` already rebuilds tables. That makes "what the model discards" -the critical list, because anything discarded is silently stripped from the -user's schema on transform: - -- `ON CONFLICT` clauses on PRIMARY KEY / UNIQUE / NOT NULL (parsed by - `_conflict_clause`, thrown away) -- FK `MATCH` and `DEFERRABLE INITIALLY DEFERRED` (parsed, thrown away — - deferred FKs are real in the wild) -- `ASC` / `DESC` in table-level `PRIMARY KEY (a DESC)` / `UNIQUE (...)` - column lists (`_column_list_group` keeps only the leading identifier; - inline single-column PK order *is* captured) -- Unknown column constraints (`_column_constraint` consumes one token and - returns `None`) - -Either capture these in the model, or have `transform()` raise -`TransformError` when the source DDL contains grammar it cannot round-trip — -the honest-failure pattern 3.38 established for un-recreatable indexes. -Silence is the one wrong answer. - -Same question for column renames: a renamed column referenced inside a CHECK -expression (stored as a raw string) cannot be mechanically rewritten, so -`transform(rename=...)` intersecting a check should raise `TransformError` -rather than emit a stale constraint. Likewise dropping a column referenced -by a table-level CHECK or multi-column UNIQUE. - -### Ship it private in 4.1 - -The module defines `Table`, `Column`, and `ForeignKey` — the package already -has `db.Table` and a `db.ForeignKey` namedtuple with different, incompatible -fields (`(table, column, other_table, other_column)` vs -`(columns, table, references, ...)`). Two public `ForeignKey` types in one -package is a confusion trap, and 4.0 is a fresh reminder that public surface -is forever. Recommendation: land it as a private module -(`sqlite_utils._parser` or similar) powering `transform()` in 4.1, promote -to documented public API in a later minor once battle-tested. Public-later -is additive; public-now-retract-later is breaking. - -Related naming nit: the parser's `Table.schema` is the attached-database -name (`main`/`temp`), while `db.Table.schema` is the DDL string. Rename one -(e.g. `database` or `schema_name`) before anything goes public. - -### Housekeeping - -- `create_table_parser.py` lives at the repo root; it needs to move inside - the `sqlite_utils/` package. -- `parse_checks` is documented as a "backwards-compatible helper" but - nothing in sqlite-utils exists for it to be compatible with — stale - comment from an earlier draft. -- The valid corpus is strong on grammar coverage but light on real-world - mess: add fixtures with comments, tabs/newlines in odd places, numeric - and blob defaults, and a couple of schemas generated by sqlite-utils - itself (double-quoted everything) and by common tools. - -## Bottom line - -Hold it for 4.1 exactly as planned — it has no 4.0 dependency and should not -delay the stable release. Before `transform()` builds on it: fix comment -handling and numeric/blob literal lexing, wire the fixtures into pytest with -expected-output snapshots, decide the discard-vs-error policy, and keep the -module private for one release. The bones are solid; both bugs are -tokenizer-level and small. diff --git a/fable-review-4.0rc1.md b/fable-review-4.0rc1.md deleted file mode 100644 index c590b10..0000000 --- a/fable-review-4.0rc1.md +++ /dev/null @@ -1,375 +0,0 @@ -# sqlite-utils 4.0rc1 pre-release review - -Final review before the stable 4.0 release, focused on issues that would be -**breaking changes if fixed after 4.0 ships**. Reviewed at commit `79117b9` -(4.0rc1 plus four commits). Every finding below was verified against the -actual code; the two most serious were reproduced end-to-end. - -Test suite at review time: **1080 passed, 16 skipped**. - -**Verdict: do not tag stable yet.** There are five release blockers, all of -which are small fixes, plus a handful of semantic decisions that are cheap to -make now and breaking to change later. Recommended path: fix the blockers, -make the documented decisions, cut an **rc2**. - ---- - -## Release blockers - -Data loss or wrong-by-default behavior that 4.0 would lock in. - -### 1. `delete_where()` never commits and poisons the connection (data loss) - -`Table.delete_where()` (`sqlite_utils/db.py:2948`) runs its DELETE via a bare -`self.db.execute()` with no `atomic()` wrapper — compare `Table.delete()` at -`db.py:2944`, which wraps correctly. The connection is left -`in_transaction=True`, so every *subsequent* `atomic()` call takes the -savepoint branch (`db.py:430-440`) and never commits either. - -Reproduced end-to-end: - -```python -db = sqlite_utils.Database("dw.db") -db["t"].insert_all([{"id": i} for i in range(3)], pk="id") -db["t"].delete_where("id = ?", [0]) # conn.in_transaction is now True -db["t"].insert({"id": 50}) -db["u"].insert({"a": 1}) -db.close() -# Reopen: rows are [0, 1, 2] — the delete, row 50, AND table u are all gone. -``` - -In 3.x this leak was latent because later operations used `with db.conn:` -which committed the pending work (verified against 3.38, where the same -sequence persists everything). The 4.0 `atomic()` design — "don't commit an -existing transaction" — converts the old latent leak into permanent silent -data loss. - -**`optimize()` (`db.py:2790`) and `rebuild_fts()` (`db.py:2752`) have the -identical leak** — both run `INSERT INTO fts(fts) VALUES(...)` via bare -`self.db.execute()`. The CLI escapes only because `cli.py:341` and -`cli.py:369` wrap the calls in `with db.conn:`. - -Tellingly, the only `delete_where` example in the docs -(`docs/python-api.rst:983-990`) already wraps it in `with db.atomic():`, -papering over the bug, while every other single-op example needs no wrapper. - -**Fix:** wrap all three in `with self.db.atomic():`; update the docs example. -These were the only leaky public write ops found — every other op -(insert/upsert/update/delete/lookup/transform/extract/create/add_column/ -create_index/enable_fts/enable_counts/m2m/convert/duplicate) was verified to -leave `in_transaction=False`. - -### 2. `drop-view` drops tables and `drop-table` drops views, silently - -`cli.py:1728` (`drop-table`) and `cli.py:1800` (`drop-view`) both use -`db[name].drop()`. `Database.__getitem__` dispatches on the actual object -type, and `Table.drop()`/`View.drop()` each issue their matching `DROP` -statement. - -Reproduced: `sqlite-utils drop-view t.db t` deleted **table `t`** with exit -code 0. The mirror case (`drop-table` on a view) also succeeds silently. - -This is a data-loss footgun that contradicts the issue #657 table/view split -that is a headline 4.0 breaking change. Fixing it later converts a silent -"success" into an error — a semver problem, so it must land in 4.0. - -**Fix:** one line each — use `db.table(name)` / `db.view(name)` and catch -`NoTable` / `NoView` for a clean `ClickException`. - -### 3. Post-rc1 `insert({})` change lets `upsert` silently insert rows - -The new `DEFAULT VALUES` branch (commit `b5d0080`, -`db.py:3184-3194`) checks `not list_mode and not all_columns` but never -checks the `upsert` flag: - -- `db["t"].upsert({}, pk="id")` executes `INSERT INTO t DEFAULT VALUES` — a - brand-new row is written — and *then* raises an unrelated `KeyError: 'id'` - from the `last_pk` computation (`db.py:3742-3746`). -- `upsert_all([{}, {}], pk="id")` (2+ records skips the `last_pk` path) - **silently inserts two new rows on every call, no error at all**. - -At rc1 both failed fast with `ZeroDivisionError` — ugly, but zero rows -written. Going from crash-without-mutation to silent insertion under -"upsert" semantics is a regression 4.0 would lock in. - -Relatedly, the compound-pk auto-detection (commit `bfd74a3`, -`db.py:3557-3560`) makes records that *omit* the pk value reachable where -they previously raised `PrimaryKeyRequired` before touching the database: -`upsert_all([{"v": "a"}, {"v": "b"}])` on an existing pk table now inserts -with `id=NULL` (never conflicts) and appends new rows on every call -(verified: two calls → 4 rows). Same for compound pks with a missing -component. Not covered by any test. - -**Fix:** raise a clean error in the empty-record and missing-pk-value paths -when `upsert=True` (an empty record has no pk value, so it can never be an -upsert). - -### 4. `Migrations.apply()` transaction semantics are accidental — decide now - -`sqlite_utils/migrations.py:84-95` runs each migration function and its -tracking-row insert with no transaction wrapper. Verified: a migration that -fails halfway leaves its partial side effects committed, records nothing, -and re-running re-executes the *entire* function including already-applied -statements — the classic double-apply hazard. - -This matches sqlite-migrate, so either behavior is defensible — but 4.0 -freezes the contract: adding per-migration transactions in 4.1 would break -migrations that manage their own transactions or run statements illegal -inside one (`VACUUM`, some `PRAGMA`s). - -**Decide now:** wrap each migration in `db.atomic()`, or explicitly document -"migrations are not transactional; write idempotent steps" so the current -behavior is the contract rather than an accident. - -Also decide now: **`_AppliedMigration.applied_at` is annotated -`datetime.datetime` (`migrations.py:21`) but is a `str` at runtime** -(`migrations.py:67` passes the TEXT column value straight through), and the -type is published via autodoc (`docs/migrations.rst:168-171`). Changing the -annotation to `str` now is a free one-liner (and the sqlite-migrate- -compatible choice); "fixing" it to a real datetime later breaks every -consumer doing string operations. - -### 5. `enable_wal()` / `disable_wal()` commit open transactions - -`ensure_autocommit_off()` (`db.py:456-472`) assigns `conn.isolation_level`, -and CPython's setter **commits any pending transaction** as a side effect. -Verified consequences: - -- `with db.atomic(): insert(2); db.enable_wal(); insert(3); raise` → **all - rows persist despite the exception**, directly contradicting the - documented rollback guarantee (`docs/python-api.rst:255`), which is *the* - headline 4.0 semantic. -- A user's own `BEGIN` + insert + `enable_wal()` → their subsequent - `rollback()` is a no-op; the insert persists. This is exactly the - "unexpectedly committing an existing transaction" bug class `atomic()` was - introduced to eliminate (changelog, issue #755). - -Pre-existing in 3.x, but 4.0 is the moment to fix. - -**Fix:** make `enable_wal`/`disable_wal` raise (or skip the isolation dance) -when `conn.in_transaction`. - ---- - -## Decisions to make now — cheap today, breaking after 4.0 - -### `Database.__enter__` / `__exit__` contract is unpinned - -`__exit__` only calls `self.close()` (`db.py:408-417`), which silently rolls -back uncommitted changes. Verified: a raw `db.execute("insert ...")` inside -`with Database(p) as db:` is discarded on exit. This diverges from -`sqlite3.Connection`'s own context manager (commits on success, does *not* -close). The docs (`docs/python-api.rst:139-145`) say only "automatically -close the connection" — no mention of rollback — and **no test anywhere uses -`with Database(...)`**, so the contract is completely unpinned. Whichever way -this might later be "improved" is a breaking behavior change. Decide now, -add one docs sentence ("any uncommitted changes are rolled back") and a -pinning test. Note blockers 1–2 make this worse: after a `delete_where`, -exiting the `with` block discards everything. - -### `atomic()` is broken on Python 3.12+ `autocommit` connections - -`db.py:441-453` uses `conn.commit()`/`conn.rollback()`, documented no-ops -when `Connection.autocommit=True`. Verified on 3.13: - -- `autocommit=True` connection → a successful `atomic()` block leaves the - transaction permanently open; rollback on exception is also a no-op. -- `autocommit=False` connection → the connection is always in a transaction, - so `atomic()` always takes the savepoint branch and never commits - anything. - -`Database(filename_or_conn)` accepts arbitrary connections (`db.py:397`). -Either handle these modes (issue literal `COMMIT`/`ROLLBACK` statements, or -reject such connections) or document supported connection modes in -`python_api_atomic` now — changing commit timing later is breaking. -(Legacy `isolation_level=None` connections were verified to work correctly.) - -### `atomic()` savepoint-branch semantics are intended but undocumented - -If the connection is already in a transaction (user ran `BEGIN` or raw DML), -`atomic()` becomes a savepoint and its successful exit does **not** persist -anything until the user commits (verified: a user `rollback()` discards the -atomic block's writes). Reasonable semantics, but the `conn.in_transaction` -sniffing must be documented before it is locked — the docs currently -describe nesting only in terms of `atomic()`-inside-`atomic()`. - -### `atomic()` issues plain deferred `BEGIN` - -`db.py:442` — no way to request `BEGIN IMMEDIATE`. A future switch of the -default would change locking/`SQLITE_BUSY` behavior (breaking); adding an -`immediate=` parameter later is fine. Documenting "deferred" now costs one -sentence. - -### No-op `-d/--detect-types` flag: keep or kill - -`cli.py:942-947` still defines `-d/--detect-types` on `insert`/`upsert` -(help says "(default)"), deliberately tested as a no-op -(`tests/test_cli.py:2299,2322`). The `detect_types` parameter of -`insert_upsert_implementation` (`cli.py:1002`) is dead code — the body only -reads `no_detect_types`. If the flag is ever to be removed, 4.0 is the only -chance; keeping it as back-compat is defensible, but note -`--detect-types --no-detect-types` together silently favors the latter with -no conflict error. - -### `--stop-before` is silently ignored for old `sqlite_migrate` objects - -The duck-typing in `_compatible_migration_set` (`cli.py:3286-3289`) exists -precisely so migration files still doing `from sqlite_migrate import -Migrations` keep working — but `cli.py:3393` always passes `stop_before=` as -a **list**, and the old plugin's `apply()` does `if name == stop_before:` -against a string. Verified with the released 0.1b0: `--stop-before step2` -applied `step2` anyway. The CLI applies the exact migration the user asked -it not to, on the one upgrade path the duck typing exists to support. - -Related: a **typo'd `--stop-before` name silently applies everything**, -including the migration you meant to stop before (unknown names simply never -match; no error from the CLI). Validation has to live in the CLI ("each -`--stop-before` value must match at least one set") because unqualified -names legitimately fan out across sets. Adding that error post-4.0 turns -currently-succeeding invocations into failures — decide now. - -### Public-API validation via bare `assert` - -User-facing errors raised via `assert` throughout `db.py` (370, 396, 1016, -1950, 2829, 3028, 3565, 3571, 3967, …). They vanish under `python -O`, and -`db.py:3028` carries `# TODO: Test this works (rolls back) - use better -exception:`. Converting `AssertionError` to `ValueError` later changes -exception types callers may catch — best done in a major, if ever. - ---- - -## Should-fix polish (non-breaking later, but ugly for a stable release) - -- **`insert`/`upsert` into a view name prints a raw traceback** — - `NoTable("Table v is actually a view")` from `cli.py:1154` is not - converted to `ClickException` (contrast `duplicate` at `cli.py:1676`). -- **Misleading `NoView` message** — `db.view("t")` where `t` is a table says - "View t does not exist" (`db.py:661`); the mirror case helpfully says - "Table v is actually a view" (`db.py:650`). -- **Stale `__getitem__` docstring** (`db.py:502`) says it "returns a Table - object"; it returns a `View` for views. Feeds autodoc. -- **Wrong `hash_id` doc in the `Table` docstring** (`db.py:1588`) — says - bool; it's a column-name string (`Optional[str]`). -- **`sqlite-utils migrate --list` writes to the database** — - `pending()`/`applied()` call `ensure_migrations_table` - (`migrations.py:48,65`), so `--list` creates the tracking table, performs - the one-way legacy sqlite-migrate schema upgrade, and — since `db_path` - has no `exists=True` (`cli.py:3338-3340`) — creates the database file - itself. A read-looking operation should not do any of that. -- **`applied()` has no `ORDER BY`** (`migrations.py:66-71`) — relies on - rowid order; `tests/test_cli_migrate.py:104-107` asserts that order. - Add `order_by="id"` to make `--list` deterministic by contract. -- **Duplicate migration names within a set**: both functions execute (side - effects committed) before an opaque `IntegrityError`; only the first is - recorded (`migrations.py:36-42`). A `ValueError` at registration would be - cheap and is effectively non-breaking to add now. -- **`pending()`/`applied()` return underscore-private dataclasses** - (`_Migration`/`_AppliedMigration`) that are excluded from autodoc - (`docs/migrations.rst:171`), so the `.name`/`.applied_at`/`.fn` fields - users must access are undocumented API. -- **`set:name` colon syntax is unvalidated** (`cli.py:3328` uses - `partition(":")`) — a set or migration name containing `:` can never be - targeted. Document/validate "no colons in names" now. -- **Bare `@migrations` decorator** gives an opaque `TypeError` - (`migrations.py:30`); a helpful message would be purely additive. -- **Tracer never sees transaction statements** — `atomic()` uses - `self.conn.execute()`/`conn.commit()` directly (`db.py:432-450`), - bypassing the tracer. Routing through `self.execute()` later would change - tracer output that downstream tests may assert on — cheap to decide now. -- **Order-dependent empty-dict semantics** (from `b5d0080`): - `insert_all([{}, {"v": "hi"}])` gives the empty record its column DEFAULTs, - but `insert_all([{"v": "hi"}, {}])` inserts explicit `NULL` for it — a - `NOT NULL ... DEFAULT` column succeeds in one ordering and raises - `IntegrityError` in the other. -- **Batch-size cliff when the first record is `{}`** (`db.py:3625-3629`) — - `num_columns` comes from the first record only, forcing `batch_size=1` - for the entire stream. Performance-only, degenerate input. -- **`pyproject.toml` build-system underpinned** — `requires = ["setuptools"]` - while the PEP 639 `license = "Apache-2.0"` string needs setuptools ≥ - 77.0.3; non-isolated sdist builds on older setuptools will fail. Pin - `setuptools>=77`. -- **Undocumented post-rc1 behavior changes** — neither upsert pk - auto-detection (`bfd74a3`) nor `insert({})`-uses-DEFAULT-VALUES - (`b5d0080`) is mentioned in `docs/` or the changelog. Note also that - `db.table(name).insert({})` only works for *existing* tables — on a - missing table it still raises `AssertionError: Tables must have at least - one column`, so the issue #759 title scenario is only partially covered. - ---- - -## Verified sound — no action needed - -- **sqlite-migrate on-disk compatibility** (the scariest area): tested - against the actual 0.1b0 sdist from PyPI. Same `_sqlite_migrations` table; - the legacy compound-pk `(migration_set, name)` schema is detected via - `table.pks != ["id"]` and upgraded in place (`migrations.py:97-117`); rows - are preserved and previously-applied migrations are **not** re-applied. - Both timestamp formats are identical. Tests cover both legacy layouts - (`tests/test_migrations.py:83-110`). No data-corruption risk for upgrading - users. The built-in `migrate` command is registered *after* plugin hooks - (`cli.py:3415-3417`), so a still-installed old plugin's command is - correctly overridden. -- Migration ordering is definition order (documented and tested); identity - is `(set name, migration name)` with a unique index; same name across sets - is fine. The new CLI is a strict superset of the old plugin's; file - discovery is now `sorted()` where the plugin iterated an unordered set. -- Nested savepoint rollback/release logic including exception paths - (`tests/test_atomic.py:44-120`); commit-time deferred-FK failure rolls - back cleanly; `transform()` inside an open transaction correctly uses - `PRAGMA defer_foreign_keys`; `_executescript` statement-splitting avoids - sqlite3's implicit commit. Savepoint naming via `secrets.token_hex(16)` is - collision-free. -- Compound-pk upsert detection happy path works for single and compound pks - on both the `ON CONFLICT` and `use_old_upsert` implementations; explicit - `pk=` still wins over detection; rowid/missing/hash_id tables still raise - `PrimaryKeyRequired` (`tests/test_upsert.py:52-97`). -- The table/view split (#657) is otherwise complete: `db.table()` on a view - raises `NoTable`, never returns a `View`; `db.view()` accepts no - table-only kwargs. The tracer-test fix (`401fb69`) and click pin - (`79117b9`, dev-group only) are correct. -- `__all__` exports are coherent (`Database`, `Migrations`, - `suggest_column_types`, `hookimpl`, `hookspec`); every `sqlite_utils.X` - docs reference resolves. The `pip` runtime dependency is genuinely - required (`cli.py:2967-2983` uses `run_module("pip")` for - `install`/`uninstall`). Classifiers match `requires-python`. No - deprecation debt: zero `DeprecationWarning` markers; clean under - `-W error::DeprecationWarning`. - ---- - -## The transform() incoming-FK branch: hold for 4.1 - -The `update_incoming_fks` work on `claude/investigate-transform-fk-KMTZ7` is -right to hold: - -- It is purely additive (`update_incoming_fks: bool = False` keyword + - `--update-incoming-fks` flag; default behavior unchanged), so 4.1 is safe - semver-wise. -- The branch predates the `atomic()` refactor — it still uses - `with self.db.conn:` directly in `transform()`, the exact pattern 4.0 - eliminated (#755) — and has real issues: `_skip_fk_validation` is mutable - state on the whole `Database` object, and the incoming-FK rebuilds execute - `transform_sql()` output raw for referencing tables, silently dropping - their indexes (the 3.38 index-recreation logic only runs for the primary - table). -- One decision belongs to 4.0: whether `transform()` should eventually - refuse/warn *by default* when a rename would leave dangling incoming FKs. - Adding a warning later is fine; making it an error by default later is - breaking. If strict-by-default is the desired end state, add just the - guard in 4.0 and ship the auto-update machinery in 4.1. - ---- - -## Suggested path to stable - -1. Fix the five blockers — three `atomic()` wrappers (`delete_where`, - `optimize`, `rebuild_fts`), two CLI lookups (`drop-table`/`drop-view`), - the `upsert` guard in the DEFAULT VALUES / missing-pk paths, the - `applied_at: str` annotation, and the `in_transaction` guard in - `enable_wal`/`disable_wal`. -2. Make and document the decisions: `apply()` transactionality, - `Database.__exit__` semantics (+ pinning test), `atomic()` supported - connection modes and deferred-`BEGIN` note, `--detect-types` keep/kill, - `--stop-before` validation and old-plugin compat. -3. Cut **rc2** — the transaction-semantics fixes deserve one more candidate - round before the semantics freeze. From 70055d74323f5a4d15d27f1d13b83590ade68258 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 20:16:15 +0000 Subject: [PATCH 036/120] Document the transaction model as a first-class concept New top-level 'Transactions and saving your changes' section in the Python API documentation, stating the fundamental contract up front: every write method commits its own changes before returning, no commit() call is ever needed. The db.atomic() documentation moves under it, alongside new subsections covering the two cases where the user owns the commit - raw db.execute() writes and manually managed transactions - and the supported connection modes. Cross-referenced from Getting started (the first insert example now says the rows are saved immediately), a warning on db.execute(), the closing-a-database section and the upgrading guide. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/python-api.rst | 132 ++++++++++++++++++++++++++++++++------------ docs/upgrading.rst | 2 +- 2 files changed, 98 insertions(+), 36 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index e888b7d..2c43070 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -33,6 +33,8 @@ Here's how to create a new SQLite database file containing a new ``chickens`` ta "color": "black", }]) +The inserted rows are saved to the database file straight away - methods like ``insert_all()`` commit their own changes, so no ``commit()`` call is needed. See :ref:`python_api_transactions` for how this works. + You can loop through those rows like this: .. code-block:: python @@ -144,7 +146,7 @@ The ``Database`` object also works as a context manager, which will automaticall db["my_table"].insert({"name": "Example"}) # Connection is automatically closed here -Exiting the block closes the connection without committing, so any changes that are still inside an open transaction at that point - for example writes made with raw ``db.execute()`` calls - will be rolled back. Commit explicitly, or use methods such as ``.insert()`` and the ``db.atomic()`` context manager which commit their own transactions. +Exiting the block closes the connection without committing, so any changes that are still inside an open transaction at that point - for example writes made with raw ``db.execute()`` calls - will be rolled back. Commit explicitly, or use methods such as ``.insert()`` and the ``db.atomic()`` context manager which commit their own transactions - see :ref:`python_api_transactions`. .. _python_api_attach: @@ -245,40 +247,8 @@ The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around Other cursor methods such as ``.fetchone()`` and ``.fetchall()`` are also available, see the `standard library documentation `__. -.. _python_api_atomic: - -Transactions with db.atomic() ------------------------------ - -Use ``db.atomic()`` to group multiple operations in a transaction: - -.. code-block:: python - - with db.atomic(): - db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") - db.table("dogs").insert({"id": 2, "name": "Pancakes"}) - -If an exception is raised, changes made inside the block will be rolled back. - -``db.atomic()`` can be nested. Nested blocks use SQLite savepoints, so an exception in an inner block can roll back to that savepoint without rolling back the entire outer transaction: - -.. code-block:: python - - with db.atomic(): - db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") - try: - with db.atomic(): - db.table("dogs").insert({"id": 2, "name": "Pancakes"}) - raise ValueError("skip this one") - except ValueError: - pass - db.table("dogs").insert({"id": 3, "name": "Marnie"}) - -The transaction is opened with a deferred ``BEGIN`` - SQLite takes the necessary locks when the first statement inside the block runs. - -The same savepoint mechanism used for nesting applies if the connection is already inside a transaction when ``db.atomic()`` is called - for example if you executed ``BEGIN`` yourself, or made writes using raw ``db.execute()`` calls. In that case the block becomes a savepoint within your transaction: exiting the block releases the savepoint, but nothing is committed to disk until your outer transaction commits. - -``db.atomic()`` is designed for connections in Python's default transaction handling mode. Connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options are not supported, because ``commit()`` and ``rollback()`` behave differently on those connections. +.. warning:: + Unlike the table methods described elsewhere in this documentation, ``db.execute()`` does **not** commit. If you use it to modify the database you are responsible for committing the change yourself - see :ref:`python_api_transactions_execute`. .. _python_api_parameters: @@ -308,6 +278,98 @@ Named parameters using ``:name`` can be filled using a dictionary: In this example ``next()`` is used to retrieve the first result in the iterator returned by the ``db.query()`` method. +.. _python_api_transactions: + +Transactions and saving your changes +==================================== + +Every method in this library that writes to the database - ``insert()``, ``upsert()``, ``update()``, ``delete()``, ``delete_where()``, ``transform()``, ``create_table()``, ``create_index()``, ``enable_fts()`` and the rest - runs inside its own transaction and commits it before returning. Your changes are saved to disk as soon as the method call finishes: + +.. code-block:: python + + db = Database("data.db") + db.table("news").insert({"headline": "Dog wins award"}) + # The new row is already saved - no commit() required + +You never need to call ``commit()`` after using these methods, and you do not need to close the database to persist your changes. + +There are exactly three situations where you need to think about transactions: + +1. You want to group several write operations together, so they either all succeed or all fail - use :ref:`db.atomic() `. +2. You are executing raw SQL writes with ``db.execute()``, which does *not* commit automatically - see :ref:`python_api_transactions_execute`. +3. You are :ref:`managing a transaction yourself `, in which case the library will never commit it for you. + +.. _python_api_atomic: + +Grouping changes with db.atomic() +--------------------------------- + +Use ``db.atomic()`` to group multiple operations in a single transaction: + +.. code-block:: python + + with db.atomic(): + db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") + db.table("dogs").insert({"id": 2, "name": "Pancakes"}) + +The transaction commits when the block exits. If an exception is raised, changes made inside the block will be rolled back. + +``db.atomic()`` can be nested. Nested blocks use SQLite savepoints, so an exception in an inner block can roll back to that savepoint without rolling back the entire outer transaction: + +.. code-block:: python + + with db.atomic(): + db.table("dogs").insert({"id": 1, "name": "Cleo"}, pk="id") + try: + with db.atomic(): + db.table("dogs").insert({"id": 2, "name": "Pancakes"}) + raise ValueError("skip this one") + except ValueError: + pass + db.table("dogs").insert({"id": 3, "name": "Marnie"}) + +The transaction is opened with a deferred ``BEGIN`` - SQLite takes the necessary locks when the first statement inside the block runs. + +.. _python_api_transactions_execute: + +Raw SQL writes with db.execute() +-------------------------------- + +:ref:`db.execute() ` is a thin wrapper around the underlying ``sqlite3`` connection, and it follows that connection's rules rather than this library's: a write statement opens a transaction that stays open until something commits it. Commit explicitly: + +.. code-block:: python + + db.execute("insert into news (headline) values (?)", ["Dog wins award"]) + db.conn.commit() + +Or wrap the calls in ``db.atomic()``, which commits for you: + +.. code-block:: python + + with db.atomic(): + db.execute("insert into news (headline) values (?)", ["Dog wins award"]) + +If you do neither, the write can be deceptive: reads on the same connection will see the new row, making it look saved, but the open transaction will be rolled back when the connection closes and the row will be gone. + +.. _python_api_transactions_manual: + +Managing transactions yourself +------------------------------ + +You can take full manual control using ``db.execute("begin")`` (or any raw write, as above) followed by ``db.conn.commit()`` or ``db.conn.rollback()``. + +The library will never commit a transaction you opened. If you call write methods such as ``insert()`` - or use ``db.atomic()`` - while your transaction is open, they participate in it using SQLite savepoints instead of committing: exiting an ``atomic()`` block releases its savepoint, but nothing is saved to disk until you commit the outer transaction yourself. If you roll back, their changes are rolled back too. + +Two related safeguards to be aware of: + +- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``RuntimeError`` if called while a transaction is open, because changing the journal mode would commit it as a side effect. +- Closing the database - explicitly with ``db.close()``, or by exiting a ``with Database(...) as db:`` block - rolls back any transaction that is still open, see :ref:`python_api_close`. + +Supported connection modes +-------------------------- + +``db.atomic()`` and the automatic per-method transactions are designed for connections in Python's default transaction handling mode. Connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options are not supported, because ``commit()`` and ``rollback()`` behave differently on those connections. + .. _python_api_table: Accessing tables diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 36d720b..553afd4 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -72,7 +72,7 @@ Python API changes **Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``. -**Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation. Changes you may notice: +**Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice: - Multi-step operations such as ``table.transform()`` no longer commit an existing transaction you have open - they use savepoints inside it instead. - ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``RuntimeError`` if called while a transaction is open, instead of silently committing it. From adea475a618e94d037caf8cc17fec3a3537a58b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 20:44:04 +0000 Subject: [PATCH 037/120] Remove View.enable_fts() The method existed only to raise NotImplementedError, since full-text search is not supported for views, and it showed up in the generated API reference as a documented View method. Calling enable_fts() on a View now raises AttributeError like any other missing method. The sqlite-utils enable-fts command uses db.table() and shows a clean error when pointed at a view instead of a traceback. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 1 + docs/upgrading.rst | 2 ++ sqlite_utils/cli.py | 4 ++-- sqlite_utils/db.py | 6 ------ tests/test_fts.py | 22 ++++++++++++++++++---- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0b41875..532b23b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Unreleased ---------- +- **Breaking change**: The ``View`` class no longer has an ``enable_fts()`` method. It existed only to raise ``NotImplementedError``, since full-text search is not supported for views - calling it now raises ``AttributeError`` instead, and the method no longer appears in the API reference. The ``sqlite-utils enable-fts`` command shows a clean error when pointed at a view. - **Breaking change**: Python API validation errors now raise ``ValueError`` instead of ``AssertionError``. Previously invalid arguments - such as ``create_table()`` with no columns, ``transform()`` on a table that does not exist, or passing both ``ignore=True`` and ``replace=True`` - were rejected using bare ``assert`` statements, which are silently skipped when Python runs with the ``-O`` flag. Code that caught ``AssertionError`` for these cases should catch ``ValueError`` instead. - **Breaking change**: The no-op ``-d/--detect-types`` flag has been removed from the ``insert`` and ``upsert`` commands. Type detection has been the default for CSV/TSV data since 4.0a1, so the flag did nothing - invocations using it should simply drop it. ``--no-detect-types`` remains available to disable detection. - **Breaking change**: ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` take effect without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 553afd4..e6f4d4d 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -70,6 +70,8 @@ Python API changes **table.convert() no longer skips falsey values.** Matching the CLI change above, ``table.convert()`` now converts every value. The ``skip_false`` parameter has been removed - previously it defaulted to ``True``, skipping empty strings and other falsey values. +**View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method. + **Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``. **Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index a2f881a..d609ad5 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -702,14 +702,14 @@ def enable_fts( _register_db_for_cleanup(db) _load_extensions(db, load_extension) try: - db[table].enable_fts( + db.table(table).enable_fts( column, fts_version=fts_version, tokenize=tokenize, create_triggers=create_triggers, replace=replace, ) - except OperationalError as ex: + except (NoTable, OperationalError) as ex: raise click.ClickException(str(ex)) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a0ca6ed..49d9f63 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -4317,12 +4317,6 @@ class View(Queryable): if not ignore: raise - def enable_fts(self, *args: object, **kwargs: object) -> None: - "``enable_fts()`` is supported on tables but not on views." - raise NotImplementedError( - "enable_fts() is supported on tables but not on views" - ) - def jsonify_if_needed(value: object) -> object: if isinstance(value, decimal.Decimal): diff --git a/tests/test_fts.py b/tests/test_fts.py index 11df898..3f7c5a9 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -461,12 +461,12 @@ def test_enable_fts_replace_handles_legacy_bracket_quoted_content_table(): assert 'content="books"' in db["books_fts"].schema -def test_enable_fts_error_message_on_views(): +def test_view_has_no_enable_fts(): db = Database(memory=True) db.create_view("hello", "select 1 + 1") - with pytest.raises(NotImplementedError) as e: - db["hello"].enable_fts() # type: ignore[call-arg] - assert e.value.args[0] == "enable_fts() is supported on tables but not on views" + # Views deliberately do not have an enable_fts() method + with pytest.raises(AttributeError): + db["hello"].enable_fts() # type: ignore[union-attr] @pytest.mark.parametrize( @@ -718,3 +718,17 @@ def test_search_quote(fresh_db): list(table.search(query)) # No exception with quote=True list(table.search(query, quote=True)) + + +def test_enable_fts_cli_on_view_errors(tmpdir): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["t"].insert({"text": "hello"}) + db.create_view("v", "select * from t") + db.close() + from click.testing import CliRunner + from sqlite_utils import cli as cli_module + + result = CliRunner().invoke(cli_module.cli, ["enable-fts", db_path, "v", "text"]) + assert result.exit_code == 1 + assert result.output.strip() == "Error: Table v is actually a view" From 0fca5908c14f7e14651def56abb4acff239ea89f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 22:19:47 +0000 Subject: [PATCH 038/120] Group the Unreleased changelog into breaking changes and everything else Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 532b23b..04d5405 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,18 +9,24 @@ Unreleased ---------- -- **Breaking change**: The ``View`` class no longer has an ``enable_fts()`` method. It existed only to raise ``NotImplementedError``, since full-text search is not supported for views - calling it now raises ``AttributeError`` instead, and the method no longer appears in the API reference. The ``sqlite-utils enable-fts`` command shows a clean error when pointed at a view. -- **Breaking change**: Python API validation errors now raise ``ValueError`` instead of ``AssertionError``. Previously invalid arguments - such as ``create_table()`` with no columns, ``transform()`` on a table that does not exist, or passing both ``ignore=True`` and ``replace=True`` - were rejected using bare ``assert`` statements, which are silently skipped when Python runs with the ``-O`` flag. Code that caught ``AssertionError`` for these cases should catch ``ValueError`` instead. -- **Breaking change**: The no-op ``-d/--detect-types`` flag has been removed from the ``insert`` and ``upsert`` commands. Type detection has been the default for CSV/TSV data since 4.0a1, so the flag did nothing - invocations using it should simply drop it. ``--no-detect-types`` remains available to disable detection. -- **Breaking change**: ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` take effect without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. +Breaking changes: + +- ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` take effect without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. +- Python API validation errors now raise ``ValueError`` instead of ``AssertionError``. Previously invalid arguments - such as ``create_table()`` with no columns, ``transform()`` on a table that does not exist, or passing both ``ignore=True`` and ``replace=True`` - were rejected using bare ``assert`` statements, which are silently skipped when Python runs with the ``-O`` flag. Code that caught ``AssertionError`` for these cases should catch ``ValueError`` instead. +- ``table.upsert()`` and ``table.upsert_all()`` now raise ``PrimaryKeyRequired`` if a record is missing a value for any primary key column, or has a value of ``None`` for one. Previously such records - which can never match an existing row - were quietly inserted as brand new rows, or triggered a confusing ``KeyError`` after the insert had already taken place. +- ``db.enable_wal()`` and ``db.disable_wal()`` now raise a ``RuntimeError`` if called while a transaction is open. Previously they would silently commit the open transaction as a side effect of changing the journal mode, breaking the rollback guarantee of ``db.atomic()`` and of user-managed transactions. +- The ``View`` class no longer has an ``enable_fts()`` method. It existed only to raise ``NotImplementedError``, since full-text search is not supported for views - calling it now raises ``AttributeError`` instead, and the method no longer appears in the API reference. The ``sqlite-utils enable-fts`` command shows a clean error when pointed at a view. +- The no-op ``-d/--detect-types`` flag has been removed from the ``insert`` and ``upsert`` commands. Type detection has been the default for CSV/TSV data since 4.0a1, so the flag did nothing - invocations using it should simply drop it. ``--no-detect-types`` remains available to disable detection. + +Everything else: + - Fixed a bug where ``table.delete_where()``, ``table.optimize()`` and ``table.rebuild_fts()`` did not commit their changes, leaving the connection inside an open transaction. Their work - and any subsequent writes - could then be silently rolled back when the connection was closed. All three now use ``db.atomic()``, consistent with the other write methods. - The ``sqlite-utils drop-table`` command now refuses to drop a view, and ``drop-view`` refuses to drop a table. Previously each would silently drop the wrong type of object if the name matched. Both now exit with an error suggesting the correct command to use. - Migrations applied by the new :ref:`migrations system ` now run inside a transaction, together with the record of the migration having been applied. If a migration raises an exception its changes are rolled back and it stays pending, so it can be safely re-applied after the error is fixed. Migrations that cannot run inside a transaction, such as those executing ``VACUUM``, can opt out using ``@migrations(transactional=False)`` - see :ref:`migrations_transactions`. - ``table.upsert()`` and ``table.upsert_all()`` now detect the primary key or compound primary key of an existing table, so the ``pk=`` argument is no longer required when upserting into a table that already has a primary key. -- ``table.upsert()`` and ``table.upsert_all()`` now raise ``PrimaryKeyRequired`` if a record is missing a value for any primary key column, or has a value of ``None`` for one. Previously such records - which can never match an existing row - were quietly inserted as brand new rows, or triggered a confusing ``KeyError`` after the insert had already taken place. - ``db.table(table_name).insert({})`` can now be used to insert a row consisting entirely of default values into an existing table, using ``INSERT INTO ... DEFAULT VALUES``. (:issue:`759`) -- ``db.enable_wal()`` and ``db.disable_wal()`` now raise a ``RuntimeError`` if called while a transaction is open. Previously they would silently commit the open transaction as a side effect of changing the journal mode, breaking the rollback guarantee of ``db.atomic()`` and of user-managed transactions. - Improvements to the ``sqlite-utils migrate`` command: ``--stop-before`` values that do not match any known migration are now an error instead of being silently ignored, ``--stop-before`` now works correctly with migration files that still use the older ``sqlite_migrate.Migrations`` class, and ``--list`` is now a read-only operation that no longer creates the database file or the migrations tracking table. ``migrations.applied()`` now returns migrations in the order they were applied. +- New documentation: :ref:`python_api_transactions` describes how transactions work and when changes are committed, and a new :ref:`upgrading` page details the changes needed to move between major versions. .. _v4_0rc1: From ff76fb7740a1f280831713274b90c07a70f8dc86 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 22:20:37 +0000 Subject: [PATCH 039/120] Clarify that db.atomic() is fine inside transactional migrations The manual-transaction warning could be read as covering db.atomic() too. Nested atomic() blocks are safe - they become savepoints inside the migration's transaction, preserving the all-or-nothing rollback guarantee. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/migrations.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/migrations.rst b/docs/migrations.rst index 61709c4..1bd1457 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -87,7 +87,7 @@ Some operations cannot run inside a transaction, for example ``VACUUM`` or chang A migration registered with ``transactional=False`` runs without a wrapping transaction, so if it fails part way through any changes it already made will not be rolled back, and re-applying will run the whole function again. -Avoid calling ``db.conn.commit()`` or otherwise managing transactions manually inside a transactional migration - register the migration with ``transactional=False`` if it needs to control its own transactions. +Avoid calling ``db.conn.commit()`` or otherwise managing transactions manually inside a transactional migration - register the migration with ``transactional=False`` if it needs to control its own transactions. Using ``with db.atomic():`` blocks inside a migration is fine: they nest as savepoints within the migration's transaction, so the migration as a whole still commits or rolls back as a single unit. See :ref:`python_api_transactions`. Applying migrations using the CLI ================================= From 6c88067ab76b9597fb1c538c53164632526a2891 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 22:25:28 +0000 Subject: [PATCH 040/120] Explain why the Database context manager rolls back instead of committing Expands the closing-a-database docs: exit is equivalent to close(), open transactions can only come from manual transaction control, auto-committing could silently persist half-finished work, and the behavior deliberately differs from sqlite3.Connection's own context manager. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/python-api.rst | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 2c43070..163dad5 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -146,7 +146,22 @@ The ``Database`` object also works as a context manager, which will automaticall db["my_table"].insert({"name": "Example"}) # Connection is automatically closed here -Exiting the block closes the connection without committing, so any changes that are still inside an open transaction at that point - for example writes made with raw ``db.execute()`` calls - will be rolled back. Commit explicitly, or use methods such as ``.insert()`` and the ``db.atomic()`` context manager which commit their own transactions - see :ref:`python_api_transactions`. +Exiting the block is equivalent to calling ``db.close()``: the connection is +closed and any transaction still open at that point is rolled back. This +matches SQLite's own behavior when a connection closes. + +This rarely matters in practice. Every method in this library that writes to +the database commits its own changes before returning, so a transaction can +only be left open here if you opened it yourself - through raw ``db.execute()`` +writes or an explicit ``BEGIN``. In that case the decision to commit stays +with you: committing automatically on exit could silently persist +half-finished work, for example if your code returned early from the block. +Commit explicitly with ``db.conn.commit()``, or wrap your writes in +``db.atomic()`` which commits for you. + +Note this differs from the ``sqlite3.Connection`` context manager in the +standard library, which commits on success but does not close the connection. +See :ref:`python_api_transactions` for the full transaction model. .. _python_api_attach: From 788c393a30dde58257ca76fa249ee0842c17b2e3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 22:31:22 +0000 Subject: [PATCH 041/120] Add db.begin(), db.commit() and db.rollback() methods Manual transaction control previously required mixing raw db.execute("begin") strings with methods on db.conn. These thin wrappers give the documentation and callers a single consistent idiom. Docs updated to use them throughout. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 1 + docs/migrations.rst | 2 +- docs/python-api.rst | 19 +++++++++++++++---- sqlite_utils/db.py | 25 +++++++++++++++++++++++++ tests/test_atomic.py | 34 +++++++++++++++++++++++++++++++++- 5 files changed, 75 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 04d5405..baff4af 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -26,6 +26,7 @@ Everything else: - ``table.upsert()`` and ``table.upsert_all()`` now detect the primary key or compound primary key of an existing table, so the ``pk=`` argument is no longer required when upserting into a table that already has a primary key. - ``db.table(table_name).insert({})`` can now be used to insert a row consisting entirely of default values into an existing table, using ``INSERT INTO ... DEFAULT VALUES``. (:issue:`759`) - Improvements to the ``sqlite-utils migrate`` command: ``--stop-before`` values that do not match any known migration are now an error instead of being silently ignored, ``--stop-before`` now works correctly with migration files that still use the older ``sqlite_migrate.Migrations`` class, and ``--list`` is now a read-only operation that no longer creates the database file or the migrations tracking table. ``migrations.applied()`` now returns migrations in the order they were applied. +- New ``db.begin()``, ``db.commit()`` and ``db.rollback()`` methods for taking manual control of transactions, as an alternative to the ``db.atomic()`` context manager. - New documentation: :ref:`python_api_transactions` describes how transactions work and when changes are committed, and a new :ref:`upgrading` page details the changes needed to move between major versions. .. _v4_0rc1: diff --git a/docs/migrations.rst b/docs/migrations.rst index 1bd1457..e685936 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -87,7 +87,7 @@ Some operations cannot run inside a transaction, for example ``VACUUM`` or chang A migration registered with ``transactional=False`` runs without a wrapping transaction, so if it fails part way through any changes it already made will not be rolled back, and re-applying will run the whole function again. -Avoid calling ``db.conn.commit()`` or otherwise managing transactions manually inside a transactional migration - register the migration with ``transactional=False`` if it needs to control its own transactions. Using ``with db.atomic():`` blocks inside a migration is fine: they nest as savepoints within the migration's transaction, so the migration as a whole still commits or rolls back as a single unit. See :ref:`python_api_transactions`. +Avoid calling ``db.commit()`` or otherwise managing transactions manually inside a transactional migration - register the migration with ``transactional=False`` if it needs to control its own transactions. Using ``with db.atomic():`` blocks inside a migration is fine: they nest as savepoints within the migration's transaction, so the migration as a whole still commits or rolls back as a single unit. See :ref:`python_api_transactions`. Applying migrations using the CLI ================================= diff --git a/docs/python-api.rst b/docs/python-api.rst index 163dad5..400c0fa 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -156,7 +156,7 @@ only be left open here if you opened it yourself - through raw ``db.execute()`` writes or an explicit ``BEGIN``. In that case the decision to commit stays with you: committing automatically on exit could silently persist half-finished work, for example if your code returned early from the block. -Commit explicitly with ``db.conn.commit()``, or wrap your writes in +Commit explicitly with ``db.commit()``, or wrap your writes in ``db.atomic()`` which commits for you. Note this differs from the ``sqlite3.Connection`` context manager in the @@ -350,12 +350,12 @@ The transaction is opened with a deferred ``BEGIN`` - SQLite takes the necessary Raw SQL writes with db.execute() -------------------------------- -:ref:`db.execute() ` is a thin wrapper around the underlying ``sqlite3`` connection, and it follows that connection's rules rather than this library's: a write statement opens a transaction that stays open until something commits it. Commit explicitly: +:ref:`db.execute() ` is a thin wrapper around the underlying ``sqlite3`` connection, and it follows that connection's rules rather than this library's: a write statement opens a transaction that stays open until something commits it. Commit explicitly with ``db.commit()``: .. code-block:: python db.execute("insert into news (headline) values (?)", ["Dog wins award"]) - db.conn.commit() + db.commit() Or wrap the calls in ``db.atomic()``, which commits for you: @@ -371,7 +371,18 @@ If you do neither, the write can be deceptive: reads on the same connection will Managing transactions yourself ------------------------------ -You can take full manual control using ``db.execute("begin")`` (or any raw write, as above) followed by ``db.conn.commit()`` or ``db.conn.rollback()``. +You can take full manual control using the ``db.begin()``, ``db.commit()`` and ``db.rollback()`` methods: + +.. code-block:: python + + db.begin() + db.table("news").insert({"headline": "Dog wins award"}) + if all_looks_good: + db.commit() + else: + db.rollback() + +``db.begin()`` raises ``sqlite3.OperationalError`` if a transaction is already open. ``db.commit()`` and ``db.rollback()`` do nothing if there is no open transaction. A raw write executed with ``db.execute()`` (as above) opens a transaction implicitly, without needing ``db.begin()``. The library will never commit a transaction you opened. If you call write methods such as ``insert()`` - or use ``db.atomic()`` - while your transaction is open, they participate in it using SQLite savepoints instead of committing: exiting an ``atomic()`` block releases its savepoint, but nothing is saved to disk until you commit the outer transaction yourself. If you roll back, their changes are rolled back too. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 49d9f63..aeafe56 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -455,6 +455,31 @@ class Database: self.conn.rollback() raise + def begin(self) -> None: + """ + Start a transaction with ``BEGIN``, taking manual control of transaction + handling. End it by calling :meth:`commit` or :meth:`rollback`. + + Raises ``sqlite3.OperationalError`` if a transaction is already open. + + Most code should use the :meth:`atomic` context manager instead, which + commits and rolls back automatically. See :ref:`python_api_transactions`. + """ + self.execute("BEGIN") + + def commit(self) -> None: + """ + Commit the current transaction. Does nothing if no transaction is open. + """ + self.conn.commit() + + def rollback(self) -> None: + """ + Roll back the current transaction, discarding its changes. Does nothing + if no transaction is open. + """ + self.conn.rollback() + @contextlib.contextmanager def ensure_autocommit_off(self) -> Generator[None, None, None]: """ diff --git a/tests/test_atomic.py b/tests/test_atomic.py index 8c4d18d..b75bc6c 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -1,6 +1,6 @@ import pytest -from sqlite_utils.db import _iter_complete_sql_statements +from sqlite_utils.db import Database, _iter_complete_sql_statements from sqlite_utils.utils import sqlite3 @@ -189,3 +189,35 @@ def test_atomic_inside_manual_transaction_uses_savepoint(fresh_db): fresh_db["t"].insert({"id": 3}, pk="id") fresh_db.conn.commit() assert [r["id"] for r in fresh_db["t"].rows] == [1, 3] + + +def test_begin_commit_rollback(tmpdir): + path = str(tmpdir / "test.db") + db = Database(path) + db["t"].insert({"id": 1}, pk="id") + db.begin() + db["t"].insert({"id": 2}, pk="id") + assert db.conn.in_transaction + db.rollback() + assert not db.conn.in_transaction + assert [r["id"] for r in db["t"].rows] == [1] + db.begin() + db["t"].insert({"id": 3}, pk="id") + db.commit() + db.close() + db2 = Database(path) + assert [r["id"] for r in db2["t"].rows] == [1, 3] + db2.close() + + +def test_begin_inside_transaction_errors(fresh_db): + fresh_db.begin() + with pytest.raises(sqlite3.OperationalError): + fresh_db.begin() + fresh_db.rollback() + + +def test_commit_and_rollback_without_transaction_are_noops(fresh_db): + fresh_db.commit() + fresh_db.rollback() + assert not fresh_db.conn.in_transaction From 96793668ed89b47f5a299105d37ab55cebeb70f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 22:32:36 +0000 Subject: [PATCH 042/120] Raise TransactionError instead of RuntimeError from enable_wal/disable_wal A dedicated sqlite_utils.db.TransactionError exception is clearer to catch than a generic RuntimeError, and gives future transaction-state guards a home. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 2 +- docs/python-api.rst | 4 ++-- docs/upgrading.rst | 2 +- sqlite_utils/db.py | 10 +++++++--- tests/test_wal.py | 5 +++-- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index baff4af..f6f97b6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,7 +14,7 @@ Breaking changes: - ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` take effect without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. - Python API validation errors now raise ``ValueError`` instead of ``AssertionError``. Previously invalid arguments - such as ``create_table()`` with no columns, ``transform()`` on a table that does not exist, or passing both ``ignore=True`` and ``replace=True`` - were rejected using bare ``assert`` statements, which are silently skipped when Python runs with the ``-O`` flag. Code that caught ``AssertionError`` for these cases should catch ``ValueError`` instead. - ``table.upsert()`` and ``table.upsert_all()`` now raise ``PrimaryKeyRequired`` if a record is missing a value for any primary key column, or has a value of ``None`` for one. Previously such records - which can never match an existing row - were quietly inserted as brand new rows, or triggered a confusing ``KeyError`` after the insert had already taken place. -- ``db.enable_wal()`` and ``db.disable_wal()`` now raise a ``RuntimeError`` if called while a transaction is open. Previously they would silently commit the open transaction as a side effect of changing the journal mode, breaking the rollback guarantee of ``db.atomic()`` and of user-managed transactions. +- ``db.enable_wal()`` and ``db.disable_wal()`` now raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open. Previously they would silently commit the open transaction as a side effect of changing the journal mode, breaking the rollback guarantee of ``db.atomic()`` and of user-managed transactions. - The ``View`` class no longer has an ``enable_fts()`` method. It existed only to raise ``NotImplementedError``, since full-text search is not supported for views - calling it now raises ``AttributeError`` instead, and the method no longer appears in the API reference. The ``sqlite-utils enable-fts`` command shows a clean error when pointed at a view. - The no-op ``-d/--detect-types`` flag has been removed from the ``insert`` and ``upsert`` commands. Type detection has been the default for CSV/TSV data since 4.0a1, so the flag did nothing - invocations using it should simply drop it. ``--no-detect-types`` remains available to disable detection. diff --git a/docs/python-api.rst b/docs/python-api.rst index 400c0fa..810f846 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -388,7 +388,7 @@ The library will never commit a transaction you opened. If you call write method Two related safeguards to be aware of: -- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``RuntimeError`` if called while a transaction is open, because changing the journal mode would commit it as a side effect. +- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, because changing the journal mode would commit it as a side effect. - Closing the database - explicitly with ``db.close()``, or by exiting a ``with Database(...) as db:`` block - rolls back any transaction that is still open, see :ref:`python_api_close`. Supported connection modes @@ -2798,7 +2798,7 @@ You can disable WAL mode using ``.disable_wal()``: Database("my_database.db").disable_wal() -The journal mode can only be changed outside of a transaction. Calling either method while a transaction is open - inside a ``db.atomic()`` block, for example - raises a ``RuntimeError``, unless the database is already in the requested mode in which case the call is a no-op. +The journal mode can only be changed outside of a transaction. Calling either method while a transaction is open - inside a ``db.atomic()`` block, for example - raises a ``sqlite_utils.db.TransactionError``, unless the database is already in the requested mode in which case the call is a no-op. You can check the current journal mode for a database using the ``journal_mode`` property: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index e6f4d4d..a18a88e 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -77,7 +77,7 @@ Python API changes **Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice: - Multi-step operations such as ``table.transform()`` no longer commit an existing transaction you have open - they use savepoints inside it instead. -- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``RuntimeError`` if called while a transaction is open, instead of silently committing it. +- ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, instead of silently committing it. - Using ``Database`` as a context manager (``with Database(path) as db:``) closes the connection on exit *without* committing - uncommitted changes from raw ``db.execute()`` calls are rolled back. Packaging changes diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index aeafe56..d53e758 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -301,6 +301,10 @@ class InvalidColumns(Exception): "Specified columns do not exist" +class TransactionError(Exception): + "Operation cannot be performed while a transaction is open" + + class DescIndex(str): pass @@ -891,7 +895,7 @@ class Database: """ Sets ``journal_mode`` to ``'wal'`` to enable Write-Ahead Log mode. - :raises RuntimeError: if called while a transaction is open - the + :raises TransactionError: if called while a transaction is open - the journal mode can only be changed outside of a transaction """ if self.journal_mode != "wal": @@ -903,7 +907,7 @@ class Database: """ Sets ``journal_mode`` back to ``'delete'`` to disable Write-Ahead Log mode. - :raises RuntimeError: if called while a transaction is open - the + :raises TransactionError: if called while a transaction is open - the journal mode can only be changed outside of a transaction """ if self.journal_mode != "delete": @@ -916,7 +920,7 @@ class Database: # any open transaction as a side effect - breaking the rollback # guarantee of atomic() and of user-managed transactions if self.conn.in_transaction: - raise RuntimeError( + raise TransactionError( "{} cannot be used while a transaction is open".format(operation) ) diff --git a/tests/test_wal.py b/tests/test_wal.py index fa3c855..ee7ecf0 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -1,5 +1,6 @@ import pytest from sqlite_utils import Database +from sqlite_utils.db import TransactionError @pytest.fixture @@ -26,7 +27,7 @@ def test_enable_disable_wal(db_path_tmpdir): def test_enable_wal_inside_transaction_raises(db_path_tmpdir): db, path, tmpdir = db_path_tmpdir db["test"].insert({"id": 1}, pk="id") - with pytest.raises(RuntimeError): + with pytest.raises(TransactionError): with db.atomic(): db["test"].insert({"id": 2}, pk="id") db.enable_wal() @@ -40,7 +41,7 @@ def test_disable_wal_inside_transaction_raises(db_path_tmpdir): db, path, tmpdir = db_path_tmpdir db.enable_wal() db["test"].insert({"id": 1}, pk="id") - with pytest.raises(RuntimeError): + with pytest.raises(TransactionError): with db.atomic(): db["test"].insert({"id": 2}, pk="id") db.disable_wal() From bcd9a26560ff9fad9993a3b5bf90c5f238982903 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 22:38:31 +0000 Subject: [PATCH 043/120] Reject Python 3.12+ autocommit connections in the Database constructor Connections created with sqlite3.connect(autocommit=True) make commit() a documented no-op, and autocommit=False connections are permanently inside a transaction - in both modes every write made by this library appeared to work in-process but was silently discarded when the connection closed. Database() now raises TransactionError for these connections instead of losing data. Tests are skipped on Python versions before 3.12, where the autocommit parameter does not exist. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 1 + docs/python-api.rst | 8 +++++++- docs/upgrading.rst | 1 + sqlite_utils/db.py | 12 ++++++++++++ tests/test_constructor.py | 30 ++++++++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f6f97b6..13bcc62 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,7 @@ Breaking changes: - ``db.enable_wal()`` and ``db.disable_wal()`` now raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open. Previously they would silently commit the open transaction as a side effect of changing the journal mode, breaking the rollback guarantee of ``db.atomic()`` and of user-managed transactions. - The ``View`` class no longer has an ``enable_fts()`` method. It existed only to raise ``NotImplementedError``, since full-text search is not supported for views - calling it now raises ``AttributeError`` instead, and the method no longer appears in the API reference. The ``sqlite-utils enable-fts`` command shows a clean error when pointed at a view. - The no-op ``-d/--detect-types`` flag has been removed from the ``insert`` and ``upsert`` commands. Type detection has been the default for CSV/TSV data since 4.0a1, so the flag did nothing - invocations using it should simply drop it. ``--no-detect-types`` remains available to disable detection. +- ``Database()`` now raises a ``sqlite_utils.db.TransactionError`` if passed a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options. ``commit()`` and ``rollback()`` behave differently on those connections, which previously caused every write made by the library to be silently discarded when the connection closed. Everything else: diff --git a/docs/python-api.rst b/docs/python-api.rst index 810f846..6041460 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -95,6 +95,8 @@ Instead of a file path you can pass in an existing SQLite connection: db = Database(sqlite3.connect("my_database.db")) +The connection must use Python's default transaction handling. Connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options are rejected with a ``sqlite_utils.db.TransactionError`` - see :ref:`python_api_transactions_modes`. + If you want to create an in-memory database, you can do so like this: .. code-block:: python @@ -391,10 +393,14 @@ Two related safeguards to be aware of: - ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, because changing the journal mode would commit it as a side effect. - Closing the database - explicitly with ``db.close()``, or by exiting a ``with Database(...) as db:`` block - rolls back any transaction that is still open, see :ref:`python_api_close`. +.. _python_api_transactions_modes: + Supported connection modes -------------------------- -``db.atomic()`` and the automatic per-method transactions are designed for connections in Python's default transaction handling mode. Connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options are not supported, because ``commit()`` and ``rollback()`` behave differently on those connections. +``db.atomic()`` and the automatic per-method transactions require a connection in Python's default transaction handling mode. Passing a connection created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options to ``Database()`` raises a ``sqlite_utils.db.TransactionError``. + +This is because ``commit()`` and ``rollback()`` behave differently on those connections - under ``autocommit=True`` they are documented no-ops - which would cause every write made by this library to be silently discarded when the connection closed, rather than failing loudly. .. _python_api_table: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index a18a88e..c9380f0 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -79,6 +79,7 @@ Python API changes - Multi-step operations such as ``table.transform()`` no longer commit an existing transaction you have open - they use savepoints inside it instead. - ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, instead of silently committing it. - Using ``Database`` as a context manager (``with Database(path) as db:``) closes the connection on exit *without* committing - uncommitted changes from raw ``db.execute()`` calls are rolled back. +- ``Database()`` rejects connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options, raising ``sqlite_utils.db.TransactionError``. On those connections every write the library made was silently discarded when the connection closed. Packaging changes ----------------- diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d53e758..1395433 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -402,6 +402,18 @@ class Database: if recreate: raise ValueError("recreate cannot be used with connections, only paths") self.conn = cast(sqlite3.Connection, filename_or_conn) + # Python 3.12+ autocommit=True/False connections make commit() + # and rollback() behave differently, silently breaking the + # transaction handling used by every write method + autocommit = getattr(self.conn, "autocommit", None) + if autocommit is not None and autocommit != getattr( + sqlite3, "LEGACY_TRANSACTION_CONTROL", -1 + ): + raise TransactionError( + "sqlite-utils requires a connection that uses the default " + "transaction handling - connections created with " + "autocommit=True or autocommit=False are not supported" + ) self._tracer: Optional[Tracer] = tracer if recursive_triggers: self.execute("PRAGMA recursive_triggers=on;") diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 0798996..c7d0fc0 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -1,6 +1,8 @@ from sqlite_utils import Database +from sqlite_utils.db import TransactionError from sqlite_utils.utils import sqlite3 import pytest +import sys def test_recursive_triggers(): @@ -54,3 +56,31 @@ def test_database_close(tmpdir, memory): db.close() with pytest.raises(sqlite3.ProgrammingError): db.execute("select 1 + 1") + + +@pytest.mark.skipif( + sys.version_info < (3, 12), + reason="sqlite3.connect(autocommit=) requires Python 3.12", +) +@pytest.mark.parametrize("autocommit", [True, False]) +def test_autocommit_connections_are_rejected(tmpdir, autocommit): + # These connection modes break commit()/rollback() in ways that + # silently lose data, so the constructor refuses them + conn = sqlite3.connect(str(tmpdir / "test.db"), autocommit=autocommit) + with pytest.raises(TransactionError): + Database(conn) + conn.close() + + +@pytest.mark.skipif( + sys.version_info < (3, 12), + reason="sqlite3.LEGACY_TRANSACTION_CONTROL requires Python 3.12", +) +def test_legacy_transaction_control_connection_is_accepted(tmpdir): + conn = sqlite3.connect( + str(tmpdir / "test.db"), autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL + ) + db = Database(conn) + db["t"].insert({"id": 1}, pk="id") + assert [r["id"] for r in db["t"].rows] == [1] + db.close() From f7ff3e2027aefb9905ebb2e611e5bbb0a62382c5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 23:15:01 +0000 Subject: [PATCH 044/120] db.execute() write statements now commit automatically Raw write statements previously opened an implicit transaction that stayed open until something committed it - the write was visible on the same connection, making it look saved, but was silently rolled back when the connection closed. execute() now commits any implicit transaction it opens, so raw writes behave like every other write in the library: committed as soon as they run, unless an explicit transaction (db.begin() or db.atomic()) is open, in which case they join it. Row-returning writes such as INSERT ... RETURNING via db.query() commit once their rows have been iterated. atomic(), commit() and rollback() now issue literal COMMIT/ROLLBACK statements, which have identical semantics in every sqlite3 connection mode. The CLI bulk command uses db.atomic() instead of 'with db.conn:'. This simplifies the transaction contract to: everything commits immediately unless you explicitly opened a transaction. Docs updated throughout; changelog and upgrading guide cover the breaking change for code that relied on rolling back uncommitted execute() writes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/changelog.rst | 1 + docs/python-api.rst | 38 +++++++++++++++--------------- docs/upgrading.rst | 3 ++- sqlite_utils/cli.py | 2 +- sqlite_utils/db.py | 49 +++++++++++++++++++++++++++++++++------ tests/test_atomic.py | 45 +++++++++++++++++++++++++++++++++-- tests/test_constructor.py | 9 ++++--- 7 files changed, 114 insertions(+), 33 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 13bcc62..fbc58d8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased Breaking changes: +- Write statements executed with ``db.execute()`` are now committed automatically, unless a transaction is already open in which case they join it. Previously they opened an implicit transaction that stayed open until something committed it - writes appeared to work when read on the same connection but were silently rolled back when the connection closed. Code that relied on rolling back uncommitted ``db.execute()`` writes should use the new ``db.begin()`` method to open an explicit transaction first. The transaction model is documented in full at :ref:`python_api_transactions`. - ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` take effect without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. - Python API validation errors now raise ``ValueError`` instead of ``AssertionError``. Previously invalid arguments - such as ``create_table()`` with no columns, ``transform()`` on a table that does not exist, or passing both ``ignore=True`` and ``replace=True`` - were rejected using bare ``assert`` statements, which are silently skipped when Python runs with the ``-O`` flag. Code that caught ``AssertionError`` for these cases should catch ``ValueError`` instead. - ``table.upsert()`` and ``table.upsert_all()`` now raise ``PrimaryKeyRequired`` if a record is missing a value for any primary key column, or has a value of ``None`` for one. Previously such records - which can never match an existing row - were quietly inserted as brand new rows, or triggered a confusing ``KeyError`` after the insert had already taken place. diff --git a/docs/python-api.rst b/docs/python-api.rst index 6041460..4b6b43e 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -152,14 +152,13 @@ Exiting the block is equivalent to calling ``db.close()``: the connection is closed and any transaction still open at that point is rolled back. This matches SQLite's own behavior when a connection closes. -This rarely matters in practice. Every method in this library that writes to -the database commits its own changes before returning, so a transaction can -only be left open here if you opened it yourself - through raw ``db.execute()`` -writes or an explicit ``BEGIN``. In that case the decision to commit stays -with you: committing automatically on exit could silently persist -half-finished work, for example if your code returned early from the block. -Commit explicitly with ``db.commit()``, or wrap your writes in -``db.atomic()`` which commits for you. +This rarely matters in practice. Everything that writes to the database - +including raw ``db.execute()`` statements - commits automatically, so a +transaction can only be open here if you explicitly started one with +``db.begin()`` and have not yet committed it. In that case the decision to +commit stays with you: committing automatically on exit could silently +persist half-finished work, for example if your code returned early from the +block. Call ``db.commit()`` when the work is complete. Note this differs from the ``sqlite3.Connection`` context manager in the standard library, which commits on success but does not close the connection. @@ -264,8 +263,8 @@ The ``db.execute()`` and ``db.executescript()`` methods provide wrappers around Other cursor methods such as ``.fetchone()`` and ``.fetchall()`` are also available, see the `standard library documentation `__. -.. warning:: - Unlike the table methods described elsewhere in this documentation, ``db.execute()`` does **not** commit. If you use it to modify the database you are responsible for committing the change yourself - see :ref:`python_api_transactions_execute`. +.. note:: + Write statements executed this way are committed automatically, unless a transaction is already open in which case they become part of it - see :ref:`python_api_transactions_execute`. .. _python_api_parameters: @@ -308,13 +307,12 @@ Every method in this library that writes to the database - ``insert()``, ``upser db.table("news").insert({"headline": "Dog wins award"}) # The new row is already saved - no commit() required -You never need to call ``commit()`` after using these methods, and you do not need to close the database to persist your changes. +The same applies to raw SQL executed with :ref:`db.execute() ` - a write statement is committed as soon as it has run. -There are exactly three situations where you need to think about transactions: +You never need to call ``commit()``, and you do not need to close the database to persist your changes. There are exactly two situations where you need to think about transactions: 1. You want to group several write operations together, so they either all succeed or all fail - use :ref:`db.atomic() `. -2. You are executing raw SQL writes with ``db.execute()``, which does *not* commit automatically - see :ref:`python_api_transactions_execute`. -3. You are :ref:`managing a transaction yourself `, in which case the library will never commit it for you. +2. You are :ref:`managing a transaction yourself ` with ``db.begin()``, in which case nothing is committed until you commit - the library will never commit a transaction you opened. .. _python_api_atomic: @@ -352,21 +350,23 @@ The transaction is opened with a deferred ``BEGIN`` - SQLite takes the necessary Raw SQL writes with db.execute() -------------------------------- -:ref:`db.execute() ` is a thin wrapper around the underlying ``sqlite3`` connection, and it follows that connection's rules rather than this library's: a write statement opens a transaction that stays open until something commits it. Commit explicitly with ``db.commit()``: +Write statements executed with :ref:`db.execute() ` follow the same rule as everything else: they are committed automatically as soon as they have run. .. code-block:: python db.execute("insert into news (headline) values (?)", ["Dog wins award"]) - db.commit() + # Already committed -Or wrap the calls in ``db.atomic()``, which commits for you: +If a transaction is open - because the call happens inside a ``db.atomic()`` block, or after ``db.begin()`` - the statement becomes part of that transaction instead, and commits when the transaction commits: .. code-block:: python with db.atomic(): db.execute("insert into news (headline) values (?)", ["Dog wins award"]) + db.execute("insert into news (headline) values (?)", ["Cat unimpressed"]) + # Both rows committed together -If you do neither, the write can be deceptive: reads on the same connection will see the new row, making it look saved, but the open transaction will be rolled back when the connection closes and the row will be gone. +One corner case: a row-returning write such as ``INSERT ... RETURNING`` executed through ``db.execute()`` cannot be auto-committed, because its rows have not been read yet - call ``db.commit()`` after fetching them, or use :ref:`db.query() ` for those statements, which commits automatically once you have iterated over the results. .. _python_api_transactions_manual: @@ -384,7 +384,7 @@ You can take full manual control using the ``db.begin()``, ``db.commit()`` and ` else: db.rollback() -``db.begin()`` raises ``sqlite3.OperationalError`` if a transaction is already open. ``db.commit()`` and ``db.rollback()`` do nothing if there is no open transaction. A raw write executed with ``db.execute()`` (as above) opens a transaction implicitly, without needing ``db.begin()``. +``db.begin()`` raises ``sqlite3.OperationalError`` if a transaction is already open. ``db.commit()`` and ``db.rollback()`` do nothing if there is no open transaction. The library will never commit a transaction you opened. If you call write methods such as ``insert()`` - or use ``db.atomic()`` - while your transaction is open, they participate in it using SQLite savepoints instead of committing: exiting an ``atomic()`` block releases its savepoint, but nothing is saved to disk until you commit the outer transaction yourself. If you roll back, their changes are rolled back too. diff --git a/docs/upgrading.rst b/docs/upgrading.rst index c9380f0..3109a7c 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -76,9 +76,10 @@ Python API changes **Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice: +- Write statements executed with raw ``db.execute()`` calls now commit automatically, unless a transaction is already open in which case they join it. Previously they opened an implicit transaction that nothing committed - if your code used ``db.execute()`` for writes and relied on ``db.conn.rollback()`` to undo them, open an explicit transaction with the new ``db.begin()`` method first. - Multi-step operations such as ``table.transform()`` no longer commit an existing transaction you have open - they use savepoints inside it instead. - ``db.enable_wal()`` and ``db.disable_wal()`` raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open, instead of silently committing it. -- Using ``Database`` as a context manager (``with Database(path) as db:``) closes the connection on exit *without* committing - uncommitted changes from raw ``db.execute()`` calls are rolled back. +- Using ``Database`` as a context manager (``with Database(path) as db:``) closes the connection on exit *without* committing - a transaction you explicitly opened with ``db.begin()`` and did not commit is rolled back. - ``Database()`` rejects connections created with the Python 3.12+ ``sqlite3.connect(..., autocommit=True)`` or ``autocommit=False`` options, raising ``sqlite_utils.db.TransactionError``. On those connections every write the library made was silently discarded when the connection closed. Packaging changes diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index d609ad5..a30f79a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1140,7 +1140,7 @@ def insert_upsert_implementation( else: doc_chunks = [docs] for doc_chunk in doc_chunks: - with db.conn: + with db.atomic(): db.conn.cursor().executemany(bulk_sql, doc_chunk) return diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1395433..ef367e1 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -324,6 +324,16 @@ CREATE TABLE IF NOT EXISTS "{}"( """.strip() +_TRANSACTION_CONTROL_PREFIXES = ( + "BEGIN", + "COMMIT", + "END", + "ROLLBACK", + "SAVEPOINT", + "RELEASE", +) + + class Database: """ Wrapper for a SQLite database connection that adds a variety of useful utility methods. @@ -462,13 +472,13 @@ class Database: try: yield self except BaseException: - self.conn.rollback() + self.conn.execute("ROLLBACK") raise else: try: - self.conn.commit() + self.conn.execute("COMMIT") except BaseException: - self.conn.rollback() + self.conn.execute("ROLLBACK") raise def begin(self) -> None: @@ -487,14 +497,16 @@ class Database: """ Commit the current transaction. Does nothing if no transaction is open. """ - self.conn.commit() + if self.conn.in_transaction: + self.conn.execute("COMMIT") def rollback(self) -> None: """ Roll back the current transaction, discarding its changes. Does nothing if no transaction is open. """ - self.conn.rollback() + if self.conn.in_transaction: + self.conn.execute("ROLLBACK") @contextlib.contextmanager def ensure_autocommit_off(self) -> Generator[None, None, None]: @@ -648,6 +660,7 @@ class Database: :raises ValueError: if the SQL statement does not return rows - use :meth:`execute` for those statements instead """ + was_in_transaction = self.conn.in_transaction cursor = self.execute(sql, params or tuple()) if cursor.description is None: raise ValueError( @@ -659,6 +672,11 @@ class Database: def rows() -> Generator[dict, None, None]: for row in cursor: yield dict(zip(keys, row)) + if not was_in_transaction and self.conn.in_transaction: + # A row-returning write such as INSERT ... RETURNING opened + # an implicit transaction - commit it now that its rows have + # been consumed + self.conn.execute("COMMIT") return rows() @@ -668,16 +686,33 @@ class Database: """ Execute SQL query and return a ``sqlite3.Cursor``. + A write statement - ``INSERT``, ``UPDATE``, ``CREATE TABLE`` and so on - + is committed automatically, unless a transaction is already open, in + which case it becomes part of that transaction. See + :ref:`python_api_transactions`. + :param sql: SQL query to execute :param parameters: Parameters to use in that query - an iterable for ``where id = ?`` parameters, or a dictionary for ``where id = :id`` """ if self._tracer: self._tracer(sql, parameters) + was_in_transaction = self.conn.in_transaction if parameters is not None: - return self.conn.execute(sql, parameters) + cursor = self.conn.execute(sql, parameters) else: - return self.conn.execute(sql) + cursor = self.conn.execute(sql) + if ( + not was_in_transaction + and self.conn.in_transaction + and cursor.description is None + and not sql.lstrip().upper().startswith(_TRANSACTION_CONTROL_PREFIXES) + ): + # The statement opened an implicit transaction - commit it, so + # that execute() behaves consistently with the rest of the + # library and identically across connection modes + self.conn.execute("COMMIT") + return cursor def executescript(self, sql: str) -> sqlite3.Cursor: """ diff --git a/tests/test_atomic.py b/tests/test_atomic.py index b75bc6c..a8e3563 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -181,13 +181,13 @@ def test_atomic_inside_manual_transaction_uses_savepoint(fresh_db): fresh_db["t"].insert({"id": 2}, pk="id") # Nothing is committed until the user's own transaction commits assert fresh_db.conn.in_transaction - fresh_db.conn.rollback() + fresh_db.rollback() assert [r["id"] for r in fresh_db["t"].rows] == [1] # And with a commit instead, the atomic block's writes persist fresh_db.execute("begin") with fresh_db.atomic(): fresh_db["t"].insert({"id": 3}, pk="id") - fresh_db.conn.commit() + fresh_db.commit() assert [r["id"] for r in fresh_db["t"].rows] == [1, 3] @@ -221,3 +221,44 @@ def test_commit_and_rollback_without_transaction_are_noops(fresh_db): fresh_db.commit() fresh_db.rollback() assert not fresh_db.conn.in_transaction + + +def test_execute_write_commits_immediately(tmpdir): + path = str(tmpdir / "test.db") + db = Database(path) + db["t"].insert({"id": 1}, pk="id") + db.execute("insert into t (id) values (2)") + # No implicit transaction is left open + assert not db.conn.in_transaction + # A completely separate connection sees the row straight away + other = sqlite3.connect(path) + assert other.execute("select count(*) from t").fetchone()[0] == 2 + other.close() + db.close() + + +def test_execute_write_respects_explicit_transaction(fresh_db): + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.begin() + fresh_db.execute("insert into t (id) values (2)") + # Still inside the explicit transaction - not committed + assert fresh_db.conn.in_transaction + fresh_db.rollback() + assert [r["id"] for r in fresh_db["t"].rows] == [1] + + +def test_query_returning_commits_after_iteration(tmpdir): + if sqlite3.sqlite_version_info < (3, 35, 0): + import pytest as _pytest + + _pytest.skip("RETURNING requires SQLite 3.35.0 or higher") + path = str(tmpdir / "test.db") + db = Database(path) + db["t"].insert({"id": 1}, pk="id") + rows = list(db.query("insert into t (id) values (2) returning id")) + assert rows == [{"id": 2}] + assert not db.conn.in_transaction + other = sqlite3.connect(path) + assert other.execute("select count(*) from t").fetchone()[0] == 2 + other.close() + db.close() diff --git a/tests/test_constructor.py b/tests/test_constructor.py index c7d0fc0..7714f26 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -35,14 +35,17 @@ def test_database_context_manager(tmpdir): path = str(tmpdir / "test.db") with Database(path) as db: db["t"].insert({"id": 1}) - # A raw write left uncommitted on purpose: + # Raw writes commit automatically too db.execute("insert into t (id) values (2)") + # An explicitly opened transaction left uncommitted on purpose: + db.begin() + db.execute("insert into t (id) values (3)") # The connection is closed... with pytest.raises(sqlite3.ProgrammingError): db.execute("select 1") - # ... and the uncommitted change was rolled back, not committed + # ... and the open explicit transaction was rolled back, not committed db2 = Database(path) - assert [r["id"] for r in db2["t"].rows] == [1] + assert [r["id"] for r in db2["t"].rows] == [1, 2] db2.close() From 2510745de42d67636bd9e9e54d3c1b513087a6f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 23:19:51 +0000 Subject: [PATCH 045/120] Add --sqlite-autocommit pytest option Runs the entire test suite against connections created with the Python 3.12+ sqlite3.connect(autocommit=True) mode, by patching connect() to default autocommit=True for internally-created connections. Errors on older Python versions. The full suite passes in both modes with no test changes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- tests/conftest.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 6fd35b8..728db7b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,11 +8,36 @@ create table Gosh2 (c1 text, c2 text, c3 text); """ +def pytest_addoption(parser): + parser.addoption( + "--sqlite-autocommit", + action="store_true", + default=False, + help=( + "Run every test against connections created with the Python 3.12+ " + "sqlite3.connect(autocommit=True) mode" + ), + ) + + def pytest_configure(config): import sys sys._called_from_test = True # type: ignore[attr-defined] + if config.getoption("--sqlite-autocommit"): + if sys.version_info < (3, 12): + raise pytest.UsageError( + "--sqlite-autocommit requires Python 3.12 or higher" + ) + real_connect = sqlite3.connect + + def autocommit_connect(*args, **kwargs): + kwargs.setdefault("autocommit", True) + return real_connect(*args, **kwargs) + + sqlite3.connect = autocommit_connect + @pytest.fixture(autouse=True) def close_all_databases(): From b49af65a6d2100fa484dec71ff35495aa820e967 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 23:20:31 +0000 Subject: [PATCH 046/120] Fix PytestRemovedIn10Warning in test_sniff parametrize received a generator from Path.glob(), which pytest 10 will reject. sorted() materializes the list and makes the parameter order deterministic as a bonus. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- tests/test_sniff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sniff.py b/tests/test_sniff.py index 62fac86..4bbdb66 100644 --- a/tests/test_sniff.py +++ b/tests/test_sniff.py @@ -6,7 +6,7 @@ import pytest sniff_dir = pathlib.Path(__file__).parent / "sniff" -@pytest.mark.parametrize("filepath", sniff_dir.glob("example*")) +@pytest.mark.parametrize("filepath", sorted(sniff_dir.glob("example*"))) def test_sniff(tmpdir, filepath): db_path = str(tmpdir / "test.db") runner = CliRunner() From 89af511f799ee235d1d35e00fc0557ef98794792 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 4 Jul 2026 16:24:07 -0700 Subject: [PATCH 047/120] Also run "pytest --sqlite-autocommit" for one matrix item in CI To test that our code works correctly with the new autocommit option introduced in Python 3.13+ https://github.com/simonw/sqlite-utils/pull/767#discussion_r3523983636 --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0f65d6a..7e1e953 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,6 +41,9 @@ jobs: - name: Run tests run: | pytest -v + - name: Run autocommit tests just on 3.14/Ubuntu + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14' + run: pytest --sqlite-autocommit - name: run mypy run: mypy sqlite_utils tests - name: run flake8 From 7a015cff52f0f740bdf1bb8a83ad5e8183db3e20 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Jul 2026 23:33:38 +0000 Subject: [PATCH 048/120] Unwrap hard-wrapped prose in the documentation The documentation style for this project is one line per paragraph, not text-wrapped RST. Unwraps three recently added paragraphs in python-api.rst and two older ones in plugins.rst. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UnLnhsH25Nnv7LHhekUfPd --- docs/plugins.rst | 7 ++----- docs/python-api.rst | 16 +++------------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 6357ad9..99588ee 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -140,9 +140,7 @@ As a special niche feature, if your plugin needs to import some files and then a prepare_connection(conn) ~~~~~~~~~~~~~~~~~~~~~~~~ -This hook is called when a new SQLite database connection is created. You can -use it to `register custom SQL functions `_, -aggregates and collations. For example: +This hook is called when a new SQLite database connection is created. You can use it to `register custom SQL functions `_, aggregates and collations. For example: .. code-block:: python @@ -154,8 +152,7 @@ aggregates and collations. For example: "hello", 1, lambda name: f"Hello, {name}!" ) -This registers a SQL function called ``hello`` which takes a single -argument and can be called like this: +This registers a SQL function called ``hello`` which takes a single argument and can be called like this: .. code-block:: sql diff --git a/docs/python-api.rst b/docs/python-api.rst index 4b6b43e..e9eb371 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -148,21 +148,11 @@ The ``Database`` object also works as a context manager, which will automaticall db["my_table"].insert({"name": "Example"}) # Connection is automatically closed here -Exiting the block is equivalent to calling ``db.close()``: the connection is -closed and any transaction still open at that point is rolled back. This -matches SQLite's own behavior when a connection closes. +Exiting the block is equivalent to calling ``db.close()``: the connection is closed and any transaction still open at that point is rolled back. This matches SQLite's own behavior when a connection closes. -This rarely matters in practice. Everything that writes to the database - -including raw ``db.execute()`` statements - commits automatically, so a -transaction can only be open here if you explicitly started one with -``db.begin()`` and have not yet committed it. In that case the decision to -commit stays with you: committing automatically on exit could silently -persist half-finished work, for example if your code returned early from the -block. Call ``db.commit()`` when the work is complete. +This rarely matters in practice. Everything that writes to the database - including raw ``db.execute()`` statements - commits automatically, so a transaction can only be open here if you explicitly started one with ``db.begin()`` and have not yet committed it. In that case the decision to commit stays with you: committing automatically on exit could silently persist half-finished work, for example if your code returned early from the block. Call ``db.commit()`` when the work is complete. -Note this differs from the ``sqlite3.Connection`` context manager in the -standard library, which commits on success but does not close the connection. -See :ref:`python_api_transactions` for the full transaction model. +Note this differs from the ``sqlite3.Connection`` context manager in the standard library, which commits on success but does not close the connection. See :ref:`python_api_transactions` for the full transaction model. .. _python_api_attach: From 0566a9f128e19475f5cb423043820fb4119051cc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 4 Jul 2026 17:40:48 -0700 Subject: [PATCH 049/120] Improve db.query() error handling and transaction safety db.query() rejected statements roll back, RETURNING commits immediately. Fixes two transaction bugs in db.query(): - A non-row-returning statement such as an UPDATE was executed and auto-committed by execute() before the ValueError was raised, so the write took effect despite being rejected. query() now runs the statement inside a savepoint and rolls it back before raising, so a rejected statement has no effect on the database - in every connection mode, including autocommit=True. - INSERT ... RETURNING only committed once the returned generator was fully exhausted, so calling query() without iterating - or partially iterating with next() - left the transaction open and the write could be rolled back on close. The write is now completed and committed at call time, as the documentation already promised. Plain SELECTs are still fetched lazily. Transaction control statements, VACUUM, ATTACH and DETACH never return rows, so query() now rejects them without executing them. PRAGMA statements skip the savepoint guard because some of them - such as pragma journal_mode=wal - refuse to run inside a transaction. Claude-Session: https://claude.ai/code/session_012U3iRfJoTZ5vd22cBSF2nJ --- docs/changelog.rst | 2 +- docs/python-api.rst | 6 +-- docs/upgrading.rst | 2 +- sqlite_utils/db.py | 77 ++++++++++++++++++++++--------- tests/test_query.py | 110 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 171 insertions(+), 26 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fbc58d8..f110e2e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,7 +12,7 @@ Unreleased Breaking changes: - Write statements executed with ``db.execute()`` are now committed automatically, unless a transaction is already open in which case they join it. Previously they opened an implicit transaction that stayed open until something committed it - writes appeared to work when read on the same connection but were silently rolled back when the connection closed. Code that relied on rolling back uncommitted ``db.execute()`` writes should use the new ``db.begin()`` method to open an explicit transaction first. The transaction model is documented in full at :ref:`python_api_transactions`. -- ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` take effect without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. +- ``db.query()`` now executes its SQL as soon as it is called, rather than waiting until the returned generator is first iterated. Rows are still fetched lazily during iteration. SQL errors are now raised at the call site, statements such as ``INSERT ... RETURNING`` are executed and committed immediately without needing to iterate over their results, and passing a statement that returns no rows - previously a silent no-op - now raises a ``ValueError`` recommending ``db.execute()`` instead. A statement rejected this way is rolled back before the error is raised, so it has no effect on the database. - Python API validation errors now raise ``ValueError`` instead of ``AssertionError``. Previously invalid arguments - such as ``create_table()`` with no columns, ``transform()`` on a table that does not exist, or passing both ``ignore=True`` and ``replace=True`` - were rejected using bare ``assert`` statements, which are silently skipped when Python runs with the ``-O`` flag. Code that caught ``AssertionError`` for these cases should catch ``ValueError`` instead. - ``table.upsert()`` and ``table.upsert_all()`` now raise ``PrimaryKeyRequired`` if a record is missing a value for any primary key column, or has a value of ``None`` for one. Previously such records - which can never match an existing row - were quietly inserted as brand new rows, or triggered a confusing ``KeyError`` after the insert had already taken place. - ``db.enable_wal()`` and ``db.disable_wal()`` now raise a ``sqlite_utils.db.TransactionError`` if called while a transaction is open. Previously they would silently commit the open transaction as a side effect of changing the journal mode, breaking the rollback guarantee of ``db.atomic()`` and of user-managed transactions. diff --git a/docs/python-api.rst b/docs/python-api.rst index e9eb371..3f7e12c 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -229,9 +229,9 @@ The ``db.query(sql)`` function executes a SQL query and returns an iterator over # {'name': 'Cleo'} # {'name': 'Pancakes'} -The SQL query is executed as soon as ``db.query()`` is called. The resulting rows are fetched lazily as you iterate, so large result sets are not loaded into memory all at once. Because execution is immediate, an error in your SQL will raise an exception straight away, and a statement such as ``INSERT ... RETURNING`` will take effect even if you do not iterate over its results. +The SQL query is executed as soon as ``db.query()`` is called. The resulting rows are fetched lazily as you iterate, so large result sets are not loaded into memory all at once. Because execution is immediate, an error in your SQL will raise an exception straight away, and a statement such as ``INSERT ... RETURNING`` will take effect - and be committed, unless a transaction is open - even if you do not iterate over its results. -``db.query()`` can only be used with SQL that returns rows. Passing a statement that returns no rows - an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause, for example - will raise a ``ValueError``. Use :ref:`db.execute() ` for those statements instead. +``db.query()`` can only be used with SQL that returns rows. Passing a statement that returns no rows - an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause, for example - will raise a ``ValueError``. The rejected statement is rolled back, so it has no effect on the database. Use :ref:`db.execute() ` for those statements instead. .. _python_api_execute: @@ -356,7 +356,7 @@ If a transaction is open - because the call happens inside a ``db.atomic()`` blo db.execute("insert into news (headline) values (?)", ["Cat unimpressed"]) # Both rows committed together -One corner case: a row-returning write such as ``INSERT ... RETURNING`` executed through ``db.execute()`` cannot be auto-committed, because its rows have not been read yet - call ``db.commit()`` after fetching them, or use :ref:`db.query() ` for those statements, which commits automatically once you have iterated over the results. +One corner case: a row-returning write such as ``INSERT ... RETURNING`` executed through ``db.execute()`` cannot be auto-committed, because its rows have not been read yet - call ``db.commit()`` after fetching them, or use :ref:`db.query() ` for those statements, which executes the write and commits it immediately. .. _python_api_transactions_manual: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 3109a7c..1974ad6 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -58,7 +58,7 @@ Python API changes **db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily. Two consequences: - Errors in your SQL now raise at the ``db.query()`` call site rather than on first iteration. -- Passing a statement that returns no rows - such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause - previously did nothing at all, silently. It now raises a ``ValueError``. Use ``db.execute()`` for statements that do not return rows. +- Passing a statement that returns no rows - such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause - previously did nothing at all, silently. It now raises a ``ValueError``, and the statement is rolled back so it has no effect on the database. Use ``db.execute()`` for statements that do not return rows. **Upserts use INSERT ... ON CONFLICT.** Upsert operations now use SQLite's ``INSERT ... ON CONFLICT SET`` syntax rather than the previous ``INSERT OR IGNORE`` followed by ``UPDATE``. If your code depends on the old behavior, pass ``use_old_upsert=True`` to the ``Database()`` constructor - see :ref:`python_api_old_upsert`. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ef367e1..8e6a194 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -652,33 +652,68 @@ class Database: Execute ``sql`` and return an iterable of dictionaries representing each row. The SQL is executed as soon as this method is called - the resulting rows - are then fetched lazily as the returned iterable is iterated over. + are then fetched lazily as the returned iterable is iterated over. A + row-returning write such as ``INSERT ... RETURNING`` takes effect + immediately, even if the results are never iterated. :param sql: SQL query to execute :param params: Parameters to use in that query - an iterable for ``where id = ?`` parameters, or a dictionary for ``where id = :id`` :raises ValueError: if the SQL statement does not return rows - use - :meth:`execute` for those statements instead + :meth:`execute` for those statements instead. The rejected statement + is rolled back, so it has no effect on the database """ - was_in_transaction = self.conn.in_transaction - cursor = self.execute(sql, params or tuple()) - if cursor.description is None: - raise ValueError( - "query() can only be used with SQL that returns rows - " - "use execute() for other statements" - ) - keys = [d[0] for d in cursor.description] - - def rows() -> Generator[dict, None, None]: - for row in cursor: - yield dict(zip(keys, row)) - if not was_in_transaction and self.conn.in_transaction: - # A row-returning write such as INSERT ... RETURNING opened - # an implicit transaction - commit it now that its rows have - # been consumed - self.conn.execute("COMMIT") - - return rows() + message = ( + "query() can only be used with SQL that returns rows - " + "use execute() for other statements" + ) + prefix = sql.lstrip().upper() + if prefix.startswith( + _TRANSACTION_CONTROL_PREFIXES + ("VACUUM", "ATTACH", "DETACH") + ): + # None of these return rows - reject them without executing anything + raise ValueError(message) + if self._tracer: + self._tracer(sql, params) + args: tuple = (params,) if params is not None else () + if prefix.startswith("PRAGMA"): + # Some PRAGMA statements refuse to run inside a transaction, so + # execute these without the savepoint guard used below. PRAGMAs + # never open an implicit transaction, so there is nothing to + # undo if this one turns out not to return rows + cursor = self.conn.execute(sql, *args) + if cursor.description is None: + raise ValueError(message) + keys = [d[0] for d in cursor.description] + return (dict(zip(keys, row)) for row in cursor) + # Execute inside a savepoint, so a statement that turns out not to + # return rows can be rolled back before the ValueError is raised + self.conn.execute('SAVEPOINT "sqlite_utils_query"') + released = False + try: + cursor = self.conn.execute(sql, *args) + if cursor.description is None: + raise ValueError(message) + keys = [d[0] for d in cursor.description] + try: + self.conn.execute('RELEASE "sqlite_utils_query"') + released = True + except sqlite3.OperationalError: + # The savepoint cannot be released while a write statement is + # still executing - this is INSERT ... RETURNING or similar, + # with unfetched rows. Fetch them so the write completes, then + # release again - committing the write immediately, unless an + # outer transaction is open + fetched = cursor.fetchall() + self.conn.execute('RELEASE "sqlite_utils_query"') + released = True + return (dict(zip(keys, row)) for row in fetched) + return (dict(zip(keys, row)) for row in cursor) + finally: + if not released: + # An error occurred - undo anything the statement changed + self.conn.execute('ROLLBACK TO "sqlite_utils_query"') + self.conn.execute('RELEASE "sqlite_utils_query"') def execute( self, sql: str, parameters: Optional[Union[Sequence, Dict[str, Any]]] = None diff --git a/tests/test_query.py b/tests/test_query.py index 91bf1ac..cf8d9f5 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -23,6 +23,56 @@ def test_query_rejects_statements_that_return_no_rows(fresh_db): with pytest.raises(ValueError) as ex: fresh_db.query("update dogs set name = 'Cleopaws'") assert "execute()" in str(ex.value) + # The rejected update was rolled back, and no transaction is left open + assert not fresh_db.conn.in_transaction + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + + +def test_query_rejected_ddl_is_rolled_back(fresh_db): + with pytest.raises(ValueError): + fresh_db.query("create table dogs (id integer primary key)") + assert not fresh_db.conn.in_transaction + assert fresh_db.table_names() == [] + + +def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db): + fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.begin() + fresh_db.execute("insert into dogs (name) values ('Pancakes')") + with pytest.raises(ValueError): + fresh_db.query("update dogs set name = 'Cleopaws'") + # The transaction is still open and the earlier insert is intact + assert fresh_db.conn.in_transaction + fresh_db.commit() + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo", "Pancakes"] + + +@pytest.mark.parametrize( + "sql", ["begin", "commit", "rollback", "vacuum", "detach database foo"] +) +def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql): + with pytest.raises(ValueError) as ex: + fresh_db.query(sql) + assert "execute()" in str(ex.value) + assert not fresh_db.conn.in_transaction + + +def test_query_error_leaves_no_transaction_open(fresh_db): + with pytest.raises(sqlite3.OperationalError): + fresh_db.query("select * from missing_table") + assert not fresh_db.conn.in_transaction + + +def test_query_pragma(tmpdir): + from sqlite_utils import Database + + db = Database(str(tmpdir / "test.db")) + # A row-returning PRAGMA works, including one that cannot run in a transaction + assert list(db.query("pragma journal_mode = wal")) == [{"journal_mode": "wal"}] + # A PRAGMA that returns no rows raises ValueError + with pytest.raises(ValueError): + db.query("pragma user_version = 5") + db.close() @pytest.mark.skipif( @@ -38,6 +88,66 @@ def test_query_insert_returning(fresh_db): assert fresh_db["dogs"].count == 2 +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 35, 0), + reason="RETURNING requires SQLite 3.35.0 or higher", +) +def test_query_insert_returning_commits_without_iteration(tmpdir): + from sqlite_utils import Database + + path = str(tmpdir / "test.db") + db = Database(path) + db["dogs"].insert({"name": "Cleo"}) + # Never iterate over the results + db.query("insert into dogs (name) values ('Pancakes') returning name") + assert not db.conn.in_transaction + # A completely separate connection sees the new row straight away + other = sqlite3.connect(path) + assert other.execute("select count(*) from dogs").fetchone()[0] == 2 + other.close() + db.close() + + +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 35, 0), + reason="RETURNING requires SQLite 3.35.0 or higher", +) +def test_query_insert_returning_partial_iteration_still_commits(tmpdir): + from sqlite_utils import Database + + path = str(tmpdir / "test.db") + db = Database(path) + db["dogs"].insert({"name": "Cleo"}) + row = next( + db.query( + "insert into dogs (name) values ('Pancakes'), ('Marnie') returning name" + ) + ) + assert row == {"name": "Pancakes"} + assert not db.conn.in_transaction + other = sqlite3.connect(path) + assert other.execute("select count(*) from dogs").fetchone()[0] == 3 + other.close() + db.close() + + +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 35, 0), + reason="RETURNING requires SQLite 3.35.0 or higher", +) +def test_query_insert_returning_respects_explicit_transaction(fresh_db): + fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.begin() + rows = list( + fresh_db.query("insert into dogs (name) values ('Pancakes') returning name") + ) + assert rows == [{"name": "Pancakes"}] + # Still inside the explicit transaction - not committed + assert fresh_db.conn.in_transaction + fresh_db.rollback() + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + + def test_execute_returning_dicts(fresh_db): # Like db.query() but returns a list, included for backwards compatibility # see https://github.com/simonw/sqlite-utils/issues/290 From a8e3a643f2a7ad1933eb900713f18f1650e45c1b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 4 Jul 2026 17:45:42 -0700 Subject: [PATCH 050/120] Release 4.0rc2 Refs #759, #629, #761, #767, #768 --- docs/changelog.rst | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f110e2e..2e1f69c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,10 +4,10 @@ Changelog =========== -.. _unreleased: +.. _v4_0rc2: -Unreleased ----------- +4.0rc2 (2026-07-04) +------------------- Breaking changes: diff --git a/pyproject.toml b/pyproject.toml index 4ad1bd9..9d1f041 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.0rc1" +version = "4.0rc2" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 87cf1c5a007ff724ed803111898d9658d53e320a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 11:49:59 -0700 Subject: [PATCH 051/120] Replace SQL prefix matching with a first-token scanner query() and execute() classified statements using sql.lstrip().upper().startswith(...), which a leading SQL comment defeated: db.query('/* c */ COMMIT') inside db.atomic() committed the caller's transaction and masked the ValueError with a 'no such savepoint' error, comment-prefixed PRAGMAs ran inside the savepoint guard where journal mode changes are refused, and a comment-prefixed BEGIN passed to db.execute() was instantly auto-committed. The new _first_keyword() helper skips leading whitespace and -- or /* */ comments - the only things SQLite's tokenizer allows before the first token - and returns that token for exact comparison, so keyword detection now matches what SQLite itself will see. Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 51 +++++++++++++++++++++++++++++------ tests/test_atomic.py | 11 ++++++++ tests/test_query.py | 63 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 116 insertions(+), 9 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8e6a194..f796a63 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -324,14 +324,51 @@ CREATE TABLE IF NOT EXISTS "{}"( """.strip() -_TRANSACTION_CONTROL_PREFIXES = ( +_TRANSACTION_CONTROL_KEYWORDS = { "BEGIN", "COMMIT", "END", "ROLLBACK", "SAVEPOINT", "RELEASE", -) +} + +# Statements that never return rows and cannot run inside (or would break +# out of) the savepoint guard used by query() +_QUERY_REJECTED_KEYWORDS = _TRANSACTION_CONTROL_KEYWORDS | { + "VACUUM", + "ATTACH", + "DETACH", +} + + +def _first_keyword(sql: str) -> str: + """ + Return the first keyword of a SQL statement, uppercased, skipping any + leading whitespace and ``--`` or ``/* ... */`` comments - the only + things SQLite's tokenizer allows before the first token. Returns an + empty string if there is no leading keyword. + """ + i, n = 0, len(sql) + while i < n: + if sql[i].isspace(): + i += 1 + elif sql.startswith("--", i): + newline = sql.find("\n", i) + if newline == -1: + return "" + i = newline + 1 + elif sql.startswith("/*", i): + end = sql.find("*/", i + 2) + if end == -1: + return "" + i = end + 2 + else: + break + j = i + while j < n and (sql[j].isalpha() or sql[j] == "_"): + j += 1 + return sql[i:j].upper() class Database: @@ -667,16 +704,14 @@ class Database: "query() can only be used with SQL that returns rows - " "use execute() for other statements" ) - prefix = sql.lstrip().upper() - if prefix.startswith( - _TRANSACTION_CONTROL_PREFIXES + ("VACUUM", "ATTACH", "DETACH") - ): + keyword = _first_keyword(sql) + if keyword in _QUERY_REJECTED_KEYWORDS: # None of these return rows - reject them without executing anything raise ValueError(message) if self._tracer: self._tracer(sql, params) args: tuple = (params,) if params is not None else () - if prefix.startswith("PRAGMA"): + if keyword == "PRAGMA": # Some PRAGMA statements refuse to run inside a transaction, so # execute these without the savepoint guard used below. PRAGMAs # never open an implicit transaction, so there is nothing to @@ -741,7 +776,7 @@ class Database: not was_in_transaction and self.conn.in_transaction and cursor.description is None - and not sql.lstrip().upper().startswith(_TRANSACTION_CONTROL_PREFIXES) + and _first_keyword(sql) not in _TRANSACTION_CONTROL_KEYWORDS ): # The statement opened an implicit transaction - commit it, so # that execute() behaves consistently with the rest of the diff --git a/tests/test_atomic.py b/tests/test_atomic.py index a8e3563..f75c4ee 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -247,6 +247,17 @@ def test_execute_write_respects_explicit_transaction(fresh_db): assert [r["id"] for r in fresh_db["t"].rows] == [1] +def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): + # A BEGIN hidden behind a leading comment must not be auto-committed + # out from under the caller + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.execute("-- start a transaction\nbegin") + assert fresh_db.conn.in_transaction + fresh_db.execute("insert into t (id) values (2)") + fresh_db.rollback() + assert [r["id"] for r in fresh_db["t"].rows] == [1] + + def test_query_returning_commits_after_iteration(tmpdir): if sqlite3.sqlite_version_info < (3, 35, 0): import pytest as _pytest diff --git a/tests/test_query.py b/tests/test_query.py index cf8d9f5..1d5c1f6 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -48,7 +48,18 @@ def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db): @pytest.mark.parametrize( - "sql", ["begin", "commit", "rollback", "vacuum", "detach database foo"] + "sql", + [ + "begin", + "commit", + "rollback", + "vacuum", + "detach database foo", + "/* comment */ commit", + "-- comment\nbegin", + "/* multi\nline */ -- and another\n vacuum", + "\t /* a */ /* b */ savepoint s1", + ], ) def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql): with pytest.raises(ValueError) as ex: @@ -57,6 +68,21 @@ def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql): assert not fresh_db.conn.in_transaction +def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db): + # A COMMIT hidden behind a leading comment must not slip past the + # keyword check - previously it committed the caller's open + # transaction before the ValueError was raised + fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.begin() + fresh_db.execute("insert into dogs (name) values ('Pancakes')") + with pytest.raises(ValueError): + fresh_db.query("/* comment */ COMMIT") + # The explicit transaction is still open and can still be rolled back + assert fresh_db.conn.in_transaction + fresh_db.rollback() + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + + def test_query_error_leaves_no_transaction_open(fresh_db): with pytest.raises(sqlite3.OperationalError): fresh_db.query("select * from missing_table") @@ -75,6 +101,41 @@ def test_query_pragma(tmpdir): db.close() +def test_query_comment_prefixed_pragma(tmpdir): + from sqlite_utils import Database + + db = Database(str(tmpdir / "test.db")) + # A leading comment must not stop a PRAGMA being recognized as one - + # previously it was executed inside the savepoint guard, where + # journal mode changes are refused + assert list(db.query("-- set WAL mode\npragma journal_mode = wal")) == [ + {"journal_mode": "wal"} + ] + db.close() + + +@pytest.mark.parametrize( + "sql,expected", + [ + ("select 1", "SELECT"), + (" \t\n select 1", "SELECT"), + ("-- comment\nbegin", "BEGIN"), + ("/* one */ /* two */ pragma user_version", "PRAGMA"), + ("/* multi\nline */vacuum", "VACUUM"), + ("insert into t values (1)", "INSERT"), + ("-- only a comment", ""), + ("/* unterminated", ""), + ("", ""), + (" ", ""), + ("123", ""), + ], +) +def test_first_keyword(sql, expected): + from sqlite_utils.db import _first_keyword + + assert _first_keyword(sql) == expected + + @pytest.mark.skipif( sqlite3.sqlite_version_info < (3, 35, 0), reason="RETURNING requires SQLite 3.35.0 or higher", From 61619498fa730b4bdaa6f4891465840ad177f2f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 11:50:10 -0700 Subject: [PATCH 052/120] Give db.query() SQL restrictions top billing in upgrading docs Rejecting statements that return no rows is the change most likely to require code edits when upgrading to 4.0, so it now leads the Python API changes section - before db.table() - with the full list of rejected statement types and a before/after example. Co-Authored-By: Claude Fable 5 --- docs/upgrading.rst | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 1974ad6..f743a3e 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -46,6 +46,18 @@ Two related things have been removed: Python API changes ------------------ +**db.query() now rejects SQL that does not return rows.** This is likely the most common change you will need to make to existing code. ``db.query()`` used to accept any SQL statement - passing one that returns no rows, such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause or a ``CREATE TABLE``, did nothing at all, silently. Those statements now raise a ``ValueError``, and are rolled back so they have no effect on the database. Transaction control statements (``BEGIN``, ``COMMIT``, ``END``, ``ROLLBACK``, ``SAVEPOINT``, ``RELEASE``) plus ``VACUUM``, ``ATTACH`` and ``DETACH`` are also rejected with a ``ValueError``, without being executed at all. Use ``db.execute()`` for statements that do not return rows: + +.. code-block:: python + + # 3.x accepted this but silently did nothing: + db.query("update dogs set name = 'Cleopaws'") + + # In 4.0 use execute() for SQL that does not return rows: + db.execute("update dogs set name = 'Cleopaws'") + +**db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily, but errors in your SQL raise at the ``db.query()`` call site rather than on first iteration, and a write with a ``RETURNING`` clause takes effect even if you never iterate over its results. + **db.table() no longer returns views.** ``db.table(name)`` now raises a ``sqlite_utils.db.NoTable`` exception if ``name`` is a SQL view. Use the new ``db.view(name)`` method for views: .. code-block:: python @@ -55,11 +67,6 @@ Python API changes ``db["name"]`` still returns either a ``Table`` or a ``View`` depending on what exists in the database. -**db.query() executes immediately.** ``db.query(sql)`` previously returned a generator that did not execute the SQL until you started iterating over it. The SQL now runs as soon as the method is called - rows are still fetched lazily. Two consequences: - -- Errors in your SQL now raise at the ``db.query()`` call site rather than on first iteration. -- Passing a statement that returns no rows - such as an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause - previously did nothing at all, silently. It now raises a ``ValueError``, and the statement is rolled back so it has no effect on the database. Use ``db.execute()`` for statements that do not return rows. - **Upserts use INSERT ... ON CONFLICT.** Upsert operations now use SQLite's ``INSERT ... ON CONFLICT SET`` syntax rather than the previous ``INSERT OR IGNORE`` followed by ``UPDATE``. If your code depends on the old behavior, pass ``use_old_upsert=True`` to the ``Database()`` constructor - see :ref:`python_api_old_upsert`. **Upsert records must include their primary keys.** ``table.upsert()`` and ``table.upsert_all()`` now raise ``sqlite_utils.db.PrimaryKeyRequired`` if a record is missing a value for any primary key column (or has ``None`` for one). Previously such records were quietly inserted as new rows. Relatedly, ``pk=`` is now optional when the table already exists with a primary key - it is detected automatically. From 623331b3f4ee78e1ed2618ccfc2e18e0bd77cd36 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 12:16:11 -0700 Subject: [PATCH 053/120] Fix for test failure against sqlean --- sqlite_utils/db.py | 12 ++++++++---- tests/test_query.py | 9 +++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f796a63..a8f8e17 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -713,10 +713,14 @@ class Database: args: tuple = (params,) if params is not None else () if keyword == "PRAGMA": # Some PRAGMA statements refuse to run inside a transaction, so - # execute these without the savepoint guard used below. PRAGMAs - # never open an implicit transaction, so there is nothing to - # undo if this one turns out not to return rows - cursor = self.conn.execute(sql, *args) + # execute these without the savepoint guard used below. Some + # adapters open an implicit transaction before comment-prefixed + # PRAGMAs, so temporarily use driver autocommit when it is safe. + if self.conn.in_transaction: + cursor = self.conn.execute(sql, *args) + else: + with self.ensure_autocommit_off(): + cursor = self.conn.execute(sql, *args) if cursor.description is None: raise ValueError(message) keys = [d[0] for d in cursor.description] diff --git a/tests/test_query.py b/tests/test_query.py index 1d5c1f6..b9822e1 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -114,6 +114,15 @@ def test_query_comment_prefixed_pragma(tmpdir): db.close() +def test_query_comment_prefixed_pragma_inside_transaction(fresh_db): + fresh_db.begin() + assert list(fresh_db.query("-- check version\npragma user_version")) == [ + {"user_version": 0} + ] + assert fresh_db.conn.in_transaction + fresh_db.rollback() + + @pytest.mark.parametrize( "sql,expected", [ From f10459cffbdaaada4a61b6b1b1b4bdb59041e88d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 13:59:25 -0700 Subject: [PATCH 054/120] table.foreign_keys now handles compound foreign keys, refs #594 This is a breaking change, since the return type is now a dataclass and not a namedtuple. --- docs/changelog.rst | 9 +++ docs/python-api.rst | 32 +++++++-- docs/upgrading.rst | 16 +++++ sqlite_utils/db.py | 129 ++++++++++++++++++++++++++++--------- tests/test_foreign_keys.py | 82 +++++++++++++++++++++++ 5 files changed, 232 insertions(+), 36 deletions(-) create mode 100644 tests/test_foreign_keys.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 2e1f69c..01d59a0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,15 @@ Changelog =========== +.. _v_unreleased: + +Unreleased +---------- + +Breaking changes: + +- ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` lists, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) + .. _v4_0rc2: 4.0rc2 (2026-07-04) diff --git a/docs/python-api.rst b/docs/python-api.rst index 3f7e12c..d1f466f 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2204,17 +2204,35 @@ Almost all SQLite tables have a ``rowid`` column, but a table with no explicitly .foreign_keys ------------- -The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey(table, column, other_table, other_column)`` named tuples. It is not available on views. +The ``.foreign_keys`` property returns any foreign key relationships for the table, as a list of ``ForeignKey`` objects. It is not available on views. + +Each ``ForeignKey`` has the following attributes: + +``table`` + The table the foreign key is defined on. +``column`` + The column on this table, or ``None`` for a compound foreign key. +``other_table`` + The table being referenced. +``other_column`` + The referenced column, or ``None`` for a compound foreign key. +``columns`` + A list of the columns on this table, always populated (a single-item list for single-column foreign keys). +``other_columns`` + A list of the referenced columns. +``is_compound`` + ``True`` if this is a compound (multi-column) foreign key. + +``ForeignKey`` was a ``namedtuple`` prior to sqlite-utils 4.0. It is now a dataclass and can no longer be unpacked or indexed as a tuple - access its fields by name instead. See :ref:`upgrading_3_to_4` for details. :: >>> db.table("Street_Tree_List").foreign_keys - [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qSiteInfo', other_table='qSiteInfo', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qSpecies', other_table='qSpecies', other_column='id'), - ForeignKey(table='Street_Tree_List', column='qCaretaker', other_table='qCaretaker', other_column='id'), - ForeignKey(table='Street_Tree_List', column='PlantType', other_table='PlantType', other_column='id')] + [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=['qLegalStatus'], other_columns=['id'], is_compound=False), + ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=['qCareAssistant'], other_columns=['id'], is_compound=False), + ...] + +Compound foreign keys - defined with ``FOREIGN KEY (col_a, col_b) REFERENCES other(col_a, col_b)`` - are returned as a single ``ForeignKey`` with ``is_compound=True``, ``column`` and ``other_column`` set to ``None``, and the participating columns available in the ``columns`` and ``other_columns`` lists. .. _python_api_introspection_schema: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index f743a3e..eb997f6 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -79,6 +79,22 @@ Python API changes **View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method. +**ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained three new fields - ``columns``, ``other_columns`` and ``is_compound`` - so that compound (multi-column) foreign keys can be represented as a single object. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead: + +.. code-block:: python + + # 3.x - tuple unpacking, no longer works: + for table, column, other_table, other_column in db["courses"].foreign_keys: + ... + + # 4.0 - access fields by name: + for fk in db["courses"].foreign_keys: + fk.table, fk.column, fk.other_table, fk.other_column + +Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. + +Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` lists instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are single-item lists. + **Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``. **Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a8f8e17..b1cdf83 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -11,6 +11,7 @@ from .utils import ( ) import binascii from collections import namedtuple +from dataclasses import dataclass, field from collections.abc import Mapping import contextlib import datetime @@ -161,9 +162,43 @@ Summary information about a column, see :ref:`python_api_analyze_column`. The ``N`` least common values as a list of ``(value, count)`` tuples, or ``None`` if the table is entirely distinct or if the number of distinct values is less than N (since they will already have been returned in ``most_common``) """ -ForeignKey = namedtuple( - "ForeignKey", ("table", "column", "other_table", "other_column") -) + + +@dataclass(order=True) +class ForeignKey: + """ + A foreign key defined on a table. + + For single-column foreign keys ``column`` and ``other_column`` hold the + column names, and ``columns``/``other_columns`` are single-item lists. + + For compound (multi-column) foreign keys ``column`` and ``other_column`` + are ``None`` - use ``columns`` and ``other_columns`` instead, and check + ``is_compound``. + + Prior to sqlite-utils 4.0 this was a ``namedtuple`` and could be unpacked + or indexed as ``(table, column, other_table, other_column)``. It is now a + dataclass - access its fields by name instead. + """ + + table: str + column: Optional[str] + other_table: str + other_column: Optional[str] + columns: List[str] = field(default_factory=list) + other_columns: List[str] = field(default_factory=list) + is_compound: bool = False + + def __post_init__(self): + # Populate columns/other_columns for single-column foreign keys + if not self.columns: + self.columns = [self.column] if self.column is not None else [] + if not self.other_columns: + self.other_columns = ( + [self.other_column] if self.other_column is not None else [] + ) + + Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns")) XIndex = namedtuple("XIndex", ("name", "columns")) XIndexColumn = namedtuple( @@ -1124,7 +1159,7 @@ class Database: if not all(isinstance(fk, (tuple, list)) for fk in foreign_keys): raise ValueError("foreign_keys= should be a list of tuples") fks = [] - for tuple_or_list in foreign_keys: + for tuple_or_list in cast(Iterable[Sequence[str]], foreign_keys): if len(tuple_or_list) == 4: if tuple_or_list[0] != name: raise ValueError( @@ -1234,7 +1269,7 @@ class Database: pk = hash_id # Soundness check foreign_keys point to existing tables for fk in foreign_keys: - if fk.other_table == name and columns.get(fk.other_column): + if fk.other_table == name and columns.get(cast(str, fk.other_column)): continue if fk.other_column != "rowid" and not any( c for c in self[fk.other_table].columns if c.name == fk.other_column @@ -1269,7 +1304,7 @@ class Database: foreign_keys_by_column[column_name].other_table ), quote_identifier( - foreign_keys_by_column[column_name].other_column + cast(str, foreign_keys_by_column[column_name].other_column) ), ) ) @@ -1499,7 +1534,9 @@ class Database: """ # foreign_keys is a list of explicit 4-tuples if not all( - len(fk) == 4 and isinstance(fk, (list, tuple)) for fk in foreign_keys + isinstance(fk, ForeignKey) + or (isinstance(fk, (list, tuple)) and len(fk) == 4) + for fk in foreign_keys ): raise ValueError( "foreign_keys must be a list of 4-tuples, " @@ -1509,7 +1546,16 @@ class Database: foreign_keys_to_create = [] # Verify that all tables and columns exist - for table, column, other_table, other_column in foreign_keys: + for fk in foreign_keys: + if isinstance(fk, ForeignKey): + table, column, other_table, other_column = ( + fk.table, + fk.column, + fk.other_table, + fk.other_column, + ) + else: + table, column, other_table, other_column = fk if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) table_obj = self.table(table) @@ -1555,8 +1601,11 @@ class Database: i.columns[0] for i in table.indexes if len(i.columns) == 1 } for fk in table.foreign_keys: - if fk.column not in existing_indexes: - table.create_index([fk.column], find_unique_name=True) + # Compound foreign keys expose their columns via fk.columns; + # single-column keys yield a one-item list + for column in fk.columns: + if column not in existing_indexes: + table.create_index([column], find_unique_name=True) def vacuum(self) -> None: "Run a SQLite ``VACUUM`` against the database." @@ -1907,21 +1956,40 @@ class Table(Queryable): @property def foreign_keys(self) -> List["ForeignKey"]: - "List of foreign keys defined on this table." - fks = [] + """ + List of foreign keys defined on this table. + + Compound (multi-column) foreign keys are returned as a single + ``ForeignKey`` with ``is_compound=True`` and populated + ``columns``/``other_columns`` lists. + """ + # PRAGMA foreign_key_list returns one row per column, grouped by "id" + # with "seq" giving the column order within a compound foreign key. + by_id: Dict[int, list] = {} for row in self.db.execute( "PRAGMA foreign_key_list({})".format(quote_identifier(self.name)) ).fetchall(): if row is not None: id, seq, table_name, from_, to_, on_update, on_delete, match = row - fks.append( - ForeignKey( - table=self.name, - column=from_, - other_table=table_name, - other_column=to_, - ) + by_id.setdefault(id, []).append((seq, table_name, from_, to_)) + fks = [] + for id in sorted(by_id): + rows = sorted(by_id[id]) # order columns by seq + other_table = rows[0][1] + columns = [row[2] for row in rows] + other_columns = [row[3] for row in rows] + is_compound = len(rows) > 1 + fks.append( + ForeignKey( + table=self.name, + column=None if is_compound else columns[0], + other_table=other_table, + other_column=None if is_compound else other_columns[0], + columns=columns, + other_columns=other_columns, + is_compound=is_compound, ) + ) return fks @property @@ -2254,17 +2322,20 @@ class Table(Queryable): else: # Construct foreign_keys from current, plus add_foreign_keys, minus drop_foreign_keys create_table_foreign_keys = [] - for table, column, other_table, other_column in self.foreign_keys: - # Copy over old foreign keys, unless we are dropping them - if (drop_foreign_keys is None) or (column not in drop_foreign_keys): - create_table_foreign_keys.append( - ForeignKey( - table, - rename.get(column) or column, - other_table, - other_column, + for fk in self.foreign_keys: + # Expand compound foreign keys into per-column references; + # for single-column keys this iterates exactly once + for column, other_column in zip(fk.columns, fk.other_columns): + # Copy over old foreign keys, unless we are dropping them + if (drop_foreign_keys is None) or (column not in drop_foreign_keys): + create_table_foreign_keys.append( + ForeignKey( + fk.table, + rename.get(column) or column, + fk.other_table, + other_column, + ) ) - ) # Add new foreign keys if add_foreign_keys is not None: for fk in self.db.resolve_foreign_keys(self.name, add_foreign_keys): diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py new file mode 100644 index 0000000..c785b59 --- /dev/null +++ b/tests/test_foreign_keys.py @@ -0,0 +1,82 @@ +"""Tests for reading compound (multi-column) foreign keys - issue #594.""" + +import pytest +from sqlite_utils import Database + +COMPOUND_SCHEMA = """ +CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + dept_name TEXT, + PRIMARY KEY (campus_name, dept_code) +); +CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + course_name TEXT, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + FOREIGN KEY (campus_name, dept_code) + REFERENCES departments(campus_name, dept_code) +); +""" + + +@pytest.fixture +def compound_db(): + db = Database(memory=True) + db.executescript(COMPOUND_SCHEMA) + return db + + +def test_compound_foreign_key(compound_db): + fks = compound_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.table == "courses" + assert fk.other_table == "departments" + assert fk.columns == ["campus_name", "dept_code"] + assert fk.other_columns == ["campus_name", "dept_code"] + # Scalar column/other_column can't sensibly hold a compound key + assert fk.column is None + assert fk.other_column is None + + +def test_single_foreign_key_gets_columns_fields(fresh_db): + fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) + fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fk = fresh_db["books"].foreign_keys[0] + assert fk.is_compound is False + assert fk.column == "author_id" + assert fk.other_column == "id" + assert fk.columns == ["author_id"] + assert fk.other_columns == ["id"] + + +def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): + # Clean break in 4.0: ForeignKey is a dataclass, not a namedtuple, so the + # old tuple unpacking and indexing patterns now fail hard. + fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1}) + fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fk = fresh_db["books"].foreign_keys[0] + with pytest.raises(TypeError): + table, column, other_table, other_column = fk + with pytest.raises(TypeError): + fk[0] + + +def test_foreign_keys_are_sortable(fresh_db): + fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id") + fresh_db["categories"].insert({"id": 1, "name": "Wildlife"}, pk="id") + fresh_db["books"].insert({"title": "Hedgehogs", "author_id": 1, "category_id": 1}) + fresh_db.add_foreign_keys( + [ + ("books", "author_id", "authors", "id"), + ("books", "category_id", "categories", "id"), + ] + ) + fks = sorted(fresh_db["books"].foreign_keys) + assert fks[0].column == "author_id" + assert fks[1].column == "category_id" From d5bf51df35a527aae19417c97eb3949ffb6df5bc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:26:56 -0700 Subject: [PATCH 055/120] Fix TypeError when sorting mixed compound/single foreign keys sorted() on a foreign_keys list containing both compound (column=None) and single-column ForeignKeys raised TypeError because dataclass ordering compared None against str. column/other_column are now excluded from comparison - ordering and equality use the always populated columns/other_columns lists instead. Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 6 ++++-- tests/test_foreign_keys.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index b1cdf83..67623fe 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -182,9 +182,11 @@ class ForeignKey: """ table: str - column: Optional[str] + # column/other_column are None for compound keys, which would break + # ordering against str values - comparison uses columns/other_columns + column: Optional[str] = field(compare=False) other_table: str - other_column: Optional[str] + other_column: Optional[str] = field(compare=False) columns: List[str] = field(default_factory=list) other_columns: List[str] = field(default_factory=list) is_compound: bool = False diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index c785b59..4b371a9 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -80,3 +80,31 @@ def test_foreign_keys_are_sortable(fresh_db): fks = sorted(fresh_db["books"].foreign_keys) assert fks[0].column == "author_id" assert fks[1].column == "category_id" + + +def test_mixed_compound_and_single_foreign_keys_are_sortable(): + # compound FKs have column=None, which must not break sorting + # against single-column FKs (None < str raises TypeError) + db = Database(memory=True) + db.executescript(""" + CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + PRIMARY KEY (campus_name, dept_code) + ); + CREATE TABLE accreditations (id INTEGER PRIMARY KEY); + CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + accreditation_id INTEGER REFERENCES accreditations(id), + FOREIGN KEY (campus_name, dept_code) + REFERENCES departments(campus_name, dept_code) + ); + """) + fks = db["courses"].foreign_keys + assert len(fks) == 2 + assert {fk.is_compound for fk in fks} == {True, False} + fks_sorted = sorted(fks) + assert fks_sorted[0].other_table == "accreditations" + assert fks_sorted[1].other_table == "departments" From 577f3011e5bbc5a043b7e4f9d10b06da76e21bf1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:30:41 -0700 Subject: [PATCH 056/120] Create tables with compound foreign keys, refs #594 create_table() and friends now accept compound foreign keys, specified as lists of column names in the existing tuple forms: foreign_keys=[ (["campus_name", "dept_code"], "departments"), (["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"]), ] The two-item form guesses the compound primary key of the other table. Compound keys are rendered as table-level FOREIGN KEY constraints, while single-column keys keep their inline REFERENCES clauses. Co-Authored-By: Claude Fable 5 --- docs/python-api.rst | 36 ++++++++++++ sqlite_utils/db.py | 110 +++++++++++++++++++++++++++++-------- tests/test_foreign_keys.py | 96 +++++++++++++++++++++++++++++++- 3 files changed, 218 insertions(+), 24 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index d1f466f..5e55256 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -817,6 +817,42 @@ You can leave off the third item in the tuple to have the referenced column auto ("author_id", "authors") ]) +.. _python_api_compound_foreign_keys: + +Compound foreign keys +~~~~~~~~~~~~~~~~~~~~~ + +To create a compound (multi-column) foreign key, use lists of column names in place of the single column names: + +.. code-block:: python + + db.table("courses").create({ + "course_code": str, + "campus_name": str, + "dept_code": str, + }, pk="course_code", foreign_keys=[ + (["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"]) + ]) + +This creates a table-level constraint: + +.. code-block:: sql + + CREATE TABLE "courses" ( + "course_code" TEXT PRIMARY KEY, + "campus_name" TEXT, + "dept_code" TEXT, + FOREIGN KEY ("campus_name", "dept_code") REFERENCES "departments"("campus_name", "dept_code") + ) + +As with single columns, you can leave off the list of other columns to reference the compound primary key of the other table: + +.. code-block:: python + + foreign_keys=[ + (["campus_name", "dept_code"], "departments") + ] + .. _python_api_table_configuration: Table configuration options diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 67623fe..12914d1 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -219,6 +219,10 @@ ForeignKeyIndicator = Union[ Tuple[str, str], Tuple[str, str, str], Tuple[str, str, str, str], + # Compound foreign keys use lists of columns: + Tuple[List[str], str], + Tuple[List[str], str, List[str]], + Tuple[str, List[str], str, List[str]], ] ForeignKeysType = Union[Iterable[ForeignKeyIndicator], List[ForeignKeyIndicator]] @@ -1142,9 +1146,12 @@ class Database: :param name: Name of table that foreign keys are being defined for :param foreign_keys: List of foreign keys, each of which can be a - string, a ForeignKey() named tuple, a tuple of (column, other_table), + string, a ForeignKey() object, a tuple of (column, other_table), or a tuple of (column, other_table, other_column), or a tuple of - (table, column, other_table, other_column) + (table, column, other_table, other_column). For compound foreign + keys the column elements can be lists of column names, e.g. + (["campus_name", "dept_code"], "departments") or + (["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"]) """ table = self.table(name) if all(isinstance(fk, ForeignKey) for fk in foreign_keys): @@ -1161,7 +1168,7 @@ class Database: if not all(isinstance(fk, (tuple, list)) for fk in foreign_keys): raise ValueError("foreign_keys= should be a list of tuples") fks = [] - for tuple_or_list in cast(Iterable[Sequence[str]], foreign_keys): + for tuple_or_list in cast(Iterable[Sequence[Any]], foreign_keys): if len(tuple_or_list) == 4: if tuple_or_list[0] != name: raise ValueError( @@ -1169,28 +1176,61 @@ class Database: tuple_or_list, name ) ) - if len(tuple_or_list) not in (2, 3, 4): + tuple_or_list = tuple_or_list[1:] + if len(tuple_or_list) not in (2, 3): raise ValueError( "foreign_keys= should be a list of tuple pairs or triples" ) - if len(tuple_or_list) in (3, 4): - if len(tuple_or_list) == 4: - tuple_or_list = cast(Tuple[str, str, str], tuple_or_list[1:]) + column_or_columns = tuple_or_list[0] + other_table = tuple_or_list[1] + if isinstance(column_or_columns, (list, tuple)): + # Compound foreign key + columns = list(column_or_columns) + if len(tuple_or_list) == 3: + other_columns = tuple_or_list[2] + if not isinstance(other_columns, (list, tuple)): + raise ValueError( + "Compound foreign key {} should reference a list " + "of other columns".format(tuple(tuple_or_list)) + ) + other_columns = list(other_columns) else: - tuple_or_list = cast(Tuple[str, str, str], tuple_or_list) - fks.append( - ForeignKey( - name, tuple_or_list[0], tuple_or_list[1], tuple_or_list[2] + # Guess the compound primary key of the other table + other_columns = self.table(other_table).pks + if len(columns) != len(other_columns): + raise ValueError( + "Compound foreign key {} should have the same number " + "of columns on both sides".format(tuple(tuple_or_list)) ) + if len(columns) == 1: + # Single-column key passed as a one-item list + fks.append( + ForeignKey(name, columns[0], other_table, other_columns[0]) + ) + else: + fks.append( + ForeignKey( + name, + None, + other_table, + None, + columns=columns, + other_columns=other_columns, + is_compound=True, + ) + ) + elif len(tuple_or_list) == 3: + fks.append( + ForeignKey(name, column_or_columns, other_table, tuple_or_list[2]) ) else: # Guess the primary key fks.append( ForeignKey( name, - tuple_or_list[0], - tuple_or_list[1], - table.guess_foreign_column(tuple_or_list[1]), + column_or_columns, + other_table, + table.guess_foreign_column(other_table), ) ) return fks @@ -1229,7 +1269,11 @@ class Database: if hash_id_columns and (hash_id is None): hash_id = "id" foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) - foreign_keys_by_column = {fk.column: fk for fk in foreign_keys} + # Compound foreign keys are rendered as table-level constraints; + # single-column ones as inline REFERENCES on their column + foreign_keys_by_column = { + fk.column: fk for fk in foreign_keys if not fk.is_compound + } # any extracts will be treated as integer columns with a foreign key extracts = resolve_extracts(extracts) for extract_column, extract_table in extracts.items(): @@ -1271,14 +1315,15 @@ class Database: pk = hash_id # Soundness check foreign_keys point to existing tables for fk in foreign_keys: - if fk.other_table == name and columns.get(cast(str, fk.other_column)): - continue - if fk.other_column != "rowid" and not any( - c for c in self[fk.other_table].columns if c.name == fk.other_column - ): - raise AlterError( - "No such column: {}.{}".format(fk.other_table, fk.other_column) - ) + for other_column in fk.other_columns: + if fk.other_table == name and columns.get(other_column): + continue + if other_column != "rowid" and not any( + c for c in self[fk.other_table].columns if c.name == other_column + ): + raise AlterError( + "No such column: {}.{}".format(fk.other_table, other_column) + ) column_defs = [] # ensure pk is a tuple @@ -1329,6 +1374,25 @@ class Database: extra_pk = ",\n PRIMARY KEY ({pks})".format( pks=", ".join([quote_identifier(p) for p in pk]) ) + # Compound foreign keys become table-level FOREIGN KEY constraints + column_names = [c[0] for c in column_items] + for fk in foreign_keys: + if not fk.is_compound: + continue + missing = [c for c in fk.columns if c not in column_names] + if missing: + raise AlterError( + "No such column: {}".format(", ".join(sorted(missing))) + ) + column_defs.append( + " FOREIGN KEY ({columns}) REFERENCES {other_table}({other_columns})".format( + columns=", ".join(quote_identifier(c) for c in fk.columns), + other_table=quote_identifier(fk.other_table), + other_columns=", ".join( + quote_identifier(c) for c in fk.other_columns + ), + ) + ) columns_sql = ",\n".join(column_defs) sql = """CREATE TABLE {if_not_exists}{table} ( {columns_sql}{extra_pk} diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 4b371a9..457f307 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -1,7 +1,8 @@ -"""Tests for reading compound (multi-column) foreign keys - issue #594.""" +"""Tests for compound (multi-column) foreign keys - issue #594.""" import pytest from sqlite_utils import Database +from sqlite_utils.db import AlterError, ForeignKey COMPOUND_SCHEMA = """ CREATE TABLE departments ( @@ -108,3 +109,96 @@ def test_mixed_compound_and_single_foreign_keys_are_sortable(): fks_sorted = sorted(fks) assert fks_sorted[0].other_table == "accreditations" assert fks_sorted[1].other_table == "departments" + + +@pytest.fixture +def departments_db(): + db = Database(memory=True) + db.create_table( + "departments", + {"campus_name": str, "dept_code": str, "dept_name": str}, + pk=("campus_name", "dept_code"), + ) + return db + + +EXPECTED_COURSES_SCHEMA = ( + 'CREATE TABLE "courses" (\n' + ' "course_code" TEXT PRIMARY KEY,\n' + ' "campus_name" TEXT,\n' + ' "dept_code" TEXT,\n' + ' FOREIGN KEY ("campus_name", "dept_code") ' + 'REFERENCES "departments"("campus_name", "dept_code")\n' + ")" +) + + +@pytest.mark.parametrize( + "foreign_keys", + ( + [ + ForeignKey( + table="courses", + column=None, + other_table="departments", + other_column=None, + columns=["campus_name", "dept_code"], + other_columns=["campus_name", "dept_code"], + is_compound=True, + ) + ], + [(["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"])], + # Two-item form guesses the other table's primary key: + [(["campus_name", "dept_code"], "departments")], + ), +) +def test_create_table_with_compound_foreign_key(departments_db, foreign_keys): + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + foreign_keys=foreign_keys, + ) + assert departments_db["courses"].schema == EXPECTED_COURSES_SCHEMA + fks = departments_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ["campus_name", "dept_code"] + assert fk.other_table == "departments" + assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_create_table_compound_foreign_key_enforced(departments_db): + departments_db.execute("PRAGMA foreign_keys = ON") + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + foreign_keys=[(["campus_name", "dept_code"], "departments")], + ) + departments_db["departments"].insert( + {"campus_name": "Berkeley", "dept_code": "CS", "dept_name": "Computer Science"} + ) + departments_db["courses"].insert( + {"course_code": "CS101", "campus_name": "Berkeley", "dept_code": "CS"} + ) + import sqlite3 + + with pytest.raises(sqlite3.IntegrityError): + departments_db.execute( + "insert into courses (course_code, campus_name, dept_code) " + "values ('X1', 'Nowhere', 'NOPE')" + ) + + +def test_create_table_compound_foreign_key_missing_other_column(departments_db): + with pytest.raises(AlterError): + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + foreign_keys=[ + (["campus_name", "dept_code"], "departments", ["campus_name", "nope"]) + ], + ) From 7d43fd50e186c359cd0c61a6655f15f0d156da87 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:33:15 -0700 Subject: [PATCH 057/120] transform() now round-trips compound foreign keys, refs #594 Compound foreign keys are passed through table.transform() intact instead of being degraded to per-column single foreign keys: - rename= applies to each member column of a compound key - drop= of any member column drops the whole constraint, matching the existing single-column behavior - drop_foreign_keys= accepts a bare column name (drops any foreign key that column participates in) or a tuple of columns (drops the compound key with exactly those columns) Co-Authored-By: Claude Fable 5 --- docs/python-api.rst | 10 ++++++ sqlite_utils/db.py | 63 ++++++++++++++++++++++++-------------- tests/test_foreign_keys.py | 48 +++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 23 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 5e55256..465df79 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1793,6 +1793,16 @@ This example drops two foreign keys - the one from ``places.country`` to ``count drop_foreign_keys=("country", "continent") ) +A bare column name drops any foreign key that column participates in, including compound foreign keys. To target a compound foreign key precisely, pass a tuple of its columns: + +.. code-block:: python + + db.table("courses").transform( + drop_foreign_keys=[("campus_name", "dept_code")] + ) + +Renaming a column with ``rename=`` updates any foreign keys that use it, and dropping a column with ``drop=`` also drops any foreign keys it participates in - for a compound foreign key this removes the whole constraint. + .. _python_api_transform_sql: Custom transformations with .transform_sql() diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 12914d1..c37f750 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2275,7 +2275,9 @@ class Table(Queryable): :param pk: New primary key for the table :param not_null: Columns to set as ``NOT NULL`` :param defaults: Default values for columns - :param drop_foreign_keys: Names of columns that should have their foreign key constraints removed + :param drop_foreign_keys: Foreign key constraints to remove - a column name + drops any foreign key that column participates in, a tuple of column names + drops the compound foreign key with exactly those columns :param add_foreign_keys: List of foreign keys to add to the table :param foreign_keys: List of foreign keys to set for the table, replacing any existing foreign keys :param column_order: List of strings specifying a full or partial column order @@ -2360,7 +2362,9 @@ class Table(Queryable): :param pk: New primary key for the table :param not_null: Columns to set as ``NOT NULL`` :param defaults: Default values for columns - :param drop_foreign_keys: Names of columns that should have their foreign key constraints removed + :param drop_foreign_keys: Foreign key constraints to remove - a column name + drops any foreign key that column participates in, a tuple of column names + drops the compound foreign key with exactly those columns :param add_foreign_keys: List of foreign keys to add to the table :param foreign_keys: List of foreign keys to set for the table, replacing any existing foreign keys :param column_order: List of strings specifying a full or partial column order @@ -2387,32 +2391,45 @@ class Table(Queryable): create_table_foreign_keys.extend(foreign_keys) else: # Construct foreign_keys from current, plus add_foreign_keys, minus drop_foreign_keys + + def fk_should_be_dropped(fk: ForeignKey) -> bool: + if drop_foreign_keys is not None: + for spec in drop_foreign_keys: + if isinstance(spec, str): + # A column name matches any foreign key it participates in + if spec in fk.columns: + return True + elif list(spec) == fk.columns: + # A tuple/list must match a compound key's columns exactly + return True + # Dropping any of a foreign key's columns drops the whole key + return any(column in drop for column in fk.columns) + + def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: + columns = [rename.get(column) or column for column in fk.columns] + if fk.is_compound: + return ForeignKey( + self.name, + None, + fk.other_table, + None, + columns=columns, + other_columns=fk.other_columns, + is_compound=True, + ) + return ForeignKey( + self.name, columns[0], fk.other_table, fk.other_columns[0] + ) + create_table_foreign_keys = [] + # Copy over old foreign keys, unless we are dropping them for fk in self.foreign_keys: - # Expand compound foreign keys into per-column references; - # for single-column keys this iterates exactly once - for column, other_column in zip(fk.columns, fk.other_columns): - # Copy over old foreign keys, unless we are dropping them - if (drop_foreign_keys is None) or (column not in drop_foreign_keys): - create_table_foreign_keys.append( - ForeignKey( - fk.table, - rename.get(column) or column, - fk.other_table, - other_column, - ) - ) + if not fk_should_be_dropped(fk): + create_table_foreign_keys.append(fk_with_renamed_columns(fk)) # Add new foreign keys if add_foreign_keys is not None: for fk in self.db.resolve_foreign_keys(self.name, add_foreign_keys): - create_table_foreign_keys.append( - ForeignKey( - self.name, - rename.get(fk.column) or fk.column, - fk.other_table, - fk.other_column, - ) - ) + create_table_foreign_keys.append(fk_with_renamed_columns(fk)) new_table_name = "{}_new_{}".format( self.name, tmp_suffix or os.urandom(6).hex() diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 457f307..d6a7071 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -202,3 +202,51 @@ def test_create_table_compound_foreign_key_missing_other_column(departments_db): (["campus_name", "dept_code"], "departments", ["campus_name", "nope"]) ], ) + + +def test_transform_preserves_compound_foreign_key(compound_db): + compound_db["courses"].transform(rename={"course_name": "title"}) + fks = compound_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ["campus_name", "dept_code"] + assert fk.other_table == "departments" + assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_transform_rename_member_column_updates_compound_foreign_key(compound_db): + compound_db["courses"].transform(rename={"campus_name": "campus"}) + fks = compound_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ["campus", "dept_code"] + # Referenced columns in the other table are unchanged + assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_transform_drop_member_column_drops_compound_foreign_key(compound_db): + # Matches single-column behavior: dropping the column silently + # drops the foreign key that used it + compound_db["courses"].transform(drop={"dept_code"}) + assert compound_db["courses"].foreign_keys == [] + assert "FOREIGN KEY" not in compound_db["courses"].schema + + +@pytest.mark.parametrize( + "drop_foreign_keys", + ( + # A bare column name matches any foreign key it participates in: + ["campus_name"], + # A tuple must match the full compound key: + [("campus_name", "dept_code")], + ), +) +def test_transform_drop_compound_foreign_key(compound_db, drop_foreign_keys): + compound_db["courses"].transform(drop_foreign_keys=drop_foreign_keys) + assert compound_db["courses"].foreign_keys == [] + # The columns themselves survive + assert {"campus_name", "dept_code"} <= set( + compound_db["courses"].columns_dict.keys() + ) From b75edf4b30dd500747f6f9f5f1dcb52c03a5b442 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:37:32 -0700 Subject: [PATCH 058/120] add_foreign_key() accepts compound keys, composite FK indexes, refs #594 table.add_foreign_key() and db.add_foreign_keys() now accept lists of column names to create compound foreign keys: table.add_foreign_key( ["campus_name", "dept_code"], "departments" ) Omitting the other columns uses the compound primary key of the other table. Duplicate detection compares the full column lists. db.index_foreign_keys() now creates a single composite index across the columns of a compound foreign key, rather than an index per column. Co-Authored-By: Claude Fable 5 --- docs/python-api.rst | 12 ++++ sqlite_utils/db.py | 138 ++++++++++++++++++++++++------------- tests/test_foreign_keys.py | 74 ++++++++++++++++++++ 3 files changed, 177 insertions(+), 47 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 465df79..882f952 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1562,6 +1562,16 @@ To ignore the case where the key already exists, use ``ignore=True``: db.table("books").add_foreign_key("author_id", "authors", "id", ignore=True) +To add a compound foreign key, pass lists of columns: + +.. code-block:: python + + db.table("courses").add_foreign_key( + ["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"] + ) + +As with single columns, omitting the other columns will use the compound primary key of the other table. ``other_table`` must always be specified for a compound foreign key. + .. _python_api_add_foreign_keys: Adding multiple foreign key constraints at once @@ -1591,6 +1601,8 @@ If you want to ensure that every foreign key column in your database has a corre db.index_foreign_keys() +Compound foreign keys get a single composite index across their columns. + .. _python_api_drop: Dropping a table or view diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index c37f750..3c78e3b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1590,13 +1590,14 @@ class Database: return candidates def add_foreign_keys( - self, foreign_keys: Iterable[Tuple[str, str, str, str]] + self, foreign_keys: Iterable[Union[ForeignKey, Tuple[str, Any, str, Any]]] ) -> None: """ See :ref:`python_api_add_foreign_keys`. :param foreign_keys: A list of ``(table, column, other_table, other_column)`` - tuples + tuples - for compound foreign keys, ``column`` and ``other_column`` can + be lists of column names """ # foreign_keys is a list of explicit 4-tuples if not all( @@ -1609,44 +1610,64 @@ class Database: "(table, column, other_table, other_column)" ) - foreign_keys_to_create = [] + foreign_keys_to_create: List[Tuple[str, Any, str, Any]] = [] # Verify that all tables and columns exist for fk in foreign_keys: if isinstance(fk, ForeignKey): - table, column, other_table, other_column = ( + table, columns, other_table, other_columns = ( fk.table, - fk.column, + fk.columns, fk.other_table, - fk.other_column, + fk.other_columns, ) else: - table, column, other_table, other_column = fk + table, column_or_columns, other_table, other_column_or_columns = fk + # Compound foreign keys use lists of columns + columns = ( + [column_or_columns] + if isinstance(column_or_columns, str) + else list(column_or_columns) + ) + other_columns = ( + [other_column_or_columns] + if isinstance(other_column_or_columns, str) + else list(other_column_or_columns) + ) if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) table_obj = self.table(table) - if column not in table_obj.columns_dict: - raise AlterError("No such column: {} in {}".format(column, table)) + for column in columns: + if column not in table_obj.columns_dict: + raise AlterError("No such column: {} in {}".format(column, table)) if not self[other_table].exists(): raise AlterError("No such other_table: {}".format(other_table)) - if ( - other_column != "rowid" - and other_column not in self[other_table].columns_dict - ): - raise AlterError( - "No such other_column: {} in {}".format(other_column, other_table) - ) + for other_column in other_columns: + if ( + other_column != "rowid" + and other_column not in self[other_table].columns_dict + ): + raise AlterError( + "No such other_column: {} in {}".format( + other_column, other_table + ) + ) # We will silently skip foreign keys that exist already if not any( fk for fk in table_obj.foreign_keys - if fk.column == column + if fk.columns == columns and fk.other_table == other_table - and fk.other_column == other_column + and fk.other_columns == other_columns ): - foreign_keys_to_create.append( - (table, column, other_table, other_column) - ) + if len(columns) == 1: + foreign_keys_to_create.append( + (table, columns[0], other_table, other_columns[0]) + ) + else: + foreign_keys_to_create.append( + (table, columns, other_table, other_columns) + ) # Group them by table by_table: Dict[str, List] = {} @@ -1663,15 +1684,12 @@ class Database: "Create indexes for every foreign key column on every table in the database." for table_name in self.table_names(): table = self.table(table_name) - existing_indexes = { - i.columns[0] for i in table.indexes if len(i.columns) == 1 - } + existing_indexes = {tuple(i.columns) for i in table.indexes} for fk in table.foreign_keys: - # Compound foreign keys expose their columns via fk.columns; - # single-column keys yield a one-item list - for column in fk.columns: - if column not in existing_indexes: - table.create_index([column], find_unique_name=True) + # A compound foreign key gets a single composite index + if tuple(fk.columns) not in existing_indexes: + table.create_index(fk.columns, find_unique_name=True) + existing_indexes.add(tuple(fk.columns)) def vacuum(self) -> None: "Run a SQLite ``VACUUM`` against the database." @@ -2862,52 +2880,78 @@ class Table(Queryable): def add_foreign_key( self, - column: str, + column: Union[str, List[str]], other_table: Optional[str] = None, - other_column: Optional[str] = None, + other_column: Optional[Union[str, List[str]]] = None, ignore: bool = False, ): """ Alter the schema to mark the specified column as a foreign key to another table. - :param column: The column to mark as a foreign key. + :param column: The column to mark as a foreign key - use a list of columns + for a compound foreign key. :param other_table: The table it refers to - if omitted, will be guessed based on the column name. :param other_column: The column on the other table it - if omitted, will be guessed. + Use a list of columns for a compound foreign key. :param ignore: Set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. """ - # Ensure column exists - if column not in self.columns_dict: - raise AlterError("No such column: {}".format(column)) + columns = [column] if isinstance(column, str) else list(column) + # Ensure columns exist + for col in columns: + if col not in self.columns_dict: + raise AlterError("No such column: {}".format(col)) # If other_table is not specified, attempt to guess it from the column if other_table is None: - other_table = self.guess_foreign_table(column) + if len(columns) > 1: + raise ValueError( + "other_table must be specified for a compound foreign key" + ) + other_table = self.guess_foreign_table(columns[0]) # If other_column is not specified, detect the primary key on other_table if other_column is None: - other_column = self.guess_foreign_column(other_table) + if len(columns) > 1: + other_columns = self.db.table(other_table).pks + else: + other_columns = [self.guess_foreign_column(other_table)] + elif isinstance(other_column, str): + other_columns = [other_column] + else: + other_columns = list(other_column) + if len(columns) != len(other_columns): + raise ValueError( + "Compound foreign key must have the same number of columns " + "on both sides" + ) - # Soundness check that the other column exists - if ( - not [c for c in self.db[other_table].columns if c.name == other_column] - and other_column != "rowid" - ): - raise AlterError("No such column: {}.{}".format(other_table, other_column)) + # Soundness check that the other columns exist + for other_col in other_columns: + if ( + not [c for c in self.db[other_table].columns if c.name == other_col] + and other_col != "rowid" + ): + raise AlterError("No such column: {}.{}".format(other_table, other_col)) # Check we do not already have an existing foreign key if any( fk for fk in self.foreign_keys - if fk.column == column + if fk.columns == columns and fk.other_table == other_table - and fk.other_column == other_column + and fk.other_columns == other_columns ): if ignore: return self else: raise AlterError( "Foreign key already exists for {} => {}.{}".format( - column, other_table, other_column + ", ".join(columns), other_table, ", ".join(other_columns) ) ) - self.db.add_foreign_keys([(self.name, column, other_table, other_column)]) + if len(columns) == 1: + self.db.add_foreign_keys( + [(self.name, columns[0], other_table, other_columns[0])] + ) + else: + self.db.add_foreign_keys([(self.name, columns, other_table, other_columns)]) return self def enable_counts(self) -> None: diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index d6a7071..1fabce7 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -250,3 +250,77 @@ def test_transform_drop_compound_foreign_key(compound_db, drop_foreign_keys): assert {"campus_name", "dept_code"} <= set( compound_db["courses"].columns_dict.keys() ) + + +@pytest.fixture +def courses_db(departments_db): + departments_db.create_table( + "courses", + {"course_code": str, "campus_name": str, "dept_code": str}, + pk="course_code", + ) + return departments_db + + +def test_add_compound_foreign_key(courses_db): + t = courses_db["courses"].add_foreign_key( + ["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"] + ) + # Returns self + assert t.name == "courses" + fks = courses_db["courses"].foreign_keys + assert len(fks) == 1 + fk = fks[0] + assert fk.is_compound is True + assert fk.columns == ["campus_name", "dept_code"] + assert fk.other_table == "departments" + assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_add_compound_foreign_key_guesses_other_columns(courses_db): + courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") + fk = courses_db["courses"].foreign_keys[0] + assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_add_compound_foreign_key_error_if_already_exists(courses_db): + courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") + with pytest.raises(AlterError) as ex: + courses_db["courses"].add_foreign_key( + ["campus_name", "dept_code"], "departments" + ) + assert "already exists" in ex.value.args[0] + # ignore=True should not raise + courses_db["courses"].add_foreign_key( + ["campus_name", "dept_code"], "departments", ignore=True + ) + + +def test_add_compound_foreign_key_error_if_column_missing(courses_db): + with pytest.raises(AlterError): + courses_db["courses"].add_foreign_key(["campus_name", "nope"], "departments") + + +def test_db_add_foreign_keys_compound(courses_db): + courses_db.add_foreign_keys( + [ + ( + "courses", + ["campus_name", "dept_code"], + "departments", + ["campus_name", "dept_code"], + ) + ] + ) + fk = courses_db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.columns == ["campus_name", "dept_code"] + + +def test_index_foreign_keys_compound_creates_composite_index(compound_db): + compound_db.index_foreign_keys() + index_columns = [i.columns for i in compound_db["courses"].indexes] + assert ["campus_name", "dept_code"] in index_columns + # No separate single-column indexes for the members + assert ["campus_name"] not in index_columns + assert ["dept_code"] not in index_columns From be27a96484b69fcc062b54c384e8713a9d500d9a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:40:57 -0700 Subject: [PATCH 059/120] Capture and preserve foreign key ON DELETE/ON UPDATE actions ForeignKey gains on_delete and on_update fields (default "NO ACTION"), populated from PRAGMA foreign_key_list. create_table_sql() renders the corresponding clauses for both inline and compound table-level foreign keys, which means table.transform() now preserves actions such as ON DELETE CASCADE - previously they were silently stripped whenever a table was transformed. Co-Authored-By: Claude Fable 5 --- docs/python-api.rst | 23 +++++++++++ sqlite_utils/db.py | 42 ++++++++++++++----- tests/test_foreign_keys.py | 84 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 10 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 882f952..f44a10b 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -853,6 +853,25 @@ As with single columns, you can leave off the list of other columns to reference (["campus_name", "dept_code"], "departments") ] +To specify ``ON DELETE`` or ``ON UPDATE`` actions, pass ``ForeignKey`` objects instead: + +.. code-block:: python + + from sqlite_utils.db import ForeignKey + + db.table("books").create({ + "id": int, + "author_id": int, + }, pk="id", foreign_keys=[ + ForeignKey( + table="books", column="author_id", + other_table="authors", other_column="id", + on_delete="CASCADE", + ) + ]) + +Foreign key actions are preserved by :ref:`table.transform() ` - prior to sqlite-utils 4.0 they were silently dropped when a table was transformed. + .. _python_api_table_configuration: Table configuration options @@ -2280,6 +2299,10 @@ Each ``ForeignKey`` has the following attributes: A list of the referenced columns. ``is_compound`` ``True`` if this is a compound (multi-column) foreign key. +``on_delete`` + The ``ON DELETE`` action, e.g. ``"CASCADE"`` - ``"NO ACTION"`` if not set. +``on_update`` + The ``ON UPDATE`` action - ``"NO ACTION"`` if not set. ``ForeignKey`` was a ``namedtuple`` prior to sqlite-utils 4.0. It is now a dataclass and can no longer be unpacked or indexed as a tuple - access its fields by name instead. See :ref:`upgrading_3_to_4` for details. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3c78e3b..1910b84 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -190,6 +190,8 @@ class ForeignKey: columns: List[str] = field(default_factory=list) other_columns: List[str] = field(default_factory=list) is_compound: bool = False + on_delete: str = "NO ACTION" + on_update: str = "NO ACTION" def __post_init__(self): # Populate columns/other_columns for single-column foreign keys @@ -201,6 +203,16 @@ class ForeignKey: ) +def _fk_actions_sql(fk: ForeignKey) -> str: + "ON UPDATE/ON DELETE clauses for a foreign key, or an empty string." + actions = "" + if fk.on_update and fk.on_update != "NO ACTION": + actions += " ON UPDATE {}".format(fk.on_update) + if fk.on_delete and fk.on_delete != "NO ACTION": + actions += " ON DELETE {}".format(fk.on_delete) + return actions + + Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns")) XIndex = namedtuple("XIndex", ("name", "columns")) XIndexColumn = namedtuple( @@ -1345,14 +1357,12 @@ class Database: "DEFAULT {}".format(self.quote_default_value(defaults[column_name])) ) if column_name in foreign_keys_by_column: + fk = foreign_keys_by_column[column_name] column_extras.append( - "REFERENCES {}({})".format( - quote_identifier( - foreign_keys_by_column[column_name].other_table - ), - quote_identifier( - cast(str, foreign_keys_by_column[column_name].other_column) - ), + "REFERENCES {}({}){}".format( + quote_identifier(fk.other_table), + quote_identifier(cast(str, fk.other_column)), + _fk_actions_sql(fk), ) ) column_type_str = COLUMN_TYPE_MAPPING[column_type] @@ -1385,12 +1395,13 @@ class Database: "No such column: {}".format(", ".join(sorted(missing))) ) column_defs.append( - " FOREIGN KEY ({columns}) REFERENCES {other_table}({other_columns})".format( + " FOREIGN KEY ({columns}) REFERENCES {other_table}({other_columns}){actions}".format( columns=", ".join(quote_identifier(c) for c in fk.columns), other_table=quote_identifier(fk.other_table), other_columns=", ".join( quote_identifier(c) for c in fk.other_columns ), + actions=_fk_actions_sql(fk), ) ) columns_sql = ",\n".join(column_defs) @@ -2055,7 +2066,9 @@ class Table(Queryable): ).fetchall(): if row is not None: id, seq, table_name, from_, to_, on_update, on_delete, match = row - by_id.setdefault(id, []).append((seq, table_name, from_, to_)) + by_id.setdefault(id, []).append( + (seq, table_name, from_, to_, on_update, on_delete) + ) fks = [] for id in sorted(by_id): rows = sorted(by_id[id]) # order columns by seq @@ -2072,6 +2085,8 @@ class Table(Queryable): columns=columns, other_columns=other_columns, is_compound=is_compound, + on_update=rows[0][4], + on_delete=rows[0][5], ) ) return fks @@ -2434,9 +2449,16 @@ class Table(Queryable): columns=columns, other_columns=fk.other_columns, is_compound=True, + on_delete=fk.on_delete, + on_update=fk.on_update, ) return ForeignKey( - self.name, columns[0], fk.other_table, fk.other_columns[0] + self.name, + columns[0], + fk.other_table, + fk.other_columns[0], + on_delete=fk.on_delete, + on_update=fk.on_update, ) create_table_foreign_keys = [] diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 1fabce7..afe333f 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -324,3 +324,87 @@ def test_index_foreign_keys_compound_creates_composite_index(compound_db): # No separate single-column indexes for the members assert ["campus_name"] not in index_columns assert ["dept_code"] not in index_columns + + +def test_foreign_key_captures_on_delete_and_on_update(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + author_id INTEGER REFERENCES authors(id) + ON DELETE CASCADE ON UPDATE RESTRICT + ); + """) + fk = db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert fk.on_update == "RESTRICT" + + +def test_foreign_key_on_delete_defaults_to_no_action(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db["books"].add_foreign_key("author_id", "authors", "id") + fk = fresh_db["books"].foreign_keys[0] + assert fk.on_delete == "NO ACTION" + assert fk.on_update == "NO ACTION" + + +def test_create_table_foreign_key_with_on_delete(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db.create_table( + "books", + {"id": int, "author_id": int}, + pk="id", + foreign_keys=[ + ForeignKey( + table="books", + column="author_id", + other_table="authors", + other_column="id", + on_delete="CASCADE", + ) + ], + ) + assert "ON DELETE CASCADE" in fresh_db["books"].schema + assert fresh_db["books"].foreign_keys[0].on_delete == "CASCADE" + + +def test_transform_preserves_on_delete_cascade(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE authors (id INTEGER PRIMARY KEY); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + title TEXT, + author_id INTEGER REFERENCES authors(id) ON DELETE CASCADE + ); + """) + db["books"].transform(rename={"title": "book_title"}) + fk = db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert fk.on_update == "NO ACTION" + assert "ON DELETE CASCADE" in db["books"].schema + + +def test_transform_preserves_compound_foreign_key_on_delete(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + PRIMARY KEY (campus_name, dept_code) + ); + CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + FOREIGN KEY (campus_name, dept_code) + REFERENCES departments(campus_name, dept_code) ON DELETE CASCADE + ); + """) + db["courses"].transform(rename={"course_code": "code"}) + fk = db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.on_delete == "CASCADE" + assert "ON DELETE CASCADE" in db["courses"].schema From 8443d7f3ba8a6762a1894afcadb342820130323e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:42:30 -0700 Subject: [PATCH 060/120] Resolve implicit primary key references in table.foreign_keys For foreign keys declared as "REFERENCES other_table" with no explicit columns, PRAGMA foreign_key_list returns None for the referenced column. The foreign_keys property now resolves these to the other table's primary key columns, so other_column=None unambiguously indicates a compound foreign key. Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 6 ++++++ tests/test_foreign_keys.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1910b84..dbb7445 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2075,6 +2075,12 @@ class Table(Queryable): other_table = rows[0][1] columns = [row[2] for row in rows] other_columns = [row[3] for row in rows] + if all(c is None for c in other_columns): + # "REFERENCES other_table" with no columns - the pragma + # returns None, meaning the other table's primary key + other_table_pks = self.db.table(other_table).pks + if len(other_table_pks) == len(columns): + other_columns = other_table_pks is_compound = len(rows) > 1 fks.append( ForeignKey( diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index afe333f..dc4ea68 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -408,3 +408,40 @@ def test_transform_preserves_compound_foreign_key_on_delete(): assert fk.is_compound is True assert fk.on_delete == "CASCADE" assert "ON DELETE CASCADE" in db["courses"].schema + + +def test_implicit_primary_key_reference_is_resolved(): + # REFERENCES authors (no column) has "to" of None in the pragma - + # it should be resolved to the primary key of the other table + db = Database(memory=True) + db.executescript(""" + CREATE TABLE authors (author_id INTEGER PRIMARY KEY); + CREATE TABLE books ( + id INTEGER PRIMARY KEY, + author_id INTEGER REFERENCES authors + ); + """) + fk = db["books"].foreign_keys[0] + assert fk.is_compound is False + assert fk.other_column == "author_id" + assert fk.other_columns == ["author_id"] + + +def test_implicit_compound_primary_key_reference_is_resolved(): + db = Database(memory=True) + db.executescript(""" + CREATE TABLE departments ( + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + PRIMARY KEY (campus_name, dept_code) + ); + CREATE TABLE courses ( + course_code TEXT PRIMARY KEY, + campus_name TEXT NOT NULL, + dept_code TEXT NOT NULL, + FOREIGN KEY (campus_name, dept_code) REFERENCES departments + ); + """) + fk = db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.other_columns == ["campus_name", "dept_code"] From 42c1dd0d5feeebf295bc3c64a9a4b71544edcabf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 14:45:08 -0700 Subject: [PATCH 061/120] Documentation for compound foreign key support, refs #594 - Changelog entries for all the compound foreign key work - Upgrading guide notes the transform() behavior changes - ForeignKey added to the API reference - Updated .foreign_keys introspection example output Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 13 +++++++++++++ docs/python-api.rst | 4 ++-- docs/reference.rst | 7 +++++++ docs/upgrading.rst | 4 +++- sqlite_utils/db.py | 3 +++ 5 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 01d59a0..b182d88 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,19 @@ Breaking changes: - ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` lists, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) +Compound foreign key support, everywhere: + +- Tables can now be created with compound foreign keys, by passing lists of column names in ``foreign_keys=``: ``foreign_keys=[(["campus_name", "dept_code"], "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. See :ref:`python_api_compound_foreign_keys`. +- ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely. +- ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept lists of column names to add a compound foreign key to an existing table. +- ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key. + +Other foreign key improvements: + +- ``ForeignKey`` now exposes ``on_delete`` and ``on_update`` fields reflecting the foreign key's ``ON DELETE``/``ON UPDATE`` actions, and ``table.transform()`` preserves those actions. Previously a transform silently stripped clauses such as ``ON DELETE CASCADE`` from the table schema. +- Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``. +- Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound. + .. _v4_0rc2: 4.0rc2 (2026-07-04) diff --git a/docs/python-api.rst b/docs/python-api.rst index f44a10b..9c31841 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2309,8 +2309,8 @@ Each ``ForeignKey`` has the following attributes: :: >>> db.table("Street_Tree_List").foreign_keys - [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=['qLegalStatus'], other_columns=['id'], is_compound=False), - ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=['qCareAssistant'], other_columns=['id'], is_compound=False), + [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=['qLegalStatus'], other_columns=['id'], is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), + ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=['qCareAssistant'], other_columns=['id'], is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), ...] Compound foreign keys - defined with ``FOREIGN KEY (col_a, col_b) REFERENCES other(col_a, col_b)`` - are returned as a single ``ForeignKey`` with ``is_compound=True``, ``column`` and ``other_column`` set to ``None``, and the participating columns available in the ``columns`` and ``other_columns`` lists. diff --git a/docs/reference.rst b/docs/reference.rst index 5b5fd25..a9fdf29 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -70,6 +70,13 @@ sqlite_utils.db.ColumnDetails .. autoclass:: sqlite_utils.db.ColumnDetails +.. _reference_db_other_foreign_key: + +sqlite_utils.db.ForeignKey +-------------------------- + +.. autoclass:: sqlite_utils.db.ForeignKey + sqlite_utils.utils ================== diff --git a/docs/upgrading.rst b/docs/upgrading.rst index eb997f6..67a6b5e 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -79,7 +79,7 @@ Python API changes **View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method. -**ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained three new fields - ``columns``, ``other_columns`` and ``is_compound`` - so that compound (multi-column) foreign keys can be represented as a single object. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead: +**ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained new fields - ``columns``, ``other_columns``, ``is_compound``, ``on_delete`` and ``on_update`` - so that compound (multi-column) foreign keys and foreign key actions can be represented. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead: .. code-block:: python @@ -95,6 +95,8 @@ Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` lists instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are single-item lists. +Two related behavior changes to ``table.transform()``: compound foreign keys now survive a transform (previously they were split into separate single-column keys), and ``ON DELETE``/``ON UPDATE`` actions such as ``ON DELETE CASCADE`` are now preserved (previously they were silently stripped from the schema). + **Validation errors raise ValueError.** Invalid arguments to Python API methods - for example ``create_table()`` with no columns, or ``ignore=True`` together with ``replace=True`` - now raise ``ValueError``. They previously raised ``AssertionError`` from bare ``assert`` statements, which were silently skipped under ``python -O``. **Transaction behavior is now well-defined.** 4.0 introduces the :ref:`db.atomic() ` context manager and uses it consistently for every write operation - the full model is described in :ref:`python_api_transactions`. Changes you may notice: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index dbb7445..009bafc 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -176,6 +176,9 @@ class ForeignKey: are ``None`` - use ``columns`` and ``other_columns`` instead, and check ``is_compound``. + ``on_delete`` and ``on_update`` hold the foreign key actions, e.g. + ``"CASCADE"`` - ``"NO ACTION"`` if not set. + Prior to sqlite-utils 4.0 this was a ``namedtuple`` and could be unpacked or indexed as ``(table, column, other_table, other_column)``. It is now a dataclass - access its fields by name instead. From c16edb2dc4a96490b69cdce2a5e8287ed3133924 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:03:23 -0700 Subject: [PATCH 062/120] Better type signatures for compound foreign key columns New ForeignKeyColumns type alias - a column name or a list/tuple of column names - used by add_foreign_key(), add_foreign_keys() (via the ForeignKeyTuple alias, replacing its Any slots) and the ForeignKeyIndicator union, which it also simplifies. Tuples now type-check anywhere lists are accepted. ForeignKey.__post_init__ normalizes tuple columns/other_columns to lists so direct construction with tuples compares equal to introspected foreign keys. Refs https://github.com/simonw/sqlite-utils/pull/770/changes#r3525703477 Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 33 ++++++++++++++++++++------------- tests/test_foreign_keys.py | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 009bafc..4e67a3c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -197,10 +197,15 @@ class ForeignKey: on_update: str = "NO ACTION" def __post_init__(self): - # Populate columns/other_columns for single-column foreign keys - if not self.columns: + # Populate columns/other_columns for single-column foreign keys, + # normalizing any tuples to lists + if self.columns: + self.columns = list(self.columns) + else: self.columns = [self.column] if self.column is not None else [] - if not self.other_columns: + if self.other_columns: + self.other_columns = list(self.other_columns) + else: self.other_columns = ( [self.other_column] if self.other_column is not None else [] ) @@ -228,16 +233,18 @@ class TransformError(Exception): pass +# A single column name, or a list/tuple of columns for a compound foreign key +ForeignKeyColumns = Union[str, List[str], Tuple[str, ...]] + +# (table, column(s), other_table, other_column(s)) +ForeignKeyTuple = Tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] + ForeignKeyIndicator = Union[ str, ForeignKey, - Tuple[str, str], - Tuple[str, str, str], - Tuple[str, str, str, str], - # Compound foreign keys use lists of columns: - Tuple[List[str], str], - Tuple[List[str], str, List[str]], - Tuple[str, List[str], str, List[str]], + Tuple[ForeignKeyColumns, str], + Tuple[ForeignKeyColumns, str, ForeignKeyColumns], + ForeignKeyTuple, ] ForeignKeysType = Union[Iterable[ForeignKeyIndicator], List[ForeignKeyIndicator]] @@ -1604,7 +1611,7 @@ class Database: return candidates def add_foreign_keys( - self, foreign_keys: Iterable[Union[ForeignKey, Tuple[str, Any, str, Any]]] + self, foreign_keys: Iterable[Union[ForeignKey, ForeignKeyTuple]] ) -> None: """ See :ref:`python_api_add_foreign_keys`. @@ -2911,9 +2918,9 @@ class Table(Queryable): def add_foreign_key( self, - column: Union[str, List[str]], + column: ForeignKeyColumns, other_table: Optional[str] = None, - other_column: Optional[Union[str, List[str]]] = None, + other_column: Optional[ForeignKeyColumns] = None, ignore: bool = False, ): """ diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index dc4ea68..54039b3 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -445,3 +445,19 @@ def test_implicit_compound_primary_key_reference_is_resolved(): fk = db["courses"].foreign_keys[0] assert fk.is_compound is True assert fk.other_columns == ["campus_name", "dept_code"] + + +def test_foreign_key_normalizes_tuple_columns_to_lists(): + # Compound columns passed as tuples are normalized to lists, so they + # compare equal to introspected ForeignKeys + fk = ForeignKey( + table="courses", + column=None, + other_table="departments", + other_column=None, + columns=("campus_name", "dept_code"), + other_columns=("campus_name", "dept_code"), + is_compound=True, + ) + assert fk.columns == ["campus_name", "dept_code"] + assert fk.other_columns == ["campus_name", "dept_code"] From d100264e9c11f73c2f86328846a33e500eeb2118 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:10:21 -0700 Subject: [PATCH 063/120] Normalize compound foreign key columns to tuples, promote tuples in docs ForeignKey.columns and .other_columns are now tuples rather than lists, both when introspected and when constructed directly (lists are normalized in __post_init__). Tuples are now the documented form for specifying compound foreign keys everywhere - in create_table() foreign_keys=, add_foreign_key() and drop_foreign_keys=: foreign_keys=[(("campus_name", "dept_code"), "departments")] Lists continue to work as input but are no longer documented. Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 8 ++-- docs/python-api.rst | 22 +++++------ docs/upgrading.rst | 2 +- sqlite_utils/db.py | 77 +++++++++++++++++++------------------- tests/test_foreign_keys.py | 73 +++++++++++++++++++----------------- 5 files changed, 92 insertions(+), 90 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b182d88..cacd5a0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,13 +11,13 @@ Unreleased Breaking changes: -- ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` lists, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) +- ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) -Compound foreign key support, everywhere: +Compound foreign key support: -- Tables can now be created with compound foreign keys, by passing lists of column names in ``foreign_keys=``: ``foreign_keys=[(["campus_name", "dept_code"], "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. See :ref:`python_api_compound_foreign_keys`. +- Tables can now be created with compound foreign keys, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. See :ref:`python_api_compound_foreign_keys`. - ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely. -- ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept lists of column names to add a compound foreign key to an existing table. +- ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept tuples of column names to add a compound foreign key to an existing table. - ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key. Other foreign key improvements: diff --git a/docs/python-api.rst b/docs/python-api.rst index 9c31841..9ff89eb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -822,7 +822,7 @@ You can leave off the third item in the tuple to have the referenced column auto Compound foreign keys ~~~~~~~~~~~~~~~~~~~~~ -To create a compound (multi-column) foreign key, use lists of column names in place of the single column names: +To create a compound (multi-column) foreign key, use tuples of column names in place of the single column names: .. code-block:: python @@ -831,7 +831,7 @@ To create a compound (multi-column) foreign key, use lists of column names in pl "campus_name": str, "dept_code": str, }, pk="course_code", foreign_keys=[ - (["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"]) + (("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")) ]) This creates a table-level constraint: @@ -845,12 +845,12 @@ This creates a table-level constraint: FOREIGN KEY ("campus_name", "dept_code") REFERENCES "departments"("campus_name", "dept_code") ) -As with single columns, you can leave off the list of other columns to reference the compound primary key of the other table: +As with single columns, you can leave off the tuple of other columns to reference the compound primary key of the other table: .. code-block:: python foreign_keys=[ - (["campus_name", "dept_code"], "departments") + (("campus_name", "dept_code"), "departments") ] To specify ``ON DELETE`` or ``ON UPDATE`` actions, pass ``ForeignKey`` objects instead: @@ -1581,12 +1581,12 @@ To ignore the case where the key already exists, use ``ignore=True``: db.table("books").add_foreign_key("author_id", "authors", "id", ignore=True) -To add a compound foreign key, pass lists of columns: +To add a compound foreign key, pass tuples of columns: .. code-block:: python db.table("courses").add_foreign_key( - ["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"] + ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code") ) As with single columns, omitting the other columns will use the compound primary key of the other table. ``other_table`` must always be specified for a compound foreign key. @@ -2294,9 +2294,9 @@ Each ``ForeignKey`` has the following attributes: ``other_column`` The referenced column, or ``None`` for a compound foreign key. ``columns`` - A list of the columns on this table, always populated (a single-item list for single-column foreign keys). + A tuple of the columns on this table, always populated (a one-item tuple for single-column foreign keys). ``other_columns`` - A list of the referenced columns. + A tuple of the referenced columns. ``is_compound`` ``True`` if this is a compound (multi-column) foreign key. ``on_delete`` @@ -2309,11 +2309,11 @@ Each ``ForeignKey`` has the following attributes: :: >>> db.table("Street_Tree_List").foreign_keys - [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=['qLegalStatus'], other_columns=['id'], is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), - ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=['qCareAssistant'], other_columns=['id'], is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), + [ForeignKey(table='Street_Tree_List', column='qLegalStatus', other_table='qLegalStatus', other_column='id', columns=('qLegalStatus',), other_columns=('id',), is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), + ForeignKey(table='Street_Tree_List', column='qCareAssistant', other_table='qCareAssistant', other_column='id', columns=('qCareAssistant',), other_columns=('id',), is_compound=False, on_delete='NO ACTION', on_update='NO ACTION'), ...] -Compound foreign keys - defined with ``FOREIGN KEY (col_a, col_b) REFERENCES other(col_a, col_b)`` - are returned as a single ``ForeignKey`` with ``is_compound=True``, ``column`` and ``other_column`` set to ``None``, and the participating columns available in the ``columns`` and ``other_columns`` lists. +Compound foreign keys - defined with ``FOREIGN KEY (col_a, col_b) REFERENCES other(col_a, col_b)`` - are returned as a single ``ForeignKey`` with ``is_compound=True``, ``column`` and ``other_column`` set to ``None``, and the participating columns available in the ``columns`` and ``other_columns`` tuples. .. _python_api_introspection_schema: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 67a6b5e..e7fd273 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -93,7 +93,7 @@ Python API changes Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. -Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` lists instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are single-item lists. +Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` tuples instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are one-item tuples. Two related behavior changes to ``table.transform()``: compound foreign keys now survive a transform (previously they were split into separate single-column keys), and ``ON DELETE``/``ON UPDATE`` actions such as ``ON DELETE CASCADE`` are now preserved (previously they were silently stripped from the schema). diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 4e67a3c..a739486 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -170,7 +170,7 @@ class ForeignKey: A foreign key defined on a table. For single-column foreign keys ``column`` and ``other_column`` hold the - column names, and ``columns``/``other_columns`` are single-item lists. + column names, and ``columns``/``other_columns`` are one-item tuples. For compound (multi-column) foreign keys ``column`` and ``other_column`` are ``None`` - use ``columns`` and ``other_columns`` instead, and check @@ -190,24 +190,24 @@ class ForeignKey: column: Optional[str] = field(compare=False) other_table: str other_column: Optional[str] = field(compare=False) - columns: List[str] = field(default_factory=list) - other_columns: List[str] = field(default_factory=list) + columns: Tuple[str, ...] = () + other_columns: Tuple[str, ...] = () is_compound: bool = False on_delete: str = "NO ACTION" on_update: str = "NO ACTION" def __post_init__(self): # Populate columns/other_columns for single-column foreign keys, - # normalizing any tuples to lists + # normalizing any lists to tuples if self.columns: - self.columns = list(self.columns) + self.columns = tuple(self.columns) else: - self.columns = [self.column] if self.column is not None else [] + self.columns = (self.column,) if self.column is not None else () if self.other_columns: - self.other_columns = list(self.other_columns) + self.other_columns = tuple(self.other_columns) else: self.other_columns = ( - [self.other_column] if self.other_column is not None else [] + (self.other_column,) if self.other_column is not None else () ) @@ -233,8 +233,8 @@ class TransformError(Exception): pass -# A single column name, or a list/tuple of columns for a compound foreign key -ForeignKeyColumns = Union[str, List[str], Tuple[str, ...]] +# A single column name, or a tuple of columns for a compound foreign key +ForeignKeyColumns = Union[str, Tuple[str, ...], List[str]] # (table, column(s), other_table, other_column(s)) ForeignKeyTuple = Tuple[str, ForeignKeyColumns, str, ForeignKeyColumns] @@ -1171,9 +1171,9 @@ class Database: string, a ForeignKey() object, a tuple of (column, other_table), or a tuple of (column, other_table, other_column), or a tuple of (table, column, other_table, other_column). For compound foreign - keys the column elements can be lists of column names, e.g. - (["campus_name", "dept_code"], "departments") or - (["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"]) + keys the column elements can be tuples of column names, e.g. + (("campus_name", "dept_code"), "departments") or + (("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")) """ table = self.table(name) if all(isinstance(fk, ForeignKey) for fk in foreign_keys): @@ -1207,18 +1207,17 @@ class Database: other_table = tuple_or_list[1] if isinstance(column_or_columns, (list, tuple)): # Compound foreign key - columns = list(column_or_columns) + columns = tuple(column_or_columns) if len(tuple_or_list) == 3: - other_columns = tuple_or_list[2] - if not isinstance(other_columns, (list, tuple)): + if not isinstance(tuple_or_list[2], (list, tuple)): raise ValueError( - "Compound foreign key {} should reference a list " + "Compound foreign key {} should reference a tuple " "of other columns".format(tuple(tuple_or_list)) ) - other_columns = list(other_columns) + other_columns = tuple(tuple_or_list[2]) else: # Guess the compound primary key of the other table - other_columns = self.table(other_table).pks + other_columns = tuple(self.table(other_table).pks) if len(columns) != len(other_columns): raise ValueError( "Compound foreign key {} should have the same number " @@ -1618,7 +1617,7 @@ class Database: :param foreign_keys: A list of ``(table, column, other_table, other_column)`` tuples - for compound foreign keys, ``column`` and ``other_column`` can - be lists of column names + be tuples of column names """ # foreign_keys is a list of explicit 4-tuples if not all( @@ -1644,16 +1643,16 @@ class Database: ) else: table, column_or_columns, other_table, other_column_or_columns = fk - # Compound foreign keys use lists of columns + # Compound foreign keys use tuples of columns columns = ( - [column_or_columns] + (column_or_columns,) if isinstance(column_or_columns, str) - else list(column_or_columns) + else tuple(column_or_columns) ) other_columns = ( - [other_column_or_columns] + (other_column_or_columns,) if isinstance(other_column_or_columns, str) - else list(other_column_or_columns) + else tuple(other_column_or_columns) ) if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) @@ -1708,9 +1707,9 @@ class Database: existing_indexes = {tuple(i.columns) for i in table.indexes} for fk in table.foreign_keys: # A compound foreign key gets a single composite index - if tuple(fk.columns) not in existing_indexes: + if fk.columns not in existing_indexes: table.create_index(fk.columns, find_unique_name=True) - existing_indexes.add(tuple(fk.columns)) + existing_indexes.add(fk.columns) def vacuum(self) -> None: "Run a SQLite ``VACUUM`` against the database." @@ -2083,12 +2082,12 @@ class Table(Queryable): for id in sorted(by_id): rows = sorted(by_id[id]) # order columns by seq other_table = rows[0][1] - columns = [row[2] for row in rows] - other_columns = [row[3] for row in rows] + columns = tuple(row[2] for row in rows) + other_columns = tuple(row[3] for row in rows) if all(c is None for c in other_columns): # "REFERENCES other_table" with no columns - the pragma # returns None, meaning the other table's primary key - other_table_pks = self.db.table(other_table).pks + other_table_pks = tuple(self.db.table(other_table).pks) if len(other_table_pks) == len(columns): other_columns = other_table_pks is_compound = len(rows) > 1 @@ -2448,14 +2447,14 @@ class Table(Queryable): # A column name matches any foreign key it participates in if spec in fk.columns: return True - elif list(spec) == fk.columns: + elif tuple(spec) == fk.columns: # A tuple/list must match a compound key's columns exactly return True # Dropping any of a foreign key's columns drops the whole key return any(column in drop for column in fk.columns) def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: - columns = [rename.get(column) or column for column in fk.columns] + columns = tuple(rename.get(column) or column for column in fk.columns) if fk.is_compound: return ForeignKey( self.name, @@ -2926,14 +2925,14 @@ class Table(Queryable): """ Alter the schema to mark the specified column as a foreign key to another table. - :param column: The column to mark as a foreign key - use a list of columns + :param column: The column to mark as a foreign key - use a tuple of columns for a compound foreign key. :param other_table: The table it refers to - if omitted, will be guessed based on the column name. :param other_column: The column on the other table it - if omitted, will be guessed. - Use a list of columns for a compound foreign key. + Use a tuple of columns for a compound foreign key. :param ignore: Set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. """ - columns = [column] if isinstance(column, str) else list(column) + columns = (column,) if isinstance(column, str) else tuple(column) # Ensure columns exist for col in columns: if col not in self.columns_dict: @@ -2948,13 +2947,13 @@ class Table(Queryable): # If other_column is not specified, detect the primary key on other_table if other_column is None: if len(columns) > 1: - other_columns = self.db.table(other_table).pks + other_columns = tuple(self.db.table(other_table).pks) else: - other_columns = [self.guess_foreign_column(other_table)] + other_columns = (self.guess_foreign_column(other_table),) elif isinstance(other_column, str): - other_columns = [other_column] + other_columns = (other_column,) else: - other_columns = list(other_column) + other_columns = tuple(other_column) if len(columns) != len(other_columns): raise ValueError( "Compound foreign key must have the same number of columns " diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 54039b3..d1bc498 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -36,8 +36,8 @@ def test_compound_foreign_key(compound_db): assert fk.is_compound is True assert fk.table == "courses" assert fk.other_table == "departments" - assert fk.columns == ["campus_name", "dept_code"] - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") + assert fk.other_columns == ("campus_name", "dept_code") # Scalar column/other_column can't sensibly hold a compound key assert fk.column is None assert fk.other_column is None @@ -51,8 +51,8 @@ def test_single_foreign_key_gets_columns_fields(fresh_db): assert fk.is_compound is False assert fk.column == "author_id" assert fk.other_column == "id" - assert fk.columns == ["author_id"] - assert fk.other_columns == ["id"] + assert fk.columns == ("author_id",) + assert fk.other_columns == ("id",) def test_foreign_key_no_longer_unpacks_as_tuple(fresh_db): @@ -142,14 +142,16 @@ EXPECTED_COURSES_SCHEMA = ( column=None, other_table="departments", other_column=None, - columns=["campus_name", "dept_code"], - other_columns=["campus_name", "dept_code"], + columns=("campus_name", "dept_code"), + other_columns=("campus_name", "dept_code"), is_compound=True, ) ], - [(["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"])], + [(("campus_name", "dept_code"), "departments", ("campus_name", "dept_code"))], # Two-item form guesses the other table's primary key: - [(["campus_name", "dept_code"], "departments")], + [(("campus_name", "dept_code"), "departments")], + # Lists work too, though tuples are the documented form: + [(["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"])], ), ) def test_create_table_with_compound_foreign_key(departments_db, foreign_keys): @@ -164,9 +166,9 @@ def test_create_table_with_compound_foreign_key(departments_db, foreign_keys): assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True - assert fk.columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") assert fk.other_table == "departments" - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") def test_create_table_compound_foreign_key_enforced(departments_db): @@ -175,7 +177,7 @@ def test_create_table_compound_foreign_key_enforced(departments_db): "courses", {"course_code": str, "campus_name": str, "dept_code": str}, pk="course_code", - foreign_keys=[(["campus_name", "dept_code"], "departments")], + foreign_keys=[(("campus_name", "dept_code"), "departments")], ) departments_db["departments"].insert( {"campus_name": "Berkeley", "dept_code": "CS", "dept_name": "Computer Science"} @@ -199,7 +201,7 @@ def test_create_table_compound_foreign_key_missing_other_column(departments_db): {"course_code": str, "campus_name": str, "dept_code": str}, pk="course_code", foreign_keys=[ - (["campus_name", "dept_code"], "departments", ["campus_name", "nope"]) + (("campus_name", "dept_code"), "departments", ("campus_name", "nope")) ], ) @@ -210,9 +212,9 @@ def test_transform_preserves_compound_foreign_key(compound_db): assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True - assert fk.columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") assert fk.other_table == "departments" - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") def test_transform_rename_member_column_updates_compound_foreign_key(compound_db): @@ -221,9 +223,9 @@ def test_transform_rename_member_column_updates_compound_foreign_key(compound_db assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True - assert fk.columns == ["campus", "dept_code"] + assert fk.columns == ("campus", "dept_code") # Referenced columns in the other table are unchanged - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") def test_transform_drop_member_column_drops_compound_foreign_key(compound_db): @@ -264,7 +266,7 @@ def courses_db(departments_db): def test_add_compound_foreign_key(courses_db): t = courses_db["courses"].add_foreign_key( - ["campus_name", "dept_code"], "departments", ["campus_name", "dept_code"] + ("campus_name", "dept_code"), "departments", ("campus_name", "dept_code") ) # Returns self assert t.name == "courses" @@ -272,33 +274,34 @@ def test_add_compound_foreign_key(courses_db): assert len(fks) == 1 fk = fks[0] assert fk.is_compound is True - assert fk.columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") assert fk.other_table == "departments" - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") def test_add_compound_foreign_key_guesses_other_columns(courses_db): + # Lists work here too, though tuples are the documented form courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") fk = courses_db["courses"].foreign_keys[0] - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") def test_add_compound_foreign_key_error_if_already_exists(courses_db): - courses_db["courses"].add_foreign_key(["campus_name", "dept_code"], "departments") + courses_db["courses"].add_foreign_key(("campus_name", "dept_code"), "departments") with pytest.raises(AlterError) as ex: courses_db["courses"].add_foreign_key( - ["campus_name", "dept_code"], "departments" + ("campus_name", "dept_code"), "departments" ) assert "already exists" in ex.value.args[0] # ignore=True should not raise courses_db["courses"].add_foreign_key( - ["campus_name", "dept_code"], "departments", ignore=True + ("campus_name", "dept_code"), "departments", ignore=True ) def test_add_compound_foreign_key_error_if_column_missing(courses_db): with pytest.raises(AlterError): - courses_db["courses"].add_foreign_key(["campus_name", "nope"], "departments") + courses_db["courses"].add_foreign_key(("campus_name", "nope"), "departments") def test_db_add_foreign_keys_compound(courses_db): @@ -306,15 +309,15 @@ def test_db_add_foreign_keys_compound(courses_db): [ ( "courses", - ["campus_name", "dept_code"], + ("campus_name", "dept_code"), "departments", - ["campus_name", "dept_code"], + ("campus_name", "dept_code"), ) ] ) fk = courses_db["courses"].foreign_keys[0] assert fk.is_compound is True - assert fk.columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") def test_index_foreign_keys_compound_creates_composite_index(compound_db): @@ -424,7 +427,7 @@ def test_implicit_primary_key_reference_is_resolved(): fk = db["books"].foreign_keys[0] assert fk.is_compound is False assert fk.other_column == "author_id" - assert fk.other_columns == ["author_id"] + assert fk.other_columns == ("author_id",) def test_implicit_compound_primary_key_reference_is_resolved(): @@ -444,20 +447,20 @@ def test_implicit_compound_primary_key_reference_is_resolved(): """) fk = db["courses"].foreign_keys[0] assert fk.is_compound is True - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.other_columns == ("campus_name", "dept_code") -def test_foreign_key_normalizes_tuple_columns_to_lists(): - # Compound columns passed as tuples are normalized to lists, so they +def test_foreign_key_normalizes_list_columns_to_tuples(): + # Compound columns passed as lists are normalized to tuples, so they # compare equal to introspected ForeignKeys fk = ForeignKey( table="courses", column=None, other_table="departments", other_column=None, - columns=("campus_name", "dept_code"), - other_columns=("campus_name", "dept_code"), + columns=["campus_name", "dept_code"], + other_columns=["campus_name", "dept_code"], is_compound=True, ) - assert fk.columns == ["campus_name", "dept_code"] - assert fk.other_columns == ["campus_name", "dept_code"] + assert fk.columns == ("campus_name", "dept_code") + assert fk.other_columns == ("campus_name", "dept_code") From 8dfbfa80b8282c14ed9732eff04ba07dbd5388e5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:36:46 -0700 Subject: [PATCH 064/120] Fix ty errors in transform_sql foreign key closures The fk_should_be_dropped and fk_with_renamed_columns closures captured the drop and rename parameters, but type checkers do not narrow captured variables inside nested functions so ty saw their Optional declared types. Bind the already-normalized values to fresh names for the closures to use. Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index a739486..1c06b62 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2439,6 +2439,11 @@ class Table(Queryable): create_table_foreign_keys.extend(foreign_keys) else: # Construct foreign_keys from current, plus add_foreign_keys, minus drop_foreign_keys + # Bind fresh names here - type checkers do not narrow captured + # variables inside nested functions, so the closures would + # otherwise see the Optional declared types of drop and rename + dropped_columns = drop + renamed_columns = rename def fk_should_be_dropped(fk: ForeignKey) -> bool: if drop_foreign_keys is not None: @@ -2451,10 +2456,12 @@ class Table(Queryable): # A tuple/list must match a compound key's columns exactly return True # Dropping any of a foreign key's columns drops the whole key - return any(column in drop for column in fk.columns) + return any(column in dropped_columns for column in fk.columns) def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: - columns = tuple(rename.get(column) or column for column in fk.columns) + columns = tuple( + renamed_columns.get(column) or column for column in fk.columns + ) if fk.is_compound: return ForeignKey( self.name, From 2f599fc7c6aeede4e97bd91487e5de5f5e190dca Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:38:05 -0700 Subject: [PATCH 065/120] Run ty in just lint, matching CI Co-Authored-By: Claude Fable 5 --- Justfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index f4f0e7f..5caa120 100644 --- a/Justfile +++ b/Justfile @@ -8,11 +8,12 @@ @run *options: uv run -- {{options}} -# Run linters: black, flake8, mypy, cog +# Run linters: black, flake8, mypy, ty, cog @lint: just run black . --check uv run flake8 uv run mypy sqlite_utils tests + uv run ty check sqlite_utils uv run cog --check README.md docs/*.rst uv run --group docs codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt From 5b61530965df0d56a09e71f29cc970b4ea3e43d4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:42:41 -0700 Subject: [PATCH 066/120] db.add_foreign_keys() no longer drops ON DELETE/ON UPDATE actions ForeignKey objects passed to db.add_foreign_keys() were flattened to plain (table, column, other_table, other_column) tuples before being handed to transform(), silently discarding their on_delete and on_update actions. The method now carries ForeignKey objects through to transform() intact - tuple inputs are converted to ForeignKey objects up front, which also simplifies the validation loop. Co-Authored-By: Claude Fable 5 --- sqlite_utils/db.py | 42 ++++++++++++++++++++++---------------- tests/test_foreign_keys.py | 35 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1c06b62..7379859 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1630,17 +1630,12 @@ class Database: "(table, column, other_table, other_column)" ) - foreign_keys_to_create: List[Tuple[str, Any, str, Any]] = [] + foreign_keys_to_create: List[ForeignKey] = [] # Verify that all tables and columns exist for fk in foreign_keys: if isinstance(fk, ForeignKey): - table, columns, other_table, other_columns = ( - fk.table, - fk.columns, - fk.other_table, - fk.other_columns, - ) + fk_object = fk else: table, column_or_columns, other_table, other_column_or_columns = fk # Compound foreign keys use tuples of columns @@ -1654,6 +1649,24 @@ class Database: if isinstance(other_column_or_columns, str) else tuple(other_column_or_columns) ) + if len(columns) == 1: + fk_object = ForeignKey( + table, columns[0], other_table, other_columns[0] + ) + else: + fk_object = ForeignKey( + table, + None, + other_table, + None, + columns=columns, + other_columns=other_columns, + is_compound=True, + ) + table = fk_object.table + columns = fk_object.columns + other_table = fk_object.other_table + other_columns = fk_object.other_columns if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) table_obj = self.table(table) @@ -1680,19 +1693,12 @@ class Database: and fk.other_table == other_table and fk.other_columns == other_columns ): - if len(columns) == 1: - foreign_keys_to_create.append( - (table, columns[0], other_table, other_columns[0]) - ) - else: - foreign_keys_to_create.append( - (table, columns, other_table, other_columns) - ) + foreign_keys_to_create.append(fk_object) # Group them by table - by_table: Dict[str, List] = {} - for fk in foreign_keys_to_create: - by_table.setdefault(fk[0], []).append(fk) + by_table: Dict[str, List[ForeignKey]] = {} + for fk_object in foreign_keys_to_create: + by_table.setdefault(fk_object.table, []).append(fk_object) for table, fks in by_table.items(): self.table(table).transform(add_foreign_keys=fks) diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index d1bc498..b22638a 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -464,3 +464,38 @@ def test_foreign_key_normalizes_list_columns_to_tuples(): ) assert fk.columns == ("campus_name", "dept_code") assert fk.other_columns == ("campus_name", "dept_code") + + +def test_add_foreign_keys_preserves_actions(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/594 review finding: + # ForeignKey objects passed to db.add_foreign_keys() were flattened + # to plain tuples, losing on_delete/on_update + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db.add_foreign_keys( + [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] + ) + fk = fresh_db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert "ON DELETE CASCADE" in fresh_db["books"].schema + + +def test_add_foreign_keys_preserves_actions_compound(courses_db): + courses_db.add_foreign_keys( + [ + ForeignKey( + table="courses", + column=None, + other_table="departments", + other_column=None, + columns=("campus_name", "dept_code"), + other_columns=("campus_name", "dept_code"), + is_compound=True, + on_delete="CASCADE", + ) + ] + ) + fk = courses_db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.on_delete == "CASCADE" + assert "ON DELETE CASCADE" in courses_db["courses"].schema From 658185d297a307d0b9d43fd83019a5acab032f18 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:46:16 -0700 Subject: [PATCH 067/120] Fix compound foreign key test failure against sqlean test_create_table_compound_foreign_key_enforced caught the stdlib sqlite3.IntegrityError, but sqlean's IntegrityError is not a subclass of it. Import sqlite3 from sqlite_utils.utils like the other test modules, so the right exception is used whichever backend is installed. Co-Authored-By: Claude Fable 5 --- tests/test_foreign_keys.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index b22638a..7fcd788 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -3,6 +3,7 @@ import pytest from sqlite_utils import Database from sqlite_utils.db import AlterError, ForeignKey +from sqlite_utils.utils import sqlite3 COMPOUND_SCHEMA = """ CREATE TABLE departments ( @@ -185,8 +186,6 @@ def test_create_table_compound_foreign_key_enforced(departments_db): departments_db["courses"].insert( {"course_code": "CS101", "campus_name": "Berkeley", "dept_code": "CS"} ) - import sqlite3 - with pytest.raises(sqlite3.IntegrityError): departments_db.execute( "insert into courses (course_code, campus_name, dept_code) " From 0ec01804055d7a09acc6c4068562c8bbb05dfdd7 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 15:49:20 -0700 Subject: [PATCH 068/120] add_foreign_key() on_delete= and on_update= parameters, closes #530 table.add_foreign_key() now accepts on_delete= and on_update= to create foreign keys with ON DELETE/ON UPDATE actions: table.add_foreign_key("author_id", "authors", "id", on_delete="CASCADE") Works for compound foreign keys too. Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/python-api.rst | 10 ++++++++++ sqlite_utils/db.py | 27 ++++++++++++++++++++++++--- tests/test_foreign_keys.py | 26 ++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cacd5a0..f6a8ea2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -23,6 +23,7 @@ Compound foreign key support: Other foreign key improvements: - ``ForeignKey`` now exposes ``on_delete`` and ``on_update`` fields reflecting the foreign key's ``ON DELETE``/``ON UPDATE`` actions, and ``table.transform()`` preserves those actions. Previously a transform silently stripped clauses such as ``ON DELETE CASCADE`` from the table schema. +- ``table.add_foreign_key()`` accepts new ``on_delete=`` and ``on_update=`` parameters for creating foreign keys with actions, e.g. ``table.add_foreign_key("author_id", "authors", "id", on_delete="CASCADE")``. (:issue:`530`) - Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``. - Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound. diff --git a/docs/python-api.rst b/docs/python-api.rst index 9ff89eb..a33b256 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1591,6 +1591,16 @@ To add a compound foreign key, pass tuples of columns: As with single columns, omitting the other columns will use the compound primary key of the other table. ``other_table`` must always be specified for a compound foreign key. +Use ``on_delete=`` and ``on_update=`` to specify ``ON DELETE`` and ``ON UPDATE`` actions for the foreign key: + +.. code-block:: python + + db.table("books").add_foreign_key( + "author_id", "authors", "id", on_delete="CASCADE" + ) + +This creates a foreign key with an ``ON DELETE CASCADE`` clause, so deleting an author will also delete their books (provided foreign key enforcement is enabled with ``PRAGMA foreign_keys = ON``). Valid actions are ``"SET NULL"``, ``"SET DEFAULT"``, ``"CASCADE"``, ``"RESTRICT"`` and the default ``"NO ACTION"``. + .. _python_api_add_foreign_keys: Adding multiple foreign key constraints at once diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 7379859..3f23c7d 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2934,6 +2934,8 @@ class Table(Queryable): other_table: Optional[str] = None, other_column: Optional[ForeignKeyColumns] = None, ignore: bool = False, + on_delete: str = "NO ACTION", + on_update: str = "NO ACTION", ): """ Alter the schema to mark the specified column as a foreign key to another table. @@ -2944,6 +2946,9 @@ class Table(Queryable): :param other_column: The column on the other table it - if omitted, will be guessed. Use a tuple of columns for a compound foreign key. :param ignore: Set this to ``True`` to ignore an existing foreign key - otherwise a ``AlterError`` will be raised. + :param on_delete: ``ON DELETE`` action for the foreign key, e.g. ``"CASCADE"`` + or ``"SET NULL"``. + :param on_update: ``ON UPDATE`` action for the foreign key. """ columns = (column,) if isinstance(column, str) else tuple(column) # Ensure columns exist @@ -2997,11 +3002,27 @@ class Table(Queryable): ) ) if len(columns) == 1: - self.db.add_foreign_keys( - [(self.name, columns[0], other_table, other_columns[0])] + fk_object = ForeignKey( + self.name, + columns[0], + other_table, + other_columns[0], + on_delete=on_delete, + on_update=on_update, ) else: - self.db.add_foreign_keys([(self.name, columns, other_table, other_columns)]) + fk_object = ForeignKey( + self.name, + None, + other_table, + None, + columns=columns, + other_columns=other_columns, + is_compound=True, + on_delete=on_delete, + on_update=on_update, + ) + self.db.add_foreign_keys([fk_object]) return self def enable_counts(self) -> None: diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 7fcd788..8950189 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -498,3 +498,29 @@ def test_add_foreign_keys_preserves_actions_compound(courses_db): assert fk.is_compound is True assert fk.on_delete == "CASCADE" assert "ON DELETE CASCADE" in courses_db["courses"].schema + + +def test_add_foreign_key_on_delete_on_update(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db["books"].add_foreign_key( + "author_id", "authors", "id", on_delete="CASCADE", on_update="RESTRICT" + ) + fk = fresh_db["books"].foreign_keys[0] + assert fk.on_delete == "CASCADE" + assert fk.on_update == "RESTRICT" + assert "ON UPDATE RESTRICT ON DELETE CASCADE" in fresh_db["books"].schema + # The cascade should actually fire + fresh_db.execute("PRAGMA foreign_keys = ON") + fresh_db.execute("delete from authors where id = 1") + assert fresh_db["books"].count == 0 + + +def test_add_compound_foreign_key_on_delete(courses_db): + courses_db["courses"].add_foreign_key( + ("campus_name", "dept_code"), "departments", on_delete="SET NULL" + ) + fk = courses_db["courses"].foreign_keys[0] + assert fk.is_compound is True + assert fk.on_delete == "SET NULL" + assert "ON DELETE SET NULL" in courses_db["courses"].schema From afbfd95273c51d6a14174e9e07fb63ed2a34f72a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 16:00:52 -0700 Subject: [PATCH 069/120] Remove sqlean.py support (#772) Closes #771, refs #769 --- .github/workflows/test.yml | 3 --- docs/changelog.rst | 1 + docs/installation.rst | 10 +++++----- docs/python-api.rst | 2 +- mypy.ini | 3 --- sqlite_utils/utils.py | 11 +++-------- tests/test_gis.py | 8 -------- 7 files changed, 10 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7e1e953..d85cbf8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,9 +31,6 @@ jobs: - name: Install SpatiaLite if: matrix.os == 'ubuntu-latest' run: sudo apt-get install libsqlite3-mod-spatialite - - name: On macOS with Python 3.10 test with sqlean.py - if: matrix.os == 'macos-latest' && matrix.python-version == '3.10' - run: pip install sqlean.py sqlite-dump - name: Build extension for --load-extension test if: matrix.os == 'ubuntu-latest' run: |- diff --git a/docs/changelog.rst b/docs/changelog.rst index f6a8ea2..7efeb95 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ Unreleased Breaking changes: - ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) +- Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library. Compound foreign key support: diff --git a/docs/installation.rst b/docs/installation.rst index beb6d4a..1333f5d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -52,15 +52,15 @@ On some platforms the ability to load additional extensions (via ``conn.load_ext You may also see the error ``sqlite3.OperationalError: table sqlite_master may not be modified`` when trying to alter an existing table. -You can work around these limitations by installing either the `pysqlite3 `__ package or the `sqlean.py `__ package, both of which provide drop-in replacements for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions. +You can work around these limitations by installing the `pysqlite3 `__ package, which provides a drop-in replacement for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions. -To install ``sqlean.py`` (which has compiled binary wheels available for all major platforms) run the following: +To install ``pysqlite3`` run the following: .. code-block:: bash - sqlite-utils install sqlean.py + sqlite-utils install pysqlite3 -``pysqlite3`` and ``sqlean.py`` do not provide implementations of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package: +``pysqlite3`` does not provide an implementation of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package: .. code-block:: bash @@ -87,4 +87,4 @@ For ``zsh``: Add this code to ``~/.zshrc`` or ``~/.bashrc`` to automatically run it when you start a new shell. -See `the Click documentation `__ for more details. \ No newline at end of file +See `the Click documentation `__ for more details. diff --git a/docs/python-api.rst b/docs/python-api.rst index a33b256..6723efb 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2184,7 +2184,7 @@ The ``db.iterdump()`` method returns a sequence of SQL strings representing a co This uses the `sqlite3.Connection.iterdump() `__ method. -If you are using ``pysqlite3`` or ``sqlean.py`` the underlying method may be missing. If you install the `sqlite-dump `__ package then the ``db.iterdump()`` method will use that implementation instead: +If you are using ``pysqlite3`` the underlying method may be missing. If you install the `sqlite-dump `__ package then the ``db.iterdump()`` method will use that implementation instead: .. code-block:: bash diff --git a/mypy.ini b/mypy.ini index de0dc83..2f6a875 100644 --- a/mypy.ini +++ b/mypy.ini @@ -16,9 +16,6 @@ ignore_errors = True [mypy-pysqlite3.*] ignore_missing_imports = True -[mypy-sqlean.*] -ignore_missing_imports = True - [mypy-sqlite_dump.*] ignore_missing_imports = True diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 0ca98fc..865ee79 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -43,15 +43,10 @@ else: dbapi2 = importlib.import_module("pysqlite3.dbapi2") OperationalError = dbapi2.OperationalError except ImportError: - try: - sqlite3 = importlib.import_module("sqlean") - dbapi2 = importlib.import_module("sqlean.dbapi2") - OperationalError = dbapi2.OperationalError - except ImportError: - import sqlite3 # noqa: F401 - from sqlite3 import dbapi2 # noqa: F401 + import sqlite3 # noqa: F401 + from sqlite3 import dbapi2 # noqa: F401 - OperationalError = dbapi2.OperationalError + OperationalError = dbapi2.OperationalError SPATIALITE_PATHS = ( diff --git a/tests/test_gis.py b/tests/test_gis.py index 1b5ed70..e8f4c08 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -6,11 +6,6 @@ from sqlite_utils.cli import cli from sqlite_utils.db import Database from sqlite_utils.utils import find_spatialite, sqlite3 -try: - import sqlean # type: ignore[import-not-found] -except ImportError: - sqlean = None - pytestmark = [ pytest.mark.skipif( @@ -20,9 +15,6 @@ pytestmark = [ not hasattr(sqlite3.Connection, "enable_load_extension"), reason="sqlite3.Connection missing enable_load_extension", ), - pytest.mark.skipif( - sqlean is not None, reason="sqlean.py is not compatible with SpatiaLite" - ), ] From a00ed60efc3a242de38f666bc4ce1e431f86dcea Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 21:17:41 -0700 Subject: [PATCH 070/120] Apply Black to tests/test_gis.py Fixes formatting left behind by afbfd95, which was merged while the Black check on its pull request was still failing. Co-Authored-By: Claude Fable 5 --- tests/test_gis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_gis.py b/tests/test_gis.py index e8f4c08..f39554e 100644 --- a/tests/test_gis.py +++ b/tests/test_gis.py @@ -6,7 +6,6 @@ from sqlite_utils.cli import cli from sqlite_utils.db import Database from sqlite_utils.utils import find_spatialite, sqlite3 - pytestmark = [ pytest.mark.skipif( not find_spatialite(), reason="Could not find SpatiaLite extension" From 07b603e562af19f80c3d00eb59b9cf29331127ce Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 21:20:39 -0700 Subject: [PATCH 071/120] Preserve duplicate column names in query results Queries returning duplicate column names - e.g. joins between tables sharing column names - silently lost values because rows were built with dict(zip(keys, row)), where the last duplicate wins. Later occurrences are now renamed with a numeric suffix: id, id becomes id, id_2 - skipping any suffix that would collide with a real column in the same query. The new utils.dedupe_keys() helper transforms the key list once per query, so the per-row dict construction is unchanged and there is no measurable performance impact. Applied in Database.query() (including the PRAGMA and RETURNING paths), Table.rows_where(), Table.search() and the CLI's JSON output. CSV, TSV and table output keep the original duplicate headers. Closes #624 --- docs/cli.rst | 2 ++ docs/python-api.rst | 11 +++++++++++ sqlite_utils/cli.py | 5 +++++ sqlite_utils/db.py | 9 +++++---- sqlite_utils/utils.py | 31 +++++++++++++++++++++++++++++++ tests/test_cli.py | 20 ++++++++++++++++++++ tests/test_fts.py | 14 ++++++++++++++ tests/test_query.py | 16 ++++++++++++++++ tests/test_rows.py | 7 +++++++ tests/test_utils.py | 17 +++++++++++++++++ 10 files changed, 128 insertions(+), 4 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index c9389d8..84a65d9 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -45,6 +45,8 @@ The default format returned for queries is JSON: [{"id": 1, "age": 4, "name": "Cleo"}, {"id": 2, "age": 2, "name": "Pancakes"}] +If the query returns more than one column with the same name, later occurrences are renamed with a numeric suffix - ``select 1 as id, 2 as id`` returns ``[{"id": 1, "id_2": 2}]``. This only applies to JSON output: :ref:`CSV and TSV ` and :ref:`table ` output keep the duplicate column headers unchanged. + .. _cli_query_nl: Newline-delimited JSON diff --git a/docs/python-api.rst b/docs/python-api.rst index 6723efb..7ac3951 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -233,6 +233,17 @@ The SQL query is executed as soon as ``db.query()`` is called. The resulting row ``db.query()`` can only be used with SQL that returns rows. Passing a statement that returns no rows - an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause, for example - will raise a ``ValueError``. The rejected statement is rolled back, so it has no effect on the database. Use :ref:`db.execute() ` for those statements instead. +If a query returns more than one column with the same name - a join between two tables that share column names, for example - later occurrences are renamed with a numeric suffix, so every value is included in the dictionary: + +.. code-block:: python + + row = next(db.query("select 1 as id, 2 as id, 3 as id")) + print(row) + # Outputs: + # {'id': 1, 'id_2': 2, 'id_3': 3} + +A suffix that would collide with another column in the query is skipped - ``select 1 as id, 2 as id, 3 as id_2`` returns ``{'id': 1, 'id_3': 2, 'id_2': 3}``. The same renaming is applied by ``table.rows_where()`` and ``table.search()``. + .. _python_api_execute: db.execute(sql, params) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index a30f79a..d5a4b1e 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -34,6 +34,7 @@ from .utils import ( OperationalError, _compile_code, chunks, + dedupe_keys, file_progress, find_spatialite, flatten as _flatten, @@ -3493,6 +3494,10 @@ FILE_COLUMNS = { def output_rows(iterator, headers, nl, arrays, json_cols): + # Duplicate column names would collide as dictionary keys, so rename + # later occurrences id, id -> id, id_2 - CSV and table output keep + # the original duplicate headers since they never build dictionaries + headers = dedupe_keys(headers) # We have to iterate two-at-a-time so we can know if we # should output a trailing comma or if we have reached # the last row. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3f23c7d..d009f35 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1,5 +1,6 @@ from .utils import ( chunks, + dedupe_keys, hash_record, sqlite3, OperationalError, @@ -786,7 +787,7 @@ class Database: cursor = self.conn.execute(sql, *args) if cursor.description is None: raise ValueError(message) - keys = [d[0] for d in cursor.description] + keys = dedupe_keys(d[0] for d in cursor.description) return (dict(zip(keys, row)) for row in cursor) # Execute inside a savepoint, so a statement that turns out not to # return rows can be rolled back before the ValueError is raised @@ -796,7 +797,7 @@ class Database: cursor = self.conn.execute(sql, *args) if cursor.description is None: raise ValueError(message) - keys = [d[0] for d in cursor.description] + keys = dedupe_keys(d[0] for d in cursor.description) try: self.conn.execute('RELEASE "sqlite_utils_query"') released = True @@ -1865,7 +1866,7 @@ class Queryable: if offset is not None: sql += " offset {}".format(offset) cursor = self.db.execute(sql, where_args or []) - columns = [c[0] for c in cursor.description] + columns = dedupe_keys(c[0] for c in cursor.description) for row in cursor: yield dict(zip(columns, row)) @@ -3398,7 +3399,7 @@ class Table(Queryable): ), args, ) - columns = [c[0] for c in cursor.description] + columns = dedupe_keys(c[0] for c in cursor.description) for row in cursor: yield dict(zip(columns, row)) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 865ee79..b39b117 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -613,6 +613,37 @@ def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) -> ).hexdigest() +def dedupe_keys(keys: Iterable[str]) -> List[str]: + """ + Rename duplicates in a list of column names so every name is unique, + by appending ``_2``, ``_3``... to later occurrences - skipping any + suffix that would collide with another column in the list. + + Used when converting SQL query rows to dictionaries, where duplicate + column names would otherwise silently overwrite each other. + + :param keys: List of column names, possibly containing duplicates + """ + keys = list(keys) + taken = set(keys) + if len(taken) == len(keys): + # No duplicates - the common case + return keys + seen: set = set() + result = [] + for key in keys: + if key in seen: + new_key = key + suffix = 2 + while new_key in seen or new_key in taken: + new_key = "{}_{}".format(key, suffix) + suffix += 1 + key = new_key + seen.add(key) + result.append(key) + return result + + def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]: for key, value in d.items(): if isinstance(value, dict): diff --git a/tests/test_cli.py b/tests/test_cli.py index d26e4dd..f19a00c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -746,6 +746,26 @@ def test_query_json_empty(db_path): assert result.output.strip() == "[]" +def test_query_json_duplicate_columns_are_deduped(db_path): + # https://github.com/simonw/sqlite-utils/issues/624 + result = CliRunner().invoke( + cli.cli, + [db_path, "select 1 as id, 2 as id, 'x' as value, 'y' as value"], + ) + assert result.output.strip() == ( + '[{"id": 1, "id_2": 2, "value": "x", "value_2": "y"}]' + ) + + +def test_query_csv_duplicate_columns_are_preserved(db_path): + # CSV output should keep the duplicate headers, not rename them + result = CliRunner().invoke( + cli.cli, + [db_path, "select 1 as id, 2 as id", "--csv"], + ) + assert result.output.replace("\r", "").strip() == "id,id\n1,2" + + def test_query_invalid_function(db_path): result = CliRunner().invoke( cli.cli, [db_path, "select bad()", "--functions", "def invalid_python"] diff --git a/tests/test_fts.py b/tests/test_fts.py index 3f7c5a9..64ec645 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -83,6 +83,20 @@ def test_enable_fts_escape_table_names(fresh_db): assert [] == list(table.search("bar")) +def test_search_duplicate_columns_are_deduped(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/624 + table = fresh_db["t"] + table.insert_all(search_records) + table.enable_fts(["text", "country"], fts_version="FTS4") + rows = list(table.search("tanuki", columns=["text", "text"])) + assert rows == [ + { + "text": "tanuki are running tricksters", + "text_2": "tanuki are running tricksters", + } + ] + + def test_search_limit_offset(fresh_db): table = fresh_db["t"] table.insert_all(search_records) diff --git a/tests/test_query.py b/tests/test_query.py index b9822e1..0b9f2ae 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -218,6 +218,22 @@ def test_query_insert_returning_respects_explicit_transaction(fresh_db): assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] +def test_query_duplicate_column_names_are_deduped(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/624 + fresh_db["one"].insert({"id": 1, "value": "left"}) + fresh_db["two"].insert({"id": 2, "value": "right"}) + rows = list( + fresh_db.query("select one.id, two.id, one.value, two.value from one, two") + ) + assert rows == [{"id": 1, "id_2": 2, "value": "left", "value_2": "right"}] + + +def test_query_deduped_column_avoids_existing_names(fresh_db): + # The renamed duplicate must not overwrite a real column called id_2 + rows = list(fresh_db.query("select 1 as id, 2 as id, 3 as id_2")) + assert rows == [{"id": 1, "id_3": 2, "id_2": 3}] + + def test_execute_returning_dicts(fresh_db): # Like db.query() but returns a list, included for backwards compatibility # see https://github.com/simonw/sqlite-utils/issues/290 diff --git a/tests/test_rows.py b/tests/test_rows.py index a8a4ca0..f050d5a 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -104,3 +104,10 @@ def test_pks_and_rows_where_compound_pk(fresh_db): (("number", 1), {"type": "number", "number": 1, "plusone": 2}), (("number", 2), {"type": "number", "number": 2, "plusone": 3}), ] + + +def test_rows_where_duplicate_select_columns_are_deduped(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/624 + fresh_db["t"].insert({"id": 1, "name": "Cleo"}) + rows = list(fresh_db["t"].rows_where(select="id, id, name")) + assert rows == [{"id": 1, "id_2": 1, "name": "Cleo"}] diff --git a/tests/test_utils.py b/tests/test_utils.py index f728bcd..3de5e94 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -83,3 +83,20 @@ def test_maximize_csv_field_size_limit(): ) def test_flatten(input, expected): assert utils.flatten(input) == expected + + +@pytest.mark.parametrize( + "input,expected", + ( + ([], []), + (["id", "name"], ["id", "name"]), + (["id", "id"], ["id", "id_2"]), + (["id", "id", "id"], ["id", "id_2", "id_3"]), + # A renamed duplicate must not clobber a real column called id_2 + (["id", "id", "id_2"], ["id", "id_3", "id_2"]), + (["id_2", "id", "id"], ["id_2", "id", "id_3"]), + (["id", "id", "id_2", "id_2"], ["id", "id_3", "id_2", "id_2_2"]), + ), +) +def test_dedupe_keys(input, expected): + assert utils.dedupe_keys(input) == expected From 02281f77ed7bfb56ff74a2f09a00aa03298e3268 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 21:42:22 -0700 Subject: [PATCH 072/120] Vendor SQLite version setup action (#775) Vendor https://github.com/asg017/sqlite-versions/tree/71ea0de37ae739c33e447af91ba71dda8fcf22e6 and make it more robust against `sqlite.org` timeouts. Closes #774 --- .../actions/setup-sqlite-version/action.yml | 39 +++++ .../setup-sqlite-version.sh | 144 ++++++++++++++++++ .github/workflows/test-sqlite-support.yml | 6 +- 3 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 .github/actions/setup-sqlite-version/action.yml create mode 100644 .github/actions/setup-sqlite-version/setup-sqlite-version.sh diff --git a/.github/actions/setup-sqlite-version/action.yml b/.github/actions/setup-sqlite-version/action.yml new file mode 100644 index 0000000..fdbc71c --- /dev/null +++ b/.github/actions/setup-sqlite-version/action.yml @@ -0,0 +1,39 @@ +name: "Setup SQLite version" +description: "Build and activate a specific SQLite version from its amalgamation archive" +inputs: + version: + description: "The SQLite version to install" + required: true + cflags: + description: "CFLAGS to use when compiling SQLite" + required: false + default: "" + skip-activate: + description: "Set to true to skip modifying the library path" + required: false + default: "false" + fallback-urls: + description: "Whitespace-separated fallback download URLs to try after sqlite.org" + required: false + default: "" +outputs: + sqlite-location: + description: "Directory containing the compiled SQLite library" + value: ${{ steps.build.outputs.sqlite-location }} +runs: + using: "composite" + steps: + - shell: bash + run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads" + - uses: actions/cache@v6 + with: + path: ${{ runner.temp }}/sqlite-versions/downloads + key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1 + - id: build + shell: bash + run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh" + env: + SQLITE_VERSION: ${{ inputs.version }} + SQLITE_CFLAGS: ${{ inputs.cflags }} + SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }} + SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }} diff --git a/.github/actions/setup-sqlite-version/setup-sqlite-version.sh b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh new file mode 100644 index 0000000..0df7290 --- /dev/null +++ b/.github/actions/setup-sqlite-version/setup-sqlite-version.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}" +cflags="${SQLITE_CFLAGS:-}" +skip_activate="${SQLITE_SKIP_ACTIVATE:-false}" +extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}" + +case "$version_spec" in + 3.46 | 3.46.0) + sqlite_version="3.46.0" + sqlite_year="2024" + amalgamation_id="3460000" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip" + ;; + 3.23.1) + sqlite_version="3.23.1" + sqlite_year="2018" + amalgamation_id="3230100" + builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3230100.zip" + ;; + *) + echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh." + exit 1 + ;; +esac + +case "$(uname -s)" in + Linux) + library_name="libsqlite3.so.0" + library_path_var="LD_LIBRARY_PATH" + ;; + Darwin) + library_name="libsqlite3.dylib" + library_path_var="DYLD_LIBRARY_PATH" + ;; + *) + echo "::error::Unsupported platform $(uname -s)" + exit 1 + ;; +esac + +runner_temp="${RUNNER_TEMP:-}" +if [ -z "$runner_temp" ]; then + runner_temp="$(mktemp -d)" +fi + +filename="sqlite-amalgamation-${amalgamation_id}" +official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip" +download_dir="${runner_temp}/sqlite-versions/downloads" +source_root="${runner_temp}/sqlite-versions/source" +source_dir="${source_root}/${filename}" +build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}" +archive_path="${download_dir}/${filename}.zip" + +mkdir -p "$download_dir" "$source_root" "$build_dir" + +download_archive() { + local url + local candidate_path="${archive_path}.tmp" + local urls=("$official_url") + + for url in $builtin_fallback_urls $extra_fallback_urls; do + urls+=("$url") + done + + rm -f "$candidate_path" + for url in "${urls[@]}"; do + echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}" + if curl \ + --fail \ + --location \ + --show-error \ + --retry 5 \ + --retry-delay 2 \ + --retry-max-time 180 \ + --retry-all-errors \ + --connect-timeout 20 \ + --max-time 240 \ + --output "$candidate_path" \ + "$url"; then + mv "$candidate_path" "$archive_path" + return 0 + fi + + echo "::warning::Download failed from ${url}" + rm -f "$candidate_path" + done + + echo "::error::Could not download SQLite ${sqlite_version} amalgamation" + return 1 +} + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + if [ ! -f "$archive_path" ]; then + download_archive + fi + + rm -rf "$source_dir" + unzip -q "$archive_path" -d "$source_root" +fi + +if [ ! -f "${source_dir}/sqlite3.c" ]; then + echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}" + exit 1 +fi + +read -r -a cflag_args <<< "$cflags" + +echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}" +gcc \ + -fPIC \ + -shared \ + "${cflag_args[@]}" \ + "${source_dir}/sqlite3.c" \ + "-I${source_dir}" \ + -o "${build_dir}/${library_name}" + +if [ "$library_name" = "libsqlite3.so.0" ]; then + ln -sf "$library_name" "${build_dir}/libsqlite3.so" +fi + +if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT" +else + echo "sqlite-location=${build_dir}" +fi + +case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in + true | 1 | yes) + echo "Skipping ${library_path_var} activation" + ;; + *) + existing_value="${!library_path_var:-}" + if [ -n "${GITHUB_ENV:-}" ]; then + if [ -n "$existing_value" ]; then + echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV" + else + echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV" + fi + fi + echo "Added ${build_dir} to ${library_path_var}" + ;; +esac diff --git a/.github/workflows/test-sqlite-support.yml b/.github/workflows/test-sqlite-support.yml index aa2ab9b..f195cba 100644 --- a/.github/workflows/test-sqlite-support.yml +++ b/.github/workflows/test-sqlite-support.yml @@ -18,16 +18,16 @@ jobs: "3.23.1", # 2018-04-10, before UPSERT ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml - name: Set up SQLite ${{ matrix.sqlite-version }} - uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 + uses: ./.github/actions/setup-sqlite-version with: version: ${{ matrix.sqlite-version }} cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" From 50938ee6f846ddd792921f5a0353c73134dbbeda Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 21:54:30 -0700 Subject: [PATCH 073/120] Rename ensure_autocommit_off() to ensure_autocommit_on(), closes #705 --- docs/changelog.rst | 1 + docs/upgrading.rst | 2 ++ sqlite_utils/db.py | 20 +++++++++++++------- tests/test_wal.py | 11 +++++++++++ 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7efeb95..0220967 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Breaking changes: - ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) - Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library. +- The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``, because the old name described the opposite of what it did. The method temporarily puts the connection into driver-level autocommit mode - by setting ``isolation_level = None`` - so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. (:issue:`705`) Compound foreign key support: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index e7fd273..09a9e2f 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -77,6 +77,8 @@ Python API changes **table.convert() no longer skips falsey values.** Matching the CLI change above, ``table.convert()`` now converts every value. The ``skip_false`` parameter has been removed - previously it defaulted to ``True``, skipping empty strings and other falsey values. +**ensure_autocommit_off() is now ensure_autocommit_on().** The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``. The old name described the opposite of what the method did: it temporarily puts the connection into driver-level autocommit mode (by setting ``isolation_level = None``), so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. The behavior is unchanged - update any calls to use the new name. + **View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method. **ForeignKey is now a dataclass, not a namedtuple.** The ``ForeignKey`` objects returned by ``table.foreign_keys`` gained new fields - ``columns``, ``other_columns``, ``is_compound``, ``on_delete`` and ``on_update`` - so that compound (multi-column) foreign keys and foreign key actions can be represented. To make room for those fields cleanly ``ForeignKey`` is now a dataclass rather than a ``namedtuple``, so it can no longer be unpacked or indexed as a tuple. Access its fields by name instead: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d009f35..d984bcf 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -610,16 +610,22 @@ class Database: self.conn.execute("ROLLBACK") @contextlib.contextmanager - def ensure_autocommit_off(self) -> Generator[None, None, None]: + def ensure_autocommit_on(self) -> Generator[None, None, None]: """ - Ensure autocommit is off for this database connection. + Ensure the connection is in driver-level autocommit mode for the + duration of a block of code. + + This temporarily sets ``isolation_level = None`` on the underlying + ``sqlite3`` connection, so the driver does not open implicit + transactions. This is useful for statements such as + ``PRAGMA journal_mode=wal`` which cannot run inside a transaction. Example usage:: - with db.ensure_autocommit_off(): + with db.ensure_autocommit_on(): # do stuff here - This will reset to the previous autocommit state at the end of the block. + The previous ``isolation_level`` is restored at the end of the block. """ old_isolation_level = self.conn.isolation_level try: @@ -783,7 +789,7 @@ class Database: if self.conn.in_transaction: cursor = self.conn.execute(sql, *args) else: - with self.ensure_autocommit_off(): + with self.ensure_autocommit_on(): cursor = self.conn.execute(sql, *args) if cursor.description is None: raise ValueError(message) @@ -1085,7 +1091,7 @@ class Database: """ if self.journal_mode != "wal": self._ensure_no_open_transaction("enable_wal()") - with self.ensure_autocommit_off(): + with self.ensure_autocommit_on(): self.execute("PRAGMA journal_mode=wal;") def disable_wal(self) -> None: @@ -1097,7 +1103,7 @@ class Database: """ if self.journal_mode != "delete": self._ensure_no_open_transaction("disable_wal()") - with self.ensure_autocommit_off(): + with self.ensure_autocommit_on(): self.execute("PRAGMA journal_mode=delete;") def _ensure_no_open_transaction(self, operation: str) -> None: diff --git a/tests/test_wal.py b/tests/test_wal.py index ee7ecf0..c5a9c60 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -49,6 +49,17 @@ def test_disable_wal_inside_transaction_raises(db_path_tmpdir): assert [r["id"] for r in db["test"].rows] == [1] +def test_ensure_autocommit_on(db_path_tmpdir): + db, path, tmpdir = db_path_tmpdir + previous_isolation_level = db.conn.isolation_level + assert previous_isolation_level is not None + with db.ensure_autocommit_on(): + # isolation_level of None means driver-level autocommit mode + assert db.conn.isolation_level is None + # Restored afterwards + assert db.conn.isolation_level == previous_isolation_level + + def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): # Calling enable_wal() when WAL is already enabled is a no-op, # so it is fine inside a transaction From 77d241959c90d34f3b2122c57a5f134ead61f9f0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 22:11:22 -0700 Subject: [PATCH 074/120] Match column names case-insensitively, closes #760 Column names passed to Python API methods are now resolved against the table schema case-insensitively, mirroring how SQLite itself compares identifiers (ASCII-only case folding). Fixes KeyError populating last_pk from insert()/upsert(), silently ignored transform() rename/drop/types options, duplicate column errors from create_table(transform=True), redundant lookup() indexes, and case-sensitive foreign key validation. Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 13 ++ sqlite_utils/db.py | 238 +++++++++++++++++++++++++++++------- tests/test_column_casing.py | 233 +++++++++++++++++++++++++++++++++++ 3 files changed, 442 insertions(+), 42 deletions(-) create mode 100644 tests/test_column_casing.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 0220967..0614dcd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,19 @@ Other foreign key improvements: - Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``. - Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound. +Case-insensitive column matching: + +Column names passed to Python API methods are now matched against the table schema case-insensitively, mirroring how SQLite itself treats identifiers. Previously many methods accepted mixed-case identifiers in the SQL they generated but then failed - or silently did nothing - when performing Python-side comparisons against the schema. (:issue:`760`) Fixes include: + +- ``table.insert()`` and ``table.upsert()`` now populate ``table.last_pk`` correctly when the ``pk=`` argument uses different casing to the table schema or the record keys - previously this raised a ``KeyError`` after the row had already been written. +- Upserts no longer raise or misbehave when the casing of ``pk=`` differs from the casing of the record keys. The primary key columns are correctly excluded from the generated ``DO UPDATE SET`` clause. +- ``table.transform()`` arguments ``types=``, ``rename=``, ``drop=``, ``pk=``, ``not_null=``, ``defaults=``, ``column_order=`` and ``drop_foreign_keys=`` all resolve column names case-insensitively. Previously options like ``rename={"name": "title"}`` against a column called ``Name`` were silently ignored. +- ``db.create_table(..., transform=True)`` now recognizes existing columns that differ only by case, instead of attempting to add them again and failing with ``duplicate column name``. The casing used in the existing schema is preserved. +- ``table.lookup()`` returns the primary key value even if ``pk=`` casing differs from the schema, and recognizes existing unique indexes case-insensitively instead of creating redundant ones. +- ``table.extract()`` and ``table.convert()`` - including ``multi=True`` and ``output=`` - accept column names in any casing. +- Foreign key columns are validated and recorded using the casing of the actual schema columns, in ``foreign_keys=`` when creating tables, ``db.add_foreign_keys()``, ``table.add_foreign_key()`` and ``table.add_column(fk_col=...)``. Duplicate foreign key detection is also case-insensitive. +- ``table.create()`` with ``pk=``, ``not_null=``, ``defaults=`` or ``column_order=`` referencing columns using different casing no longer creates an unwanted extra primary key column or raises a ``ValueError``. + .. _v4_0rc2: 4.0rc2 (2026-07-04) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d984bcf..90d49c6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -85,6 +85,34 @@ def quote_identifier(identifier: str) -> str: return '"{}"'.format(identifier.replace('"', '""')) +_IDENTIFIER_CASEFOLD = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz" +) + + +def fold_identifier_case(identifier: str) -> str: + """ + Lowercase an identifier using the same rules SQLite uses - only ASCII + characters are folded, other characters are left unchanged. + """ + return identifier.translate(_IDENTIFIER_CASEFOLD) + + +def resolve_casing(name: str, candidates: Iterable[str]) -> str: + """ + SQLite treats identifiers as case-insensitive. Return the entry in + ``candidates`` that matches ``name`` case-insensitively, preferring an + exact match. If nothing matches, return ``name`` unchanged. + """ + if name in candidates: + return name + folded = fold_identifier_case(name) + for candidate in candidates: + if fold_identifier_case(candidate) == folded: + return candidate + return name + + pd: Any = None try: pd = importlib.import_module("pandas") @@ -1263,6 +1291,48 @@ class Database: ) return fks + def _resolve_foreign_key_casing( + self, fk: ForeignKey, columns: Iterable[str] + ) -> ForeignKey: + """ + Return ``fk`` with its column references resolved to match the casing + of the actual columns. ``columns`` provides the column names of + ``fk.table``, which may be a table that is still being created. + """ + resolved_columns = tuple(resolve_casing(c, columns) for c in fk.columns) + if fk.other_table == fk.table: + other_candidates: Iterable[str] = columns + else: + other_candidates = self[fk.other_table].columns_dict + resolved_other_columns = tuple( + resolve_casing(c, other_candidates) for c in fk.other_columns + ) + if ( + resolved_columns == fk.columns + and resolved_other_columns == fk.other_columns + ): + return fk + if fk.is_compound: + return ForeignKey( + fk.table, + None, + fk.other_table, + None, + columns=resolved_columns, + other_columns=resolved_other_columns, + is_compound=True, + on_delete=fk.on_delete, + on_update=fk.on_update, + ) + return ForeignKey( + fk.table, + resolved_columns[0], + fk.other_table, + resolved_other_columns[0], + on_delete=fk.on_delete, + on_update=fk.on_update, + ) + def create_table_sql( self, name: str, @@ -1296,11 +1366,14 @@ class Database: """ if hash_id_columns and (hash_id is None): hash_id = "id" - foreign_keys = self.resolve_foreign_keys(name, foreign_keys or []) + resolved_fks: List[ForeignKey] = [ + self._resolve_foreign_key_casing(fk, columns) + for fk in self.resolve_foreign_keys(name, foreign_keys or []) + ] # Compound foreign keys are rendered as table-level constraints; # single-column ones as inline REFERENCES on their column foreign_keys_by_column = { - fk.column: fk for fk in foreign_keys if not fk.is_compound + fk.column: fk for fk in resolved_fks if not fk.is_compound } # any extracts will be treated as integer columns with a foreign key extracts = resolve_extracts(extracts) @@ -1315,8 +1388,10 @@ class Database: name, extract_column, extract_table, "id" ) # Soundness check not_null, and defaults if provided - not_null = not_null or set() - defaults = defaults or {} + not_null = {resolve_casing(n, columns) for n in not_null or set()} + defaults = {resolve_casing(n, columns): v for n, v in (defaults or {}).items()} + if column_order is not None: + column_order = [resolve_casing(c, columns) for c in column_order] if not columns: raise ValueError("Tables must have at least one column") if not all(n in columns for n in not_null): @@ -1342,7 +1417,7 @@ class Database: column_items.insert(0, (hash_id, str)) pk = hash_id # Soundness check foreign_keys point to existing tables - for fk in foreign_keys: + for fk in resolved_fks: for other_column in fk.other_columns: if fk.other_table == name and columns.get(other_column): continue @@ -1356,12 +1431,14 @@ class Database: column_defs = [] # ensure pk is a tuple single_pk = None - if isinstance(pk, list) and len(pk) == 1 and isinstance(pk[0], str): + if isinstance(pk, (list, tuple)) and len(pk) == 1 and isinstance(pk[0], str): pk = pk[0] if isinstance(pk, str): - single_pk = pk + single_pk = pk = resolve_casing(pk, [c[0] for c in column_items]) if pk not in [c[0] for c in column_items]: column_items.insert(0, (pk, int)) + elif pk: + pk = [resolve_casing(p, [c[0] for c in column_items]) for p in pk] for column_name, column_type in column_items: column_extras = [] if column_name == single_pk: @@ -1402,7 +1479,7 @@ class Database: ) # Compound foreign keys become table-level FOREIGN KEY constraints column_names = [c[0] for c in column_items] - for fk in foreign_keys: + for fk in resolved_fks: if not fk.is_compound: continue missing = [c for c in fk.columns if c not in column_names] @@ -1483,6 +1560,11 @@ class Database: should_transform = False # First add missing columns and figure out columns to drop existing_columns = table.columns_dict + # Match existing columns case-insensitively, the way SQLite does + columns = { + resolve_casing(col_name, existing_columns): col_type + for col_name, col_type in columns.items() + } missing_columns = dict( (col_name, col_type) for col_name, col_type in columns.items() @@ -1506,18 +1588,28 @@ class Database: current_pks = table.pks desired_pk = None if isinstance(pk, str): - desired_pk = [pk] + desired_pk = [resolve_casing(pk, existing_columns)] elif pk: - desired_pk = list(pk) + desired_pk = [resolve_casing(p, existing_columns) for p in pk] if desired_pk and current_pks != desired_pk: should_transform = True # Any not-null changes? current_not_null = {c.name for c in table.columns if c.notnull} - desired_not_null = set(not_null) if not_null else set() + desired_not_null = ( + {resolve_casing(n, existing_columns) for n in not_null} + if not_null + else set() + ) if current_not_null != desired_not_null: should_transform = True # How about defaults? - if defaults and defaults != table.default_values: + if ( + defaults + and { + resolve_casing(c, existing_columns): v for c, v in defaults.items() + } + != table.default_values + ): should_transform = True # Only run .transform() if there is something to do if should_transform: @@ -1671,12 +1763,15 @@ class Database: is_compound=True, ) table = fk_object.table - columns = fk_object.columns other_table = fk_object.other_table - other_columns = fk_object.other_columns if not self.table(table).exists(): raise AlterError("No such table: {}".format(table)) table_obj = self.table(table) + fk_object = self._resolve_foreign_key_casing( + fk_object, table_obj.columns_dict + ) + columns = fk_object.columns + other_columns = fk_object.other_columns for column in columns: if column not in table_obj.columns_dict: raise AlterError("No such column: {} in {}".format(column, table)) @@ -1693,12 +1788,16 @@ class Database: ) ) # We will silently skip foreign keys that exist already + columns_folded = tuple(fold_identifier_case(c) for c in columns) + other_columns_folded = tuple(fold_identifier_case(c) for c in other_columns) if not any( fk for fk in table_obj.foreign_keys - if fk.columns == columns - and fk.other_table == other_table - and fk.other_columns == other_columns + if tuple(fold_identifier_case(c) for c in fk.columns) == columns_folded + and fold_identifier_case(fk.other_table) + == fold_identifier_case(other_table) + and tuple(fold_identifier_case(c) for c in fk.other_columns) + == other_columns_folded ): foreign_keys_to_create.append(fk_object) @@ -2438,6 +2537,31 @@ class Table(Queryable): rename = rename or {} drop = drop or set() + # Resolve column references against the existing schema, matching + # case-insensitively the way SQLite does + existing_columns = self.columns_dict + types = {resolve_casing(c, existing_columns): t for c, t in types.items()} + rename = {resolve_casing(c, existing_columns): v for c, v in rename.items()} + drop = {resolve_casing(c, existing_columns) for c in drop} + if pk is not DEFAULT and pk is not None: + if isinstance(pk, str): + pk = resolve_casing(pk, existing_columns) + else: + pk = [resolve_casing(p, existing_columns) for p in pk] + if isinstance(not_null, dict): + not_null = { + resolve_casing(c, existing_columns): v + for c, v in cast(Dict[str, Any], not_null).items() + } + elif isinstance(not_null, set): + not_null = {resolve_casing(c, existing_columns) for c in not_null} + if defaults is not None: + defaults = { + resolve_casing(c, existing_columns): v for c, v in defaults.items() + } + if column_order is not None: + column_order = [resolve_casing(c, existing_columns) for c in column_order] + create_table_foreign_keys: List[ForeignKeyIndicator] = [] if foreign_keys is not None: @@ -2452,28 +2576,37 @@ class Table(Queryable): create_table_foreign_keys.extend(foreign_keys) else: # Construct foreign_keys from current, plus add_foreign_keys, minus drop_foreign_keys - # Bind fresh names here - type checkers do not narrow captured - # variables inside nested functions, so the closures would - # otherwise see the Optional declared types of drop and rename - dropped_columns = drop - renamed_columns = rename + # The casing of columns in a foreign key definition can differ + # from the casing of the columns themselves, so these comparisons + # are all case-folded + dropped_columns_folded = {fold_identifier_case(c) for c in drop} + renamed_columns_folded = { + fold_identifier_case(k): v for k, v in rename.items() + } def fk_should_be_dropped(fk: ForeignKey) -> bool: + fk_columns_folded = tuple(fold_identifier_case(c) for c in fk.columns) if drop_foreign_keys is not None: for spec in drop_foreign_keys: if isinstance(spec, str): # A column name matches any foreign key it participates in - if spec in fk.columns: + if fold_identifier_case(spec) in fk_columns_folded: return True - elif tuple(spec) == fk.columns: + elif ( + tuple(fold_identifier_case(s) for s in spec) + == fk_columns_folded + ): # A tuple/list must match a compound key's columns exactly return True # Dropping any of a foreign key's columns drops the whole key - return any(column in dropped_columns for column in fk.columns) + return any( + column in dropped_columns_folded for column in fk_columns_folded + ) def fk_with_renamed_columns(fk: ForeignKey) -> ForeignKey: columns = tuple( - renamed_columns.get(column) or column for column in fk.columns + renamed_columns_folded.get(fold_identifier_case(column)) or column + for column in fk.columns ) if fk.is_compound: return ForeignKey( @@ -2662,6 +2795,8 @@ class Table(Queryable): rename = rename or {} if isinstance(columns, str): columns = [columns] + columns = [resolve_casing(c, self.columns_dict) for c in columns] + rename = {resolve_casing(k, self.columns_dict): v for k, v in rename.items()} if not set(columns).issubset(self.columns_dict.keys()): raise InvalidColumns( "Invalid columns {} for table with columns {}".format( @@ -2853,6 +2988,7 @@ class Table(Queryable): raise AlterError("table '{}' does not exist".format(fk)) # if fk_col specified, must be a valid column if fk_col is not None: + fk_col = resolve_casing(fk_col, self.db[fk].columns_dict) if fk_col not in self.db[fk].columns_dict: raise AlterError("table '{}' has no column {}".format(fk, fk_col)) else: @@ -2958,6 +3094,7 @@ class Table(Queryable): :param on_update: ``ON UPDATE`` action for the foreign key. """ columns = (column,) if isinstance(column, str) else tuple(column) + columns = tuple(resolve_casing(c, self.columns_dict) for c in columns) # Ensure columns exist for col in columns: if col not in self.columns_dict: @@ -2979,6 +3116,9 @@ class Table(Queryable): other_columns = (other_column,) else: other_columns = tuple(other_column) + other_columns = tuple( + resolve_casing(c, self.db[other_table].columns_dict) for c in other_columns + ) if len(columns) != len(other_columns): raise ValueError( "Compound foreign key must have the same number of columns " @@ -2996,9 +3136,12 @@ class Table(Queryable): if any( fk for fk in self.foreign_keys - if fk.columns == columns - and fk.other_table == other_table - and fk.other_columns == other_columns + if tuple(fold_identifier_case(c) for c in fk.columns) + == tuple(fold_identifier_case(c) for c in columns) + and fold_identifier_case(fk.other_table) + == fold_identifier_case(other_table) + and tuple(fold_identifier_case(c) for c in fk.other_columns) + == tuple(fold_identifier_case(c) for c in other_columns) ): if ignore: return self @@ -3547,6 +3690,7 @@ class Table(Queryable): """ if isinstance(columns, str): columns = [columns] + columns = [resolve_casing(c, self.columns_dict) for c in columns] if multi: return self._convert_multi( @@ -3561,6 +3705,7 @@ class Table(Queryable): if output is not None: if len(columns) != 1: raise ValueError("output= can only be used with a single column") + output = resolve_casing(output, self.columns_dict) if output not in self.columns_dict: self.add_column(output, output_type or "text") @@ -3760,6 +3905,8 @@ class Table(Queryable): # Everything from here on is for upsert=True pk_cols = [pk] if isinstance(pk, str) else list(pk) + # The records may use different casing for the pk columns than pk= + pk_cols = [resolve_casing(c, all_columns) for c in pk_cols] # Every record must provide a value for every primary key column - a # NULL primary key never matches ON CONFLICT, so the record would be # inserted as a brand new row instead of upserted @@ -3810,10 +3957,7 @@ class Table(Queryable): # At this point we need compatibility UPSERT for SQLite < 3.24.0 # (INSERT OR IGNORE + second UPDATE stage) queries_and_params = [] - if isinstance(pk, str): - pks = [pk] - else: - pks = pk + pks = pk_cols self.last_pk = None for record_values in values: record = dict(zip(all_columns, record_values)) @@ -4226,9 +4370,9 @@ class Table(Queryable): if hash_id: self.last_pk = row[hash_id] elif isinstance(pk, str): - self.last_pk = row[pk] + self.last_pk = row[resolve_casing(pk, row)] else: - self.last_pk = tuple(row[p] for p in pk) + self.last_pk = tuple(row[resolve_casing(p, row)] for p in pk) else: self.last_pk = self.last_rowid else: @@ -4240,11 +4384,14 @@ class Table(Queryable): # hash_id not supported in list mode for last_pk pass elif isinstance(pk, str): - pk_index = column_names.index(pk) + pk_index = column_names.index(resolve_casing(pk, column_names)) self.last_pk = first_record_list[pk_index] else: self.last_pk = tuple( - first_record_list[column_names.index(p)] for p in pk + first_record_list[ + column_names.index(resolve_casing(p, column_names)) + ] + for p in pk ) else: first_record_dict = cast(Dict[str, Any], first_record) @@ -4252,9 +4399,12 @@ class Table(Queryable): self.last_pk = hash_record(first_record_dict, hash_id_columns) else: self.last_pk = ( - first_record_dict[pk] + first_record_dict[resolve_casing(pk, first_record_dict)] if isinstance(pk, str) - else tuple(first_record_dict[p] for p in pk) + else tuple( + first_record_dict[resolve_casing(p, first_record_dict)] + for p in pk + ) ) if analyze: @@ -4399,8 +4549,12 @@ class Table(Queryable): combined_values.update(extra_values) if self.exists(): self.add_missing_columns([combined_values]) - unique_column_sets = [set(i.columns) for i in self.indexes] - if set(lookup_values.keys()) not in unique_column_sets: + unique_column_sets = [ + {fold_identifier_case(c) for c in i.columns} for i in self.indexes + ] + if { + fold_identifier_case(c) for c in lookup_values + } not in unique_column_sets: self.create_index(lookup_values.keys(), unique=True) wheres = [ "{} = ?".format(quote_identifier(column)) for column in lookup_values @@ -4411,7 +4565,7 @@ class Table(Queryable): ) ) try: - return rows[0][pk] + return rows[0][resolve_casing(pk, rows[0])] except IndexError: return self.insert( combined_values, diff --git a/tests/test_column_casing.py b/tests/test_column_casing.py new file mode 100644 index 0000000..ce11345 --- /dev/null +++ b/tests/test_column_casing.py @@ -0,0 +1,233 @@ +""" +SQLite treats column names as case-insensitive. These tests exercise the +places where sqlite-utils performs Python-side lookups of column names +provided by the caller, which should match the schema case-insensitively. + +https://github.com/simonw/sqlite-utils/issues/760 +""" + +import pytest + +from sqlite_utils import Database +from sqlite_utils.db import ForeignKey + + +def test_insert_populates_last_pk_case_insensitively(fresh_db): + books = fresh_db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.insert({"Id": 1, "Title": "One"}, pk="id") + assert books.last_pk == 1 + + +def test_insert_populates_last_pk_compound_pk_case_insensitively(fresh_db): + books = fresh_db["books"] + books.create({"Author": str, "Position": int, "Title": str}) + books.insert( + {"Author": "Sue", "Position": 1, "Title": "One"}, pk=("author", "position") + ) + assert books.last_pk == ("Sue", 1) + + +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_pk_case_differs_from_schema(use_old_upsert): + db = Database(memory=True, use_old_upsert=use_old_upsert) + books = db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.insert({"Id": 1, "Title": "One"}) + books.upsert({"id": 1, "title": "Won"}, pk="id") + assert list(books.rows) == [{"Id": 1, "Title": "Won"}] + assert books.last_pk == 1 + + +@pytest.mark.parametrize("use_old_upsert", (False, True)) +def test_upsert_record_key_case_differs_from_pk(use_old_upsert): + # all_columns comes from the record keys, pk= from the caller + db = Database(memory=True, use_old_upsert=use_old_upsert) + books = db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.upsert({"ID": 1, "Title": "One"}, pk="id") + assert list(books.rows) == [{"Id": 1, "Title": "One"}] + assert books.last_pk == 1 + + +def test_upsert_inferred_pk_case_differs_from_record_keys(fresh_db): + # pk is inferred from the existing schema as "Id", records use "id" + books = fresh_db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.upsert({"id": 1, "title": "One"}) + assert list(books.rows) == [{"Id": 1, "Title": "One"}] + assert books.last_pk == 1 + + +def test_upsert_list_mode_pk_case_insensitive(fresh_db): + books = fresh_db["books"] + books.create({"Id": int, "Title": str}, pk="Id") + books.upsert_all([["id", "title"], [1, "One"]], pk="Id") + assert list(books.rows) == [{"Id": 1, "Title": "One"}] + assert books.last_pk == 1 + + +def test_lookup_pk_case_insensitive(fresh_db): + fresh_db["species"].create({"ID": int, "Name": str}, pk="ID") + fresh_db["species"].insert({"ID": 5, "Name": "Palm"}) + fresh_db["species"].create_index(["Name"], unique=True) + assert fresh_db["species"].lookup({"Name": "Palm"}, pk="id") == 5 + + +def test_lookup_does_not_create_redundant_index(fresh_db): + fresh_db["species"].create({"id": int, "Name": str}, pk="id") + fresh_db["species"].create_index(["Name"], unique=True) + fresh_db["species"].lookup({"name": "Palm"}) + assert len(fresh_db["species"].indexes) == 1 + + +def test_create_table_transform_same_columns_different_case(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db["t"].insert({"Name": "Cleo", "Age": 5}) + fresh_db.create_table("t", {"name": str, "age": int}, transform=True) + # Schema casing is preserved - SQLite considers these the same columns + assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} + assert list(fresh_db["t"].rows) == [{"Name": "Cleo", "Age": 5}] + + +def test_create_table_transform_case_insensitive_with_changes(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db.create_table("t", {"name": str, "age": str, "size": int}, transform=True) + # age changed type, size added, Name untouched + assert fresh_db["t"].columns_dict == {"Name": str, "Age": str, "size": int} + + +def test_transform_types_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str, "Age": str}) + fresh_db["t"].transform(types={"age": int}) + assert fresh_db["t"].columns_dict == {"Name": str, "Age": int} + + +def test_transform_rename_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str}) + fresh_db["t"].transform(rename={"name": "title"}) + assert fresh_db["t"].columns_dict == {"title": str} + + +def test_transform_drop_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db["t"].transform(drop=["name"]) + assert fresh_db["t"].columns_dict == {"Age": int} + + +def test_transform_not_null_and_defaults_case_insensitive(fresh_db): + fresh_db["t"].create({"Name": str, "Age": int}) + fresh_db["t"].transform(not_null={"name"}, defaults={"age": 3}) + columns = {c.name: c for c in fresh_db["t"].columns} + assert columns["Name"].notnull + assert fresh_db["t"].default_values == {"Age": 3} + + +def test_transform_pk_case_insensitive(fresh_db): + fresh_db["t"].create({"Id": int, "Name": str}) + fresh_db["t"].transform(pk="id") + assert fresh_db["t"].pks == ["Id"] + assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} + + +def test_transform_drop_foreign_keys_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create( + {"id": int, "Parent_ID": int}, + pk="id", + foreign_keys=[("Parent_ID", "parent", "Id")], + ) + fresh_db["child"].transform(drop_foreign_keys=["parent_id"]) + assert fresh_db["child"].foreign_keys == [] + + +def test_add_foreign_key_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") + fresh_db["child"].add_foreign_key("parent_id", "parent", "id") + fks = fresh_db["child"].foreign_keys + assert len(fks) == 1 + # The foreign key should use the schema casing of the columns + assert fks[0].column == "Parent_ID" + assert fks[0].other_column == "Id" + + +def test_add_foreign_keys_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create({"id": int, "Parent_ID": int}, pk="id") + fresh_db.add_foreign_keys([("child", "parent_id", "parent", "id")]) + fks = fresh_db["child"].foreign_keys + assert len(fks) == 1 + assert fks[0].column == "Parent_ID" + assert fks[0].other_column == "Id" + + +def test_add_foreign_key_detects_existing_case_insensitively(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create( + {"id": int, "Parent_ID": int}, + pk="id", + foreign_keys=[("Parent_ID", "parent", "Id")], + ) + # ignore=True should treat this as already existing, not add a duplicate + fresh_db["child"].add_foreign_key("parent_id", "parent", "id", ignore=True) + assert len(fresh_db["child"].foreign_keys) == 1 + + +def test_add_column_fk_col_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create({"id": int}, pk="id") + fresh_db["child"].add_column("parent_id", int, fk="parent", fk_col="id") + fks = fresh_db["child"].foreign_keys + assert len(fks) == 1 + assert fks[0].other_column == "Id" + + +def test_extract_case_insensitive(fresh_db): + fresh_db["trees"].insert({"id": 1, "Species": "Palm"}, pk="id") + fresh_db["trees"].extract("species") + assert fresh_db["trees"].columns_dict == {"id": int, "Species_id": int} + assert list(fresh_db["Species"].rows) == [{"id": 1, "Species": "Palm"}] + + +def test_convert_multi_case_insensitive(fresh_db): + fresh_db["t"].insert({"id": 1, "Name": "Cleo"}, pk="id") + fresh_db["t"].convert("name", lambda v: {"upper": v.upper()}, multi=True) + assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "upper": "CLEO"}] + + +def test_convert_output_case_insensitive(fresh_db): + fresh_db["t"].insert({"id": 1, "Name": "Cleo", "Upper": None}, pk="id") + fresh_db["t"].convert("name", lambda v: v.upper(), output="upper") + assert list(fresh_db["t"].rows) == [{"id": 1, "Name": "Cleo", "Upper": "CLEO"}] + + +def test_create_table_sql_pk_case_insensitive(fresh_db): + fresh_db["t"].create({"Id": int, "Name": str}, pk="id") + # Should not have created an extra lowercase "id" column + assert fresh_db["t"].columns_dict == {"Id": int, "Name": str} + assert fresh_db["t"].pks == ["Id"] + + +def test_create_table_not_null_and_defaults_case_insensitive(fresh_db): + fresh_db["t"].create( + {"Name": str, "Age": int}, not_null={"name"}, defaults={"age": 1} + ) + columns = {c.name: c for c in fresh_db["t"].columns} + assert columns["Name"].notnull + assert fresh_db["t"].default_values == {"Age": 1} + + +def test_create_table_foreign_keys_case_insensitive(fresh_db): + fresh_db["parent"].create({"Id": int}, pk="Id") + fresh_db["child"].create( + {"id": int, "Parent_ID": int}, + pk="id", + foreign_keys=[("parent_id", "parent", "id")], + ) + fks = fresh_db["child"].foreign_keys + assert fks == [ + ForeignKey( + table="child", column="Parent_ID", other_table="parent", other_column="Id" + ) + ] From a4acc3958c456298bb017cbdea515fe4d5374ed3 Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Mon, 6 Jul 2026 07:33:55 +0200 Subject: [PATCH 075/120] Fix transform() corrupting TRUE/FALSE/NULL column defaults into strings * Fix transform() corrupting TRUE/FALSE/NULL column defaults quote_default_value() passed keyword-literal defaults (TRUE, FALSE, NULL) through self.quote(), wrapping them in quotes. So transform() rebuilt a column declared "INTEGER DEFAULT TRUE" as "INTEGER DEFAULT 'TRUE'", and a later default insert stored the text 'TRUE' instead of the integer 1 (likewise 'FALSE' for 0 and 'NULL' for null) - silent value corruption. Return these keyword literals unquoted, as already done for the CURRENT_TIME/DATE/TIMESTAMP literals. PR #764 --- docs/changelog.rst | 4 ++++ sqlite_utils/db.py | 5 +++++ tests/test_default_value.py | 5 +++++ tests/test_transform.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0614dcd..43a4653 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -42,6 +42,10 @@ Column names passed to Python API methods are now matched against the table sche - Foreign key columns are validated and recorded using the casing of the actual schema columns, in ``foreign_keys=`` when creating tables, ``db.add_foreign_keys()``, ``table.add_foreign_key()`` and ``table.add_column(fk_col=...)``. Duplicate foreign key detection is also case-insensitive. - ``table.create()`` with ``pk=``, ``not_null=``, ``defaults=`` or ``column_order=`` referencing columns using different casing no longer creates an unwanted extra primary key column or raises a ``ValueError``. +Everything else: + +- Fixed a bug where ``table.transform()`` could convert ``DEFAULT TRUE``, ``DEFAULT FALSE`` and ``DEFAULT NULL`` column defaults into quoted string defaults when rebuilding a table. Thanks, `Vincent Gao `__. (`#764 `__) + .. _v4_0rc2: 4.0rc2 (2026-07-04) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 90d49c6..0807892 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -983,6 +983,11 @@ class Database: if str(value).upper() in ("CURRENT_TIME", "CURRENT_DATE", "CURRENT_TIMESTAMP"): return value + if isinstance(value, str) and value.upper() in ("TRUE", "FALSE", "NULL"): + # Keyword literals must stay unquoted; quoting them would turn the + # default into a string ('TRUE' instead of 1, 'NULL' instead of null). + return value + if str(value).endswith(")"): # Expr return "({})".format(value) diff --git a/tests/test_default_value.py b/tests/test_default_value.py index c5e4b17..3724d99 100644 --- a/tests/test_default_value.py +++ b/tests/test_default_value.py @@ -21,6 +21,11 @@ EXAMPLES = [ # Strings ("TEXT DEFAULT 'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'"), ('TEXT DEFAULT "CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"'), + # Boolean and null keyword literals must stay unquoted + ("INTEGER DEFAULT TRUE", "TRUE", "TRUE"), + ("INTEGER DEFAULT FALSE", "FALSE", "FALSE"), + ("INTEGER DEFAULT true", "true", "true"), + ("TEXT DEFAULT NULL", "NULL", "NULL"), ] diff --git a/tests/test_transform.py b/tests/test_transform.py index 5eb501d..71518be 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -224,6 +224,40 @@ def test_transform_rename_pk(fresh_db): ) +def test_transform_preserves_keyword_literal_defaults(fresh_db): + # transform() used to requote keyword-literal defaults (DEFAULT TRUE became + # DEFAULT 'TRUE'), so a default insert stored the text 'TRUE' instead of the + # integer 1 -- silent value corruption on every rebuilt table. + fresh_db.execute( + "CREATE TABLE t (" + " id INTEGER PRIMARY KEY," + " is_active INTEGER DEFAULT TRUE," + " flag INTEGER DEFAULT FALSE," + " note TEXT DEFAULT NULL" + ")" + ) + table = fresh_db["t"] + table.insert({"id": 1}) + before = fresh_db.execute("SELECT is_active, flag, note FROM t").fetchone() + assert before == (1, 0, None) + + # Rebuild the table via an unrelated change. + table.transform(rename={"note": "note2"}) + + # The keyword literals stay unquoted in the schema ... + assert "DEFAULT TRUE" in table.schema + assert "DEFAULT FALSE" in table.schema + assert "DEFAULT NULL" in table.schema + assert "'TRUE'" not in table.schema + + # ... and a fresh default insert still yields 1 / 0 / NULL, not strings. + table.insert({"id": 2}) + after = fresh_db.execute( + "SELECT is_active, flag, note2 FROM t WHERE id = 2" + ).fetchone() + assert after == (1, 0, None) + + def test_transform_not_null(fresh_db): dogs = fresh_db["dogs"] dogs.insert({"id": 1, "name": "Cleo", "age": "5"}, pk="id") From 3e8b7403a286e0f9ae4cfb9b8ff2e010c3edb2de Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 22:35:35 -0700 Subject: [PATCH 076/120] Release 4.0rc3 Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4889420844 Refs #530, #594, #624, #705, #760, #764, #769, #770, #771, #772, #774, #775 --- docs/changelog.rst | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 43a4653..4b48a40 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,10 +4,10 @@ Changelog =========== -.. _v_unreleased: +.. _v4_0rc3: -Unreleased ----------- +4.0rc3 (2026-07-05) +------------------- Breaking changes: diff --git a/pyproject.toml b/pyproject.toml index 9d1f041..bff9390 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.0rc2" +version = "4.0rc3" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From d1f5e06816f2fdaa2f626c8cd6814a209bf51a7b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 23:12:54 -0700 Subject: [PATCH 077/120] Add subheadings to 4.0rc3 release notes So I can link to those sections. --- docs/changelog.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4b48a40..6412101 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,27 +9,31 @@ 4.0rc3 (2026-07-05) ------------------- -Breaking changes: +Breaking changes +~~~~~~~~~~~~~~~~ - ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) - Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library. - The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``, because the old name described the opposite of what it did. The method temporarily puts the connection into driver-level autocommit mode - by setting ``isolation_level = None`` - so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. (:issue:`705`) -Compound foreign key support: +Compound foreign key support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Tables can now be created with compound foreign keys, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. See :ref:`python_api_compound_foreign_keys`. - ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely. - ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept tuples of column names to add a compound foreign key to an existing table. - ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key. -Other foreign key improvements: +Other foreign key improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - ``ForeignKey`` now exposes ``on_delete`` and ``on_update`` fields reflecting the foreign key's ``ON DELETE``/``ON UPDATE`` actions, and ``table.transform()`` preserves those actions. Previously a transform silently stripped clauses such as ``ON DELETE CASCADE`` from the table schema. - ``table.add_foreign_key()`` accepts new ``on_delete=`` and ``on_update=`` parameters for creating foreign keys with actions, e.g. ``table.add_foreign_key("author_id", "authors", "id", on_delete="CASCADE")``. (:issue:`530`) - Foreign keys declared as ``REFERENCES other_table`` with no explicit column are now resolved to the other table's primary key by ``table.foreign_keys``, instead of reporting ``other_column=None``. - Fixed a ``TypeError`` when sorting ``ForeignKey`` objects where some were compound. -Case-insensitive column matching: +Case-insensitive column matching +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Column names passed to Python API methods are now matched against the table schema case-insensitively, mirroring how SQLite itself treats identifiers. Previously many methods accepted mixed-case identifiers in the SQL they generated but then failed - or silently did nothing - when performing Python-side comparisons against the schema. (:issue:`760`) Fixes include: @@ -42,7 +46,8 @@ Column names passed to Python API methods are now matched against the table sche - Foreign key columns are validated and recorded using the casing of the actual schema columns, in ``foreign_keys=`` when creating tables, ``db.add_foreign_keys()``, ``table.add_foreign_key()`` and ``table.add_column(fk_col=...)``. Duplicate foreign key detection is also case-insensitive. - ``table.create()`` with ``pk=``, ``not_null=``, ``defaults=`` or ``column_order=`` referencing columns using different casing no longer creates an unwanted extra primary key column or raises a ``ValueError``. -Everything else: +Everything else +~~~~~~~~~~~~~~~ - Fixed a bug where ``table.transform()`` could convert ``DEFAULT TRUE``, ``DEFAULT FALSE`` and ``DEFAULT NULL`` column defaults into quoted string defaults when rebuilding a table. Thanks, `Vincent Gao `__. (`#764 `__) From af3894a096ed44433ca4d40bf7bf701a71f4f097 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 23:19:00 -0700 Subject: [PATCH 078/120] Changelog headings for 4.0rc3 --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6412101..085d9d5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,14 +12,14 @@ Breaking changes ~~~~~~~~~~~~~~~~ -- ``table.foreign_keys`` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) +- :ref:`table.foreign_keys ` now returns ``ForeignKey`` objects that are dataclasses rather than ``namedtuple`` instances, so they can no longer be unpacked or indexed as ``(table, column, other_table, other_column)`` tuples - access their fields by name instead. Compound (multi-column) foreign keys are now represented as a single ``ForeignKey`` with ``is_compound=True`` and populated ``columns``/``other_columns`` tuples, where ``column`` and ``other_column`` are ``None``. Previously they were returned as one ``ForeignKey`` per column, misleadingly suggesting several independent foreign keys. See :ref:`upgrading_3_to_4` for details. (:issue:`594`) - Removed support for using ``sqlean.py`` as a drop-in replacement for the Python standard library ``sqlite3`` module. ``sqlite-utils`` will now use ``pysqlite3`` if it is installed, otherwise it will use ``sqlite3`` from the standard library. - The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``, because the old name described the opposite of what it did. The method temporarily puts the connection into driver-level autocommit mode - by setting ``isolation_level = None`` - so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. (:issue:`705`) Compound foreign key support ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Tables can now be created with compound foreign keys, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. See :ref:`python_api_compound_foreign_keys`. +- Tables can now be created with :ref:`compound foreign keys `, by passing tuples of column names in ``foreign_keys=``: ``foreign_keys=[(("campus_name", "dept_code"), "departments")]``. The referenced columns default to the compound primary key of the other table. Compound keys are rendered as table-level ``FOREIGN KEY`` constraints in the generated schema. - ``table.transform()`` now preserves compound foreign keys, applying any column renames to them. Dropping a column that is part of a compound foreign key drops the whole constraint, matching the existing single-column behavior. ``drop_foreign_keys=`` accepts a bare column name - dropping any foreign key that column participates in - or a tuple of columns to target a compound key precisely. - ``table.add_foreign_key()`` and ``db.add_foreign_keys()`` accept tuples of column names to add a compound foreign key to an existing table. - ``db.index_foreign_keys()`` creates a single composite index for a compound foreign key. From 5f81752cf5e2a9a3f6ef5bff481887582b71b03f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 5 Jul 2026 23:19:40 -0700 Subject: [PATCH 079/120] Fix for frustrating error in 'just docs' locally fatal: refusing to fetch into branch 'refs/heads/main' checked out at '/Users/simon/Dropbox/dev/sqlite-utils' I'm not going to pretend to understand this fix, it works. --- docs/conf.py | 58 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 04a2301..4f29b39 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from subprocess import Popen, PIPE -from beanbag_docutils.sphinx.ext.github import github_linkcode_resolve +import inspect +from pathlib import Path +from subprocess import Popen, PIPE, check_output +import sys # This file is execfile()d with the current directory set to its # containing dir. @@ -45,14 +47,52 @@ extlinks = { } +def _linkcode_git_ref(): + try: + return check_output(["git", "rev-parse", "HEAD"]).decode("utf8").strip() + except Exception: + return "main" + + def linkcode_resolve(domain, info): - return github_linkcode_resolve( - domain=domain, - info=info, - allowed_module_names=["sqlite_utils"], - github_org_id="simonw", - github_repo_id="sqlite-utils", - branch="main", + if domain != "py": + return None + + module_name = info.get("module") + if not module_name or module_name.split(".")[0] != "sqlite_utils": + return None + + module = sys.modules.get(module_name) + if module is None: + return None + + obj = module + for part in info.get("fullname", "").split("."): + obj = getattr(obj, part, None) + if obj is None: + return None + + if isinstance(obj, property): + obj = obj.fget + + try: + obj = inspect.unwrap(obj) + source_file = inspect.getsourcefile(obj) + _, line_number = inspect.getsourcelines(obj) + except Exception: + return None + + if source_file is None: + return None + + try: + filename = Path(source_file).resolve().relative_to(Path(__file__).parent.parent) + except ValueError: + return None + + return ( + "https://github.com/simonw/sqlite-utils/blob/" + f"{_linkcode_git_ref()}/{filename}#L{line_number}" ) From d516e585433792c2bc2e99f2bc6eaa3227c462ae Mon Sep 17 00:00:00 2001 From: Johnson K C Date: Mon, 6 Jul 2026 09:54:17 -0700 Subject: [PATCH 080/120] Honor --no-headers for --fmt and --table output (#566) (#751) Co-authored-by: Claude Opus 4.8 (1M context) --- docs/cli-reference.rst | 16 +++++++-------- sqlite_utils/cli.py | 18 ++++++++++++++--- tests/test_cli.py | 44 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index cbac48b..1eeaabf 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -116,7 +116,7 @@ See :ref:`cli_query`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, @@ -185,7 +185,7 @@ See :ref:`cli_memory`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, @@ -422,7 +422,7 @@ See :ref:`cli_search`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -688,7 +688,7 @@ See :ref:`cli_tables`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -730,7 +730,7 @@ See :ref:`cli_views`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -777,7 +777,7 @@ See :ref:`cli_rows`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, @@ -817,7 +817,7 @@ See :ref:`cli_triggers`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, @@ -857,7 +857,7 @@ See :ref:`cli_indexes`. --arrays Output rows as arrays instead of objects --csv Output CSV --tsv Output TSV - --no-headers Omit CSV headers + --no-headers Omit headers from CSV/TSV and table/--fmt output -t, --table Output as a formatted table --fmt TEXT Table format - one of asciidoc, colon_grid, double_grid, double_outline, fancy_grid, fancy_outline, diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index d5a4b1e..8918439 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -111,7 +111,11 @@ def output_options(fn): ), click.option("--csv", is_flag=True, help="Output CSV"), click.option("--tsv", is_flag=True, help="Output TSV"), - click.option("--no-headers", is_flag=True, help="Omit CSV headers"), + click.option( + "--no-headers", + is_flag=True, + help="Omit headers from CSV/TSV and table/--fmt output", + ), click.option( "-t", "--table", is_flag=True, help="Output as a formatted table" ), @@ -240,7 +244,13 @@ def tables( yield row if table or fmt: - print(tabulate.tabulate(_iter(), headers=headers, tablefmt=fmt or "simple")) + print( + tabulate.tabulate( + _iter(), + headers=() if no_headers else headers, + tablefmt=fmt or "simple", + ) + ) elif csv or tsv: writer = csv_std.writer(sys.stdout, dialect="excel-tab" if tsv else "excel") if not no_headers: @@ -2145,7 +2155,9 @@ def _execute_query( elif fmt or table: print( tabulate.tabulate( - list(cursor), headers=headers, tablefmt=fmt or "simple" + list(cursor), + headers=() if no_headers else headers, + tablefmt=fmt or "simple", ) ) elif csv or tsv: diff --git a/tests/test_cli.py b/tests/test_cli.py index f19a00c..d11c42f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -195,6 +195,50 @@ def test_output_table(db_path, options, expected): assert expected == result.output.strip() +@pytest.mark.parametrize( + "fmt_option", [["--fmt", "simple"], ["-t"], ["--fmt", "github"]] +) +def test_output_table_no_headers(db_path, fmt_option): + # --no-headers should omit the header row from --fmt/--table output too, not + # just from --csv/--tsv (#566). Previously the flag was silently ignored for + # tabulate formats and the column names were always printed. + db = Database(db_path) + with db.conn: + db["dogs"].insert_all( + [ + {"id": 1, "name": "Cleo", "age": 4}, + {"id": 2, "name": "Pancakes", "age": 2}, + ] + ) + sql = "select id, name, age from dogs order by id" + + with_headers = CliRunner().invoke(cli.cli, ["query", db_path, sql] + fmt_option) + without_headers = CliRunner().invoke( + cli.cli, ["query", db_path, sql] + fmt_option + ["--no-headers"] + ) + assert with_headers.exit_code == 0 + assert without_headers.exit_code == 0 + + # The column names appear when headers are shown, and must not appear at all + # once --no-headers is passed. + assert "name" in with_headers.output + for header in ("id", "name", "age"): + assert ( + header not in without_headers.output + ), f"header {header!r} leaked into --no-headers output" + # The data is still all present. + for value in ("Cleo", "Pancakes", "1", "2", "4"): + assert value in without_headers.output + + # The rows command shares the same code path. + rows_no_headers = CliRunner().invoke( + cli.cli, ["rows", db_path, "dogs"] + fmt_option + ["--no-headers"] + ) + assert rows_no_headers.exit_code == 0 + assert "name" not in rows_no_headers.output + assert "Cleo" in rows_no_headers.output + + def test_create_index(db_path): db = Database(db_path) assert [] == db["Gosh"].indexes From 815b6a7d3dcd262b9ef415482d197a2141b60427 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 11:10:07 -0700 Subject: [PATCH 081/120] JSON output no longer escapes non-ASCII characters, new --ascii option (#777) Closes #625 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JaHan1NhaTRAxJ9LQtSLf9 --- docs/changelog.rst | 7 +++++++ docs/cli-reference.rst | 11 +++++++++++ docs/cli.rst | 27 +++++++++++++++++++++++++++ sqlite_utils/cli.py | 35 +++++++++++++++++++++++++++++------ tests/test_cli.py | 28 ++++++++++++++++++++++++++++ tests/test_cli_convert.py | 19 +++++++++++++++++++ 6 files changed, 121 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 085d9d5..556e497 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _unreleased: + +Unreleased +---------- + +- JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) + .. _v4_0rc3: 4.0rc3 (2026-07-05) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 1eeaabf..49eba52 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -129,6 +129,8 @@ See :ref:`cli_query`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX -r, --raw Raw output, first column of first row --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query @@ -198,6 +200,8 @@ See :ref:`cli_memory`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX -r, --raw Raw output, first column of first row --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query @@ -435,6 +439,7 @@ See :ref:`cli_search`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -701,6 +706,7 @@ See :ref:`cli_tables`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --columns Include list of columns for each table --schema Include schema for each table --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -743,6 +749,7 @@ See :ref:`cli_views`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --columns Include list of columns for each view --schema Include schema for each view --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -790,6 +797,8 @@ See :ref:`cli_rows`. simple_outline, textile, tsv, unsafehtml, youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as + \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -830,6 +839,7 @@ See :ref:`cli_triggers`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -870,6 +880,7 @@ See :ref:`cli_indexes`. youtrack --json-cols Detect JSON cols and output them as JSON, not escaped strings + --ascii Escape non-ASCII characters in JSON output as \uXXXX --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. diff --git a/docs/cli.rst b/docs/cli.rst index 84a65d9..957ee64 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -111,6 +111,33 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +.. _cli_query_json_ascii: + +Unicode characters in JSON +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +JSON output includes unicode characters directly, without escaping them: + +.. code-block:: bash + + sqlite-utils dogs.db "select '日本語' as text" + +.. code-block:: output + + [{"text": "日本語"}] + +Use ``--ascii`` to escape non-ASCII characters as ``\uXXXX`` sequences instead: + +.. code-block:: bash + + sqlite-utils dogs.db "select '日本語' as text" --ascii + +.. code-block:: output + + [{"text": "\u65e5\u672c\u8a9e"}] + +The ``--ascii`` option can help on systems that cannot display or process UTF-8, such as Windows consoles using a legacy code page. On Windows, setting the ``PYTHONUTF8=1`` environment variable is an alternative fix for ``UnicodeEncodeError`` crashes when redirecting output to a file. + .. _cli_query_binary_json: Binary data in JSON diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8918439..8ee2091 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -131,6 +131,13 @@ def output_options(fn): is_flag=True, default=False, ), + click.option( + "--ascii", + "ascii_", + help="Escape non-ASCII characters in JSON output as \\uXXXX", + is_flag=True, + default=False, + ), ) ): fn = decorator(fn) @@ -199,6 +206,7 @@ def tables( table, fmt, json_cols, + ascii_, columns, schema, load_extension, @@ -258,7 +266,7 @@ def tables( for row in _iter(): writer.writerow(row) else: - for line in output_rows(_iter(), headers, nl, arrays, json_cols): + for line in output_rows(_iter(), headers, nl, arrays, json_cols, ascii_): click.echo(line) @@ -296,6 +304,7 @@ def views( table, fmt, json_cols, + ascii_, columns, schema, load_extension, @@ -321,6 +330,7 @@ def views( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, columns=columns, schema=schema, load_extension=load_extension, @@ -1857,6 +1867,7 @@ def query( table, fmt, json_cols, + ascii_, raw, raw_lines, param, @@ -1895,6 +1906,7 @@ def query( nl, arrays, json_cols, + ascii_, ) @@ -1969,6 +1981,7 @@ def memory( table, fmt, json_cols, + ascii_, raw, raw_lines, param, @@ -2108,6 +2121,7 @@ def memory( nl, arrays, json_cols, + ascii_, ) @@ -2125,6 +2139,7 @@ def _execute_query( nl, arrays, json_cols, + ascii_, ): with db.conn: try: @@ -2167,7 +2182,7 @@ def _execute_query( for row in cursor: writer.writerow(row) else: - for line in output_rows(cursor, headers, nl, arrays, json_cols): + for line in output_rows(cursor, headers, nl, arrays, json_cols, ascii_): click.echo(line) @@ -2211,6 +2226,7 @@ def search( table, fmt, json_cols, + ascii_, load_extension, ): """Execute a full-text search against this table @@ -2257,6 +2273,7 @@ def search( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, param=[("query", q)], load_extension=load_extension, ) @@ -2317,6 +2334,7 @@ def rows( table, fmt, json_cols, + ascii_, load_extension, ): """Output all rows in the specified table @@ -2351,6 +2369,7 @@ def rows( fmt=fmt, param=param, json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -2377,6 +2396,7 @@ def triggers( table, fmt, json_cols, + ascii_, load_extension, ): """Show triggers configured in this database @@ -2406,6 +2426,7 @@ def triggers( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -2434,6 +2455,7 @@ def indexes( table, fmt, json_cols, + ascii_, load_extension, ): """Show indexes for the whole database or specific tables @@ -2475,6 +2497,7 @@ def indexes( table=table, fmt=fmt, json_cols=json_cols, + ascii_=ascii_, load_extension=load_extension, ) @@ -3112,7 +3135,7 @@ def convert( if multi: def preview(v): - return json.dumps(fn(v), default=repr) if v else v + return json.dumps(fn(v), default=repr, ensure_ascii=False) if v else v else: @@ -3458,7 +3481,7 @@ def migrate(db_path, migrations, stop_before, list_, verbose): @cli.command(name="plugins") def plugins_list(): "List installed plugins" - click.echo(json.dumps(get_plugins(), indent=2)) + click.echo(json.dumps(get_plugins(), indent=2, ensure_ascii=False)) ensure_plugins_loaded() @@ -3505,7 +3528,7 @@ FILE_COLUMNS = { } -def output_rows(iterator, headers, nl, arrays, json_cols): +def output_rows(iterator, headers, nl, arrays, json_cols, ascii_=False): # Duplicate column names would collide as dictionary keys, so rename # later occurrences id, id -> id, id_2 - CSV and table output keep # the original duplicate headers since they never build dictionaries @@ -3526,7 +3549,7 @@ def output_rows(iterator, headers, nl, arrays, json_cols): data = dict(zip(headers, data)) line = "{firstchar}{serialized}{maybecomma}{lastchar}".format( firstchar=("[" if first else " ") if not nl else "", - serialized=json.dumps(data, default=json_binary), + serialized=json.dumps(data, default=json_binary, ensure_ascii=ascii_), maybecomma="," if (not nl and not is_last) else "", lastchar="]" if (is_last and not nl) else "", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index d11c42f..9e17969 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1047,6 +1047,34 @@ def test_query_json_with_json_cols(db_path): assert expected == result_rows.output.strip() +def test_query_json_unicode_not_escaped_by_default(db_path): + db = Database(db_path) + with db.conn: + db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text"]) + assert result.exit_code == 0 + assert result.output.strip() == '[{"id": 1, "text": "Japanese 日本語"}]' + # Same for --nl + result = CliRunner().invoke(cli.cli, [db_path, "select id, text from text", "--nl"]) + assert result.exit_code == 0 + assert result.output.strip() == '{"id": 1, "text": "Japanese 日本語"}' + + +@pytest.mark.parametrize("command", ["query", "rows"]) +def test_query_json_ascii_option(db_path, command): + db = Database(db_path) + with db.conn: + db["text"].insert({"id": 1, "text": "Japanese 日本語"}, pk="id") + if command == "query": + args = [db_path, "select id, text from text", "--ascii"] + else: + args = ["rows", db_path, "text", "--ascii"] + result = CliRunner().invoke(cli.cli, args) + assert result.exit_code == 0 + expected = '[{"id": 1, "text": "Japanese ' + "\\u65e5\\u672c\\u8a9e" + '"}]' + assert result.output.strip() == expected + + @pytest.mark.parametrize( "content,is_binary", [(b"\x00\x0fbinary", True), ("this is text", False), (1, False), (1.5, False)], diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 443e72c..6c3f5c5 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -215,6 +215,25 @@ def test_convert_multi_dryrun(test_db_and_path): ) +def test_convert_multi_dryrun_unicode_not_escaped(test_db_and_path): + db_path = test_db_and_path[1] + result = CliRunner().invoke( + cli.cli, + [ + "convert", + db_path, + "example", + "dt", + "{'text': 'Japanese 日本語'}", + "--dry-run", + "--multi", + ], + ) + assert result.exit_code == 0 + # Preview should match what jsonify_if_needed() would actually store + assert '{"text": "Japanese 日本語"}' in result.output + + @pytest.mark.parametrize("drop", (True, False)) def test_convert_output_column(test_db_and_path, drop): db, db_path = test_db_and_path From 6225eba5c8daf97a5e459e9fcce2e46bcd022f50 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 20:13:52 -0700 Subject: [PATCH 082/120] Raise InvalidColumns on insert(pk="invalid") Closes #732 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 18 ++++++++++++++++++ tests/test_create.py | 16 ++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 556e497..a68a9b6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) +- ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) .. _v4_0rc3: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0807892..68aee3e 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -4229,6 +4229,24 @@ class Table(Queryable): if hash_id: pk = hash_id + if pk and not hash_id and self.exists(): + pk_cols = [pk] if isinstance(pk, str) else list(pk) + existing_columns = self.columns_dict + missing_pk_cols = [ + col + for col in pk_cols + if resolve_casing(col, existing_columns) not in existing_columns + ] + if missing_pk_cols: + raise InvalidColumns( + "Invalid primary key column{} {} for table {} with columns {}".format( + "s" if len(missing_pk_cols) > 1 else "", + missing_pk_cols, + self.name, + list(existing_columns), + ) + ) + if ignore and replace: raise ValueError("Use either ignore=True or replace=True, not both") all_columns = [] diff --git a/tests/test_create.py b/tests/test_create.py index decefcf..7fbd7d5 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -3,6 +3,7 @@ from sqlite_utils.db import ( Database, DescIndex, AlterError, + InvalidColumns, NoObviousTable, OperationalError, ForeignKey, @@ -925,6 +926,21 @@ def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db assert rows == [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] +@pytest.mark.parametrize("num_rows", (0, 1, 2, 3, 10)) +def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): + # https://github.com/simonw/sqlite-utils/issues/732 + fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") + rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + + with pytest.raises(InvalidColumns) as ex: + fresh_db["t"].insert_all(rows, pk="not_a_column", alter=True) + + assert ex.value.args == ( + "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", + ) + assert fresh_db["t"].count == 0 + + def test_insert_ignore(fresh_db): fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") # Should raise an error if we try this again From 221774f25a53c76874b28af1c2dd170c76bbd9f1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 20:22:08 -0700 Subject: [PATCH 083/120] Fix for table.insert(..., pk=..., ignore=True), closes #554 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 56 ++++++++++++++++++++++++++++++++++++-------- tests/test_create.py | 18 ++++++++++++++ 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a68a9b6..2b2e5cf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased - JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) - ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) +- Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) .. _v4_0rc3: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 68aee3e..916d2f0 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -4386,18 +4386,54 @@ class Table(Queryable): if num_records_processed == 1: # For an insert we need to use result.lastrowid if not upsert and result is not None: - self.last_rowid = result.lastrowid - if (hash_id or pk) and self.last_rowid: - # Set self.last_pk to the pk(s) for that rowid - row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] - if hash_id: - self.last_pk = row[hash_id] - elif isinstance(pk, str): - self.last_pk = row[resolve_casing(pk, row)] + ignored_insert = ignore and result.rowcount == 0 + if ignored_insert: + if list_mode: + first_record_list = cast(Sequence[Any], first_record) + if hash_id: + pass + elif isinstance(pk, str): + pk_index = column_names.index( + resolve_casing(pk, column_names) + ) + self.last_pk = first_record_list[pk_index] + elif pk: + self.last_pk = tuple( + first_record_list[ + column_names.index(resolve_casing(p, column_names)) + ] + for p in pk + ) else: - self.last_pk = tuple(row[resolve_casing(p, row)] for p in pk) + first_record_dict = cast(Dict[str, Any], first_record) + if hash_id: + self.last_pk = hash_record( + first_record_dict, hash_id_columns + ) + elif isinstance(pk, str): + self.last_pk = first_record_dict[ + resolve_casing(pk, first_record_dict) + ] + elif pk: + self.last_pk = tuple( + first_record_dict[resolve_casing(p, first_record_dict)] + for p in pk + ) else: - self.last_pk = self.last_rowid + self.last_rowid = result.lastrowid + if (hash_id or pk) and self.last_rowid: + # Set self.last_pk to the pk(s) for that rowid + row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] + if hash_id: + self.last_pk = row[hash_id] + elif isinstance(pk, str): + self.last_pk = row[resolve_casing(pk, row)] + else: + self.last_pk = tuple( + row[resolve_casing(p, row)] for p in pk + ) + else: + self.last_pk = self.last_rowid else: # For an upsert use first_record from earlier if list_mode: diff --git a/tests/test_create.py b/tests/test_create.py index 7fbd7d5..2788920 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -953,6 +953,24 @@ def test_insert_ignore(fresh_db): assert rows == [{"id": 1, "bar": 2}] +def test_insert_ignore_with_pk_after_other_table_insert(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/554 + user = {"id": "abc", "name": "david"} + + fresh_db["users"].insert(user, pk="id") + fresh_db["comments"].insert_all( + [ + {"id": "def", "text": "ok"}, + {"id": "ghi", "text": "great"}, + ], + ) + + table = fresh_db["users"].insert(user, pk="id", ignore=True) + + assert table.last_pk == "abc" + assert list(fresh_db["users"].rows) == [user] + + def test_insert_hash_id(fresh_db): dogs = fresh_db["dogs"] id = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk From b8aa1368571b09f765af37783579224f74812bf3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 20:37:02 -0700 Subject: [PATCH 084/120] Remove beanbag-docutils - we stopped needing that in 5f81752 --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bff9390..01f1735 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,6 @@ dev = [ "tabulate>=0.10.0", ] docs = [ - "beanbag-docutils>=2.0", "codespell", "furo", "pygments-csv-lexer", From 2616dec7957a366da3260cf87eb79c78bae39d14 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 20:39:29 -0700 Subject: [PATCH 085/120] .extract() and .lookup() no longer extract null values, closes #186 - table.extract() and the sqlite-utils extract command now skip rows where every extracted column is null: the new foreign key column is left null and no all-null record is added to the lookup table. Rows with at least one non-null extracted column are extracted as before. - The extracts= option to insert() and friends keeps None values as null instead of creating a lookup record for them - previously each insert batch added a duplicate null row to the lookup table. - table.lookup() compares values using IS rather than = so lookup values containing None match existing rows correctly, instead of inserting a duplicate row on every call. Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 3 ++ docs/cli.rst | 2 ++ docs/python-api.rst | 6 ++++ docs/upgrading.rst | 2 ++ sqlite_utils/db.py | 18 +++++++--- tests/test_extract.py | 80 ++++++++++++++++++++++++++++++++++++++++-- tests/test_extracts.py | 48 +++++++++++++++++++++++++ tests/test_lookup.py | 24 +++++++++++++ 8 files changed, 176 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2b2e5cf..e773b11 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,9 @@ Unreleased ---------- +- **Breaking change**: ``table.extract()`` - and the ``sqlite-utils extract`` command - no longer extract rows where every extracted column is ``null``. Those rows now keep a ``null`` value in the new foreign key column instead of pointing at an all-``null`` record in the lookup table. When extracting multiple columns, rows are still extracted if at least one of the columns has a value. (:issue:`186`) +- The ``extracts=`` option to ``table.insert()`` and friends no longer creates a lookup table record for ``None`` values - the column value stays ``null``. Previously every batch of inserted rows containing a ``None`` value would add a duplicate ``null`` record to the lookup table. +- Fixed a bug where ``table.lookup()`` inserted a duplicate row on every call if any of the lookup values were ``None``. Lookup values are now compared using ``IS`` so that ``None`` values match existing rows correctly. - JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) - ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) - Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) diff --git a/docs/cli.rst b/docs/cli.rst index 957ee64..1f95bbd 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -2286,6 +2286,8 @@ The ``sqlite-utils extract`` command can be used to extract specified columns in Take a look at the Python API documentation for :ref:`python_api_extract` for a detailed description of how this works, including examples of table schemas before and after running an extraction operation. +Rows where every extracted column is ``null`` are not extracted - those rows get a ``null`` value in their new foreign key column and no record is created for them in the lookup table. + The command takes a database, table and one or more columns that should be extracted. To extract the ``species`` column from the ``trees`` table you would run: .. code-block:: bash diff --git a/docs/python-api.rst b/docs/python-api.rst index 7ac3951..0e61cd4 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1279,6 +1279,8 @@ To create a species record with a note on when it was first seen, you can use th The first time this is called the record will be created for ``name="Palm"``. Any subsequent calls with that name will ignore the second argument, even if it includes different values. +``None`` values are matched correctly: calling ``.lookup()`` a second time with the same values will return the primary key of the existing row even if some of those values are ``None``. + ``.lookup()`` also accepts keyword arguments, which are passed through to the :ref:`insert() method ` and can be used to influence the shape of the created table. Supported parameters are: - ``pk`` - which defaults to ``id`` @@ -1324,6 +1326,8 @@ To extract the ``species`` column out to a separate ``Species`` table, you can d "species": "Common Juniper" }, extracts={"species": "Species"}) +``None`` values are not extracted: no record is created for them in the lookup table and the column value stays ``null``. + .. _python_api_m2m: Working with many-to-many relationships @@ -2022,6 +2026,8 @@ This produces a lookup table like so: "latin" TEXT ) +Rows where every extracted column is ``null`` are not extracted: no record is created for them in the lookup table and their foreign key column is left as ``null``. When extracting multiple columns, rows where at least one of the extracted columns has a value will be extracted as usual. + .. _python_api_hash: Setting an ID based on the hash of the row contents diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 09a9e2f..88e987d 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -77,6 +77,8 @@ Python API changes **table.convert() no longer skips falsey values.** Matching the CLI change above, ``table.convert()`` now converts every value. The ``skip_false`` parameter has been removed - previously it defaulted to ``True``, skipping empty strings and other falsey values. +**Null values are no longer extracted into lookup tables.** ``table.extract()`` and the ``sqlite-utils extract`` command leave rows alone if every extracted column is ``null`` - the new foreign key column is left as ``null`` instead of pointing at an all-``null`` record in the lookup table. The ``extracts=`` insert option similarly keeps ``None`` values as ``null``. Relatedly, ``table.lookup()`` now compares values using ``IS`` so that looking up a value containing ``None`` returns the existing matching row - previously it inserted a duplicate row on every call. + **ensure_autocommit_off() is now ensure_autocommit_on().** The ``db.ensure_autocommit_off()`` context manager has been renamed to ``db.ensure_autocommit_on()``. The old name described the opposite of what the method did: it temporarily puts the connection into driver-level autocommit mode (by setting ``isolation_level = None``), so that statements such as ``PRAGMA journal_mode=wal`` can run outside of an implicit transaction. The behavior is unchanged - update any calls to use the new name. **View.enable_fts() has been removed.** The ``View`` class previously had an ``enable_fts()`` method that existed only to raise ``NotImplementedError`` - full-text search is not supported for views. Calling it now raises ``AttributeError`` like any other missing method. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 916d2f0..0f2924c 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2841,14 +2841,20 @@ class Table(Queryable): ) lookup_columns = [(rename.get(col) or col) for col in columns] lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True) + # Rows where every extracted column is null are left alone - they + # get a null foreign key and no lookup table record, see #186 + all_columns_are_null = " AND ".join( + "{} IS NULL".format(quote_identifier(c)) for c in columns + ) self.db.execute( - "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {}".format( + "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {} WHERE NOT ({all_null})".format( quote_identifier(table), quote_identifier(self.name), lookup_columns=", ".join( quote_identifier(c) for c in lookup_columns ), table_cols=", ".join(quote_identifier(c) for c in columns), + all_null=all_columns_are_null, ) ) @@ -2857,7 +2863,7 @@ class Table(Queryable): # And populate it self.db.execute( - "UPDATE {} SET {} = (SELECT id FROM {} WHERE {where})".format( + "UPDATE {} SET {} = (SELECT id FROM {} WHERE {where}) WHERE NOT ({all_null})".format( quote_identifier(self.name), quote_identifier(magic_lookup_column), quote_identifier(table), @@ -2870,6 +2876,7 @@ class Table(Queryable): ) for column in columns ), + all_null=all_columns_are_null, ) ) # Figure out the right column order @@ -3858,7 +3865,7 @@ class Table(Queryable): # Only process extracts if there are any if has_extracts: for i, key in enumerate(all_columns): - if key in extracts: + if key in extracts and record_values[i] is not None: record_values[i] = self.db.table(extracts[key]).lookup( {"value": record_values[i]} ) @@ -3878,7 +3885,7 @@ class Table(Queryable): ), ) ) - if key in extracts: + if key in extracts and value is not None: extract_table = extracts[key] value = self.db.table(extract_table).lookup({"value": value}) record_values.append(value) @@ -4615,8 +4622,9 @@ class Table(Queryable): fold_identifier_case(c) for c in lookup_values } not in unique_column_sets: self.create_index(lookup_values.keys(), unique=True) + # IS rather than = so that null values are matched correctly wheres = [ - "{} = ?".format(quote_identifier(column)) for column in lookup_values + "{} IS ?".format(quote_identifier(column)) for column in lookup_values ] rows = list( self.rows_where( diff --git a/tests/test_extract.py b/tests/test_extract.py index d24c597..1c0fa01 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -189,9 +189,85 @@ def test_extract_works_with_null_values(fresh_db): ) assert list(fresh_db["listens"].rows) == [ {"id": 1, "track_title": "foo", "album_id": 1}, - {"id": 2, "track_title": "baz", "album_id": 2}, + {"id": 2, "track_title": "baz", "album_id": None}, ] assert list(fresh_db["albums"].rows) == [ {"id": 1, "album_title": "bar"}, - {"id": 2, "album_title": None}, + ] + + +def test_extract_null_values_single_column(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/186 + fresh_db["species"].insert({"id": 1, "species": "Wolf"}, pk="id") + fresh_db["individuals"].insert_all( + [ + {"id": 10, "name": "Terriana", "species": "Fox"}, + {"id": 11, "name": "Spenidorm", "species": None}, + {"id": 12, "name": "Grantheim", "species": "Wolf"}, + {"id": 13, "name": "Turnutopia", "species": None}, + {"id": 14, "name": "Wargal", "species": "Wolf"}, + ], + pk="id", + ) + fresh_db["individuals"].extract("species") + # No null row should have been added to species + assert list(fresh_db["species"].rows) == [ + {"id": 1, "species": "Wolf"}, + {"id": 2, "species": "Fox"}, + ] + assert list(fresh_db["individuals"].rows) == [ + {"id": 10, "name": "Terriana", "species_id": 2}, + {"id": 11, "name": "Spenidorm", "species_id": None}, + {"id": 12, "name": "Grantheim", "species_id": 1}, + {"id": 13, "name": "Turnutopia", "species_id": None}, + {"id": 14, "name": "Wargal", "species_id": 1}, + ] + + +def test_extract_null_values_multiple_columns(fresh_db): + # A row should be extracted if at least one column is not null - + # only rows where ALL extracted columns are null are left alone + fresh_db["circulation"].insert_all( + [ + {"id": 1, "title": "title one", "creator": "creator one", "year": 2018}, + {"id": 2, "title": "title two", "creator": None, "year": 2019}, + {"id": 3, "title": None, "creator": None, "year": 2020}, + {"id": 4, "title": None, "creator": None, "year": 2021}, + ], + pk="id", + ) + fresh_db["circulation"].extract( + ["title", "creator"], table="books", fk_column="book_id" + ) + assert list(fresh_db["books"].rows) == [ + {"id": 1, "title": "title one", "creator": "creator one"}, + {"id": 2, "title": "title two", "creator": None}, + ] + assert list(fresh_db["circulation"].rows) == [ + {"id": 1, "book_id": 1, "year": 2018}, + {"id": 2, "book_id": 2, "year": 2019}, + {"id": 3, "book_id": None, "year": 2020}, + {"id": 4, "book_id": None, "year": 2021}, + ] + + +def test_extract_null_values_existing_lookup_table_with_null_row(fresh_db): + # Even if the lookup table already contains an all-null row, rows where + # every extracted column is null should keep a null foreign key + fresh_db["species"].insert({"id": 1, "species": None}, pk="id") + fresh_db["individuals"].insert_all( + [ + {"id": 10, "name": "Terriana", "species": "Fox"}, + {"id": 11, "name": "Spenidorm", "species": None}, + ], + pk="id", + ) + fresh_db["individuals"].extract("species") + assert list(fresh_db["species"].rows) == [ + {"id": 1, "species": None}, + {"id": 2, "species": "Fox"}, + ] + assert list(fresh_db["individuals"].rows) == [ + {"id": 10, "name": "Terriana", "species_id": 2}, + {"id": 11, "name": "Spenidorm", "species_id": None}, ] diff --git a/tests/test_extracts.py b/tests/test_extracts.py index eb4f37e..7add79a 100644 --- a/tests/test_extracts.py +++ b/tests/test_extracts.py @@ -67,3 +67,51 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): {"id": 2, "species_id": 1}, {"id": 3, "species_id": 2}, ] == list(fresh_db["Trees"].rows) + + +def test_extracts_null_values(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/186 + # Null values should stay null, not be extracted into the lookup table + fresh_db["Trees"].insert_all( + [ + {"id": 1, "species_id": "Oak"}, + {"id": 2, "species_id": None}, + {"id": 3, "species_id": "Palm"}, + {"id": 4, "species_id": None}, + ], + extracts={"species_id": "Species"}, + ) + assert list(fresh_db["Species"].rows) == [ + {"id": 1, "value": "Oak"}, + {"id": 2, "value": "Palm"}, + ] + assert list(fresh_db["Trees"].rows) == [ + {"id": 1, "species_id": 1}, + {"id": 2, "species_id": None}, + {"id": 3, "species_id": 2}, + {"id": 4, "species_id": None}, + ] + + +def test_extracts_null_values_list_mode(fresh_db): + # Same as test_extracts_null_values but for list-based records + fresh_db["Trees"].insert_all( + [ + ["id", "species_id"], + [1, "Oak"], + [2, None], + [3, "Palm"], + [4, None], + ], + extracts={"species_id": "Species"}, + ) + assert list(fresh_db["Species"].rows) == [ + {"id": 1, "value": "Oak"}, + {"id": 2, "value": "Palm"}, + ] + assert list(fresh_db["Trees"].rows) == [ + {"id": 1, "species_id": 1}, + {"id": 2, "species_id": None}, + {"id": 3, "species_id": 2}, + {"id": 4, "species_id": None}, + ] diff --git a/tests/test_lookup.py b/tests/test_lookup.py index a36b464..da4f18b 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -157,3 +157,27 @@ def test_lookup_with_extra_insert_parameters(fresh_db): def test_lookup_new_table_strict(fresh_db, strict): fresh_db["species"].lookup({"name": "Palm"}, strict=strict) assert fresh_db["species"].strict == strict or not fresh_db.supports_strict + + +def test_lookup_null_value_idempotent(fresh_db): + # https://github.com/simonw/sqlite-utils/issues/186 + # Repeated lookups of a null value should return the same row, + # not insert a duplicate row each time + species = fresh_db["species"] + first_id = species.lookup({"name": None}) + second_id = species.lookup({"name": None}) + assert first_id == second_id + assert list(species.rows) == [{"id": first_id, "name": None}] + + +def test_lookup_compound_key_with_null_idempotent(fresh_db): + species = fresh_db["species"] + palm_id = species.lookup({"name": "Palm", "type": None}) + oak_id = species.lookup({"name": "Oak", "type": "Tree"}) + assert palm_id == species.lookup({"name": "Palm", "type": None}) + assert oak_id == species.lookup({"name": "Oak", "type": "Tree"}) + assert palm_id != oak_id + assert list(species.rows) == [ + {"id": palm_id, "name": "Palm", "type": None}, + {"id": oak_id, "name": "Oak", "type": "Tree"}, + ] From f2fbcf60d8bbc95ca996fd078b42fc60831c4f6c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:03:15 -0700 Subject: [PATCH 086/120] --no-headers fix in changelog, refs #566 --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e773b11..14e26a1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ Unreleased - The ``extracts=`` option to ``table.insert()`` and friends no longer creates a lookup table record for ``None`` values - the column value stays ``null``. Previously every batch of inserted rows containing a ``None`` value would add a duplicate ``null`` record to the lookup table. - Fixed a bug where ``table.lookup()`` inserted a duplicate row on every call if any of the lookup values were ``None``. Lookup values are now compared using ``IS`` so that ``None`` values match existing rows correctly. - JSON output from the command-line tool no longer escapes non-ASCII characters, so ``sqlite-utils data.db "select '日本語' as text"`` now outputs ``[{"text": "日本語"}]``. This matches how values were already stored by ``insert`` and how CSV/TSV output already behaved. A new ``--ascii`` option restores the previous behavior of escaping non-ASCII characters, for output destinations that cannot handle UTF-8 - see :ref:`cli_query_json_ascii`. The option is available on the ``query``, ``rows``, ``search``, ``tables``, ``views``, ``triggers``, ``indexes`` and ``memory`` commands. The ``convert --multi --dry-run`` preview and ``plugins`` output also no longer escape non-ASCII characters. (:issue:`625`) +- ``--no-headers`` now omits the header row from ``--fmt`` and ``--table`` output, not just CSV and TSV output. (:issue:`566`) - ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) - Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) From 7d861181687e88194931c52ebfd57eb31f34c8d8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:19:20 -0700 Subject: [PATCH 087/120] Fix failed db.execute() write leaves a phantom transaction open Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 16 ++++++++++++---- tests/test_atomic.py | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 14e26a1..3bdab36 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,7 @@ Unreleased - ``--no-headers`` now omits the header row from ``--fmt`` and ``--table`` output, not just CSV and TSV output. (:issue:`566`) - ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) - Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) +- Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. .. _v4_0rc3: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0f2924c..534f145 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -870,10 +870,18 @@ class Database: if self._tracer: self._tracer(sql, parameters) was_in_transaction = self.conn.in_transaction - if parameters is not None: - cursor = self.conn.execute(sql, parameters) - else: - cursor = self.conn.execute(sql) + try: + if parameters is not None: + cursor = self.conn.execute(sql, parameters) + else: + cursor = self.conn.execute(sql) + except Exception: + if not was_in_transaction and self.conn.in_transaction: + # The failed statement opened an implicit transaction that + # nothing would ever commit - roll it back, otherwise it + # would capture every subsequent write + self.conn.execute("ROLLBACK") + raise if ( not was_in_transaction and self.conn.in_transaction diff --git a/tests/test_atomic.py b/tests/test_atomic.py index f75c4ee..1a4b4ae 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -258,6 +258,47 @@ def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): assert [r["id"] for r in fresh_db["t"].rows] == [1] +def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir): + # A failed write must not leave the driver's implicit transaction open - + # that would silently disable auto-commit for every subsequent write + path = str(tmpdir / "test.db") + db = Database(path) + db["t"].insert({"id": 1}, pk="id") + with pytest.raises(sqlite3.IntegrityError): + db.execute("insert into t (id) values (1)") + assert not db.conn.in_transaction + # Subsequent writes commit as normal and survive closing the connection + db["other"].insert({"id": 2}) + db.close() + db2 = Database(path) + assert db2["other"].exists() + db2.close() + + +def test_execute_failed_write_preserves_explicit_transaction(fresh_db): + # A failed write inside an explicit transaction must not roll back + # the caller's earlier work - only the caller decides that + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.begin() + fresh_db.execute("insert into t (id) values (2)") + with pytest.raises(sqlite3.IntegrityError): + fresh_db.execute("insert into t (id) values (1)") + assert fresh_db.conn.in_transaction + fresh_db.commit() + assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] + + +def test_execute_failed_write_inside_atomic_preserves_block(fresh_db): + # A caught failure inside an atomic() block must leave the block's + # transaction open so its other work still commits + fresh_db["t"].insert({"id": 1}, pk="id") + with fresh_db.atomic(): + fresh_db.execute("insert into t (id) values (2)") + with pytest.raises(sqlite3.IntegrityError): + fresh_db.execute("insert into t (id) values (1)") + assert [r["id"] for r in fresh_db["t"].rows] == [1, 2] + + def test_query_returning_commits_after_iteration(tmpdir): if sqlite3.sqlite_version_info < (3, 35, 0): import pytest as _pytest From adc10df98102c76c86c77108615792ba238a0ae3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:22:29 -0700 Subject: [PATCH 088/120] Fix for db.query("; COMMIT") bypasses the first-token scanner Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 11 ++++++----- tests/test_atomic.py | 13 +++++++++++++ tests/test_query.py | 27 +++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3bdab36..30ad989 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,7 @@ Unreleased - ``table.insert_all(..., pk=...)`` now raises ``InvalidColumns`` if ``pk=`` names columns that do not exist in an existing table. Previously this behaved inconsistently, with single-row inserts raising a ``KeyError`` while other row counts succeeded. (:issue:`732`) - Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. +- Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. .. _v4_0rc3: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 534f145..bd023c5 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -436,14 +436,15 @@ _QUERY_REJECTED_KEYWORDS = _TRANSACTION_CONTROL_KEYWORDS | { def _first_keyword(sql: str) -> str: """ - Return the first keyword of a SQL statement, uppercased, skipping any - leading whitespace and ``--`` or ``/* ... */`` comments - the only - things SQLite's tokenizer allows before the first token. Returns an - empty string if there is no leading keyword. + Return the first keyword of a SQL statement, uppercased, skipping + everything the sqlite3 driver tolerates before the first real token: + whitespace, ``--`` or ``/* ... */`` comments, empty statements + (bare ``;``) and a UTF-8 byte order mark. Returns an empty string if + there is no leading keyword. """ i, n = 0, len(sql) while i < n: - if sql[i].isspace(): + if sql[i].isspace() or sql[i] in (";", "\ufeff"): i += 1 elif sql.startswith("--", i): newline = sql.find("\n", i) diff --git a/tests/test_atomic.py b/tests/test_atomic.py index 1a4b4ae..0d25b84 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -258,6 +258,19 @@ def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): assert [r["id"] for r in fresh_db["t"].rows] == [1] +@pytest.mark.parametrize("begin_sql", ["; begin", "\ufeffbegin"]) +def test_execute_prefixed_begin_leaves_transaction_open(fresh_db, begin_sql): + # sqlite3 tolerates empty statements and a UTF-8 BOM before the first + # real token, so a BEGIN behind either must not be auto-committed + # out from under the caller + fresh_db["t"].insert({"id": 1}, pk="id") + fresh_db.execute(begin_sql) + assert fresh_db.conn.in_transaction + fresh_db.execute("insert into t (id) values (2)") + fresh_db.rollback() + assert [r["id"] for r in fresh_db["t"].rows] == [1] + + def test_execute_failed_write_rolls_back_implicit_transaction(tmpdir): # A failed write must not leave the driver's implicit transaction open - # that would silently disable auto-commit for every subsequent write diff --git a/tests/test_query.py b/tests/test_query.py index 0b9f2ae..c2f6731 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -59,6 +59,10 @@ def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db): "-- comment\nbegin", "/* multi\nline */ -- and another\n vacuum", "\t /* a */ /* b */ savepoint s1", + "; commit", + ";;\n ; rollback", + "; /* comment */ vacuum", + "\ufeffbegin", ], ) def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql): @@ -83,6 +87,23 @@ def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db): assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] +@pytest.mark.parametrize("sql", ["; COMMIT", "\ufeffCOMMIT"]) +def test_query_prefixed_commit_does_not_commit_transaction(fresh_db, sql): + # sqlite3 tolerates empty statements and a UTF-8 BOM before the first + # real token, so the keyword scanner must skip them too - previously + # '; COMMIT' slipped past the check and committed the caller's open + # transaction before raising OperationalError + fresh_db["dogs"].insert({"name": "Cleo"}) + fresh_db.begin() + fresh_db.execute("insert into dogs (name) values ('Pancakes')") + with pytest.raises(ValueError): + fresh_db.query(sql) + # The explicit transaction is still open and can still be rolled back + assert fresh_db.conn.in_transaction + fresh_db.rollback() + assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"] + + def test_query_error_leaves_no_transaction_open(fresh_db): with pytest.raises(sqlite3.OperationalError): fresh_db.query("select * from missing_table") @@ -137,6 +158,12 @@ def test_query_comment_prefixed_pragma_inside_transaction(fresh_db): ("", ""), (" ", ""), ("123", ""), + ("; commit", "COMMIT"), + (";;\n ; rollback", "ROLLBACK"), + ("; -- comment\n begin", "BEGIN"), + ("\ufeffcommit", "COMMIT"), + ("\ufeff ; select 1", "SELECT"), + (";", ""), ], ) def test_first_keyword(sql, expected): From 66934918c689238b19a74b2f09794002cc985094 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:30:47 -0700 Subject: [PATCH 089/120] Document that rejected write PRAGMAs in db.query() still take effect db.query() promises that a statement rejected with ValueError is rolled back and has no effect. PRAGMA statements are the exception: some of them refuse to run inside a transaction, so they execute outside the savepoint guard - a row-less PRAGMA such as "PRAGMA user_version = 5" therefore takes effect despite the ValueError. Documenting this as a known limitation rather than fixing it, since a fix would need a hardcoded list of row-returning PRAGMAs. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/python-api.rst | 2 ++ sqlite_utils/db.py | 5 ++++- tests/test_query.py | 12 ++++++++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 30ad989..027765c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -18,6 +18,7 @@ Unreleased - Fixed an ``IndexError`` from ``table.insert(..., pk=..., ignore=True)`` when an ignored insert followed writes to another table on the same connection. ``last_pk`` is now populated from the explicit primary key value instead of looking up a stale ``lastrowid``. (:issue:`554`) - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. +- Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. .. _v4_0rc3: diff --git a/docs/python-api.rst b/docs/python-api.rst index 0e61cd4..516e2aa 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -233,6 +233,8 @@ The SQL query is executed as soon as ``db.query()`` is called. The resulting row ``db.query()`` can only be used with SQL that returns rows. Passing a statement that returns no rows - an ``INSERT`` or ``UPDATE`` without a ``RETURNING`` clause, for example - will raise a ``ValueError``. The rejected statement is rolled back, so it has no effect on the database. Use :ref:`db.execute() ` for those statements instead. +There is one exception to the rolled-back guarantee: a ``PRAGMA`` statement that returns no rows, such as ``PRAGMA user_version = 5``, still raises a ``ValueError`` but will already have taken effect. Some PRAGMA statements refuse to run inside a transaction, so PRAGMAs are executed outside the savepoint that is used to roll back other rejected statements. Use ``db.execute()`` for PRAGMA statements that do not return rows. + If a query returns more than one column with the same name - a join between two tables that share column names, for example - later occurrences are renamed with a numeric suffix, so every value is included in the dictionary: .. code-block:: python diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index bd023c5..f6153cd 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -797,7 +797,10 @@ class Database: parameters, or a dictionary for ``where id = :id`` :raises ValueError: if the SQL statement does not return rows - use :meth:`execute` for those statements instead. The rejected statement - is rolled back, so it has no effect on the database + is rolled back, so it has no effect on the database. One exception: + a row-less ``PRAGMA`` statement takes effect despite the + ``ValueError``, because PRAGMAs run outside the savepoint guard - + some of them refuse to run inside a transaction """ message = ( "query() can only be used with SQL that returns rows - " diff --git a/tests/test_query.py b/tests/test_query.py index c2f6731..f4aa336 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -122,6 +122,18 @@ def test_query_pragma(tmpdir): db.close() +def test_query_rejected_pragma_still_takes_effect(fresh_db): + # Documented limitation: PRAGMAs run outside the savepoint guard, + # because some of them refuse to run inside a transaction - so a + # row-less PRAGMA takes effect even though it raises ValueError. + # If this test starts failing because the pragma was rolled back, + # the limitation has been fixed - update the docs in python-api.rst + # and the query() docstring to remove the carve-out + with pytest.raises(ValueError): + fresh_db.query("pragma user_version = 5") + assert fresh_db.execute("pragma user_version").fetchone()[0] == 5 + + def test_query_comment_prefixed_pragma(tmpdir): from sqlite_utils import Database From 8e015d024c34f2a5e5059e0f10fcdf41410613fe Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:34:02 -0700 Subject: [PATCH 090/120] Compound primary keys now resolve in PRIMARY KEY declaration order PRAGMA table_info sets is_pk to the 1-based position of each column within the PRIMARY KEY, which can differ from table column order. table.pks previously returned table column order, so an implicit compound FOREIGN KEY ... REFERENCES other was introspected with its referenced columns inverted, and transform() baked that inverted order into the rewritten schema - failing with IntegrityError on valid data, or silently reversing the constraint with foreign keys off. table.pks, compound foreign key guessing (create, add_foreign_key) and transform() now all use declaration order, and transform() no longer reorders a compound PRIMARY KEY (b, a) into table column order. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 31 +++++++++++++++------- tests/test_foreign_keys.py | 54 ++++++++++++++++++++++++++++++++++++++ tests/test_introspect.py | 15 +++++++++++ 4 files changed, 91 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 027765c..e7c582e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed a bug where compound primary key columns were returned in table column order instead of ``PRIMARY KEY`` declaration order. For a table declared as ``CREATE TABLE other (b TEXT, a TEXT, PRIMARY KEY (a, b))`` an implicit ``FOREIGN KEY (x, y) REFERENCES other`` was introspected as referencing ``(b, a)`` when SQLite resolves it as ``(a, b)`` - running ``transform()`` on such a table then rewrote the schema with the inverted column order, silently reversing the meaning of the constraint and causing foreign key errors on valid data. ``table.pks``, compound foreign key guessing and ``transform()`` now all use the primary key declaration order, and ``transform()`` no longer reorders a compound ``PRIMARY KEY (b, a)`` into table column order. .. _v4_0rc3: diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f6153cd..1e10b11 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2012,10 +2012,9 @@ class Queryable: :param offset: Integer for SQL offset """ column_names = [column.name for column in self.columns] - pks = [column.name for column in self.columns if column.is_pk] - if not pks: + pks = self.pks + if self.use_rowid: column_names.insert(0, "rowid") - pks = ["rowid"] select = ",".join(quote_identifier(column_name) for column_name in column_names) for row in self.rows_where( select=select, @@ -2148,8 +2147,18 @@ class Table(Queryable): @property def pks(self) -> List[str]: - "Primary key columns for this table." - names = [column.name for column in self.columns if column.is_pk] + """ + Primary key columns for this table, in PRIMARY KEY declaration order - + ``PRAGMA table_info`` sets ``is_pk`` to the 1-based position of each + column within the primary key, which can differ from the order of the + columns in the table. SQLite uses the declaration order to resolve + implicit foreign key references, so this order matters. + """ + pk_columns = sorted( + (column for column in self.columns if column.is_pk), + key=lambda column: column.is_pk, + ) + names = [column.name for column in pk_columns] if not names: names = ["rowid"] return names @@ -2673,7 +2682,8 @@ class Table(Queryable): if pk is DEFAULT: pks_renamed = tuple( - rename.get(p.name) or p.name for p in self.columns if p.is_pk + rename.get(pk_name) or pk_name + for pk_name in (self.pks if not self.use_rowid else []) ) if len(pks_renamed) == 1: pk = pks_renamed[0] @@ -3017,7 +3027,10 @@ class Table(Queryable): raise AlterError("table '{}' has no column {}".format(fk, fk_col)) else: # automatically set fk_col to first primary_key of fk table - pks = [c for c in self.db[fk].columns if c.is_pk] + pks = sorted( + (c for c in self.db[fk].columns if c.is_pk), + key=lambda c: c.is_pk, + ) if pks: fk_col = pks[0].name fk_col_type = pks[0].type @@ -3770,9 +3783,7 @@ class Table(Queryable): # First we execute the function pk_to_values = {} new_column_types: Dict[str, Set[type]] = {} - pks = [column.name for column in self.columns if column.is_pk] - if not pks: - pks = ["rowid"] + pks = self.pks with progressbar( length=self.count, silent=not show_progress, label="1: Evaluating" diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 8950189..1672069 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -524,3 +524,57 @@ def test_add_compound_foreign_key_on_delete(courses_db): assert fk.is_compound is True assert fk.on_delete == "SET NULL" assert "ON DELETE SET NULL" in courses_db["courses"].schema + + +def test_implicit_compound_foreign_key_resolves_pk_declaration_order(fresh_db): + # The other table's PRIMARY KEY declares its columns in a different + # order to the table's column order. SQLite resolves the implicit + # "REFERENCES other" using PRIMARY KEY declaration order, so the + # introspected other_columns must too + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db.execute( + "create table child (x text, y text, foreign key (x, y) references other)" + ) + fk = fresh_db["child"].foreign_keys[0] + assert fk.other_columns == ("a", "b") + + +def test_transform_implicit_compound_foreign_key_stays_valid(fresh_db): + # transform() rewrites the implicit FK with explicit columns - they + # must be in PRIMARY KEY declaration order or valid data fails the + # foreign key check with an IntegrityError + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db.execute( + "create table child (x text, y text, foreign key (x, y) references other)" + ) + fresh_db.execute("PRAGMA foreign_keys = ON") + fresh_db["other"].insert({"a": "A", "b": "B"}) + fresh_db["child"].insert({"x": "A", "y": "B"}) + fresh_db["child"].transform(types={"x": str}) + assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + # The constraint still points the right way around + fresh_db["child"].insert({"x": "A", "y": "B"}) + with pytest.raises(sqlite3.IntegrityError): + fresh_db["child"].insert({"x": "B", "y": "A"}) + + +def test_create_compound_foreign_key_guesses_pk_declaration_order(fresh_db): + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db["other"].insert({"a": "A", "b": "B"}) + fresh_db["child"].create( + {"id": int, "x": str, "y": str}, + pk="id", + foreign_keys=[(("x", "y"), "other")], + ) + assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + fresh_db.execute("PRAGMA foreign_keys = ON") + fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}) + with pytest.raises(sqlite3.IntegrityError): + fresh_db["child"].insert({"id": 2, "x": "B", "y": "A"}) + + +def test_add_compound_foreign_key_guesses_pk_declaration_order(fresh_db): + fresh_db.execute("create table other (b text, a text, primary key (a, b))") + fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}, pk="id") + fresh_db["child"].add_foreign_key(("x", "y"), "other") + assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") diff --git a/tests/test_introspect.py b/tests/test_introspect.py index 4ff3f77..8b6765d 100644 --- a/tests/test_introspect.py +++ b/tests/test_introspect.py @@ -321,3 +321,18 @@ def test_table_default_values(fresh_db, value): ) default_values = fresh_db["default_values"].default_values assert default_values == {"value": value} + + +def test_pks_use_primary_key_declaration_order(fresh_db): + # PRIMARY KEY (a, b) declared against columns stored in order (b, a) - + # pks must follow the declaration order, which is what SQLite uses to + # resolve implicit foreign key references and compound pk lookups + fresh_db.execute("create table t (b text, a text, primary key (a, b))") + assert fresh_db["t"].pks == ["a", "b"] + + +def test_transform_preserves_compound_pk_declaration_order(fresh_db): + fresh_db.execute("create table t (a text, b text, c text, primary key (b, a))") + fresh_db["t"].transform(drop={"c"}) + assert fresh_db["t"].pks == ["b", "a"] + assert 'PRIMARY KEY ("b", "a")' in fresh_db["t"].schema From 404e935b6348c57e507fba36eb06daf462257af8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:35:45 -0700 Subject: [PATCH 091/120] ForeignKey is now a frozen dataclass, restoring hashability The namedtuple-to-dataclass change made ForeignKey unhashable, breaking set(table.foreign_keys) and dict-key usage that worked in 3.x. frozen=True restores immutability and hashability. Equality and hashing cover all compared fields including on_delete/on_update - two foreign keys differing only in their actions are different constraints. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/upgrading.rst | 2 +- sqlite_utils/db.py | 24 +++++++++++++++++------- tests/test_foreign_keys.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e7c582e..b1c6108 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``ForeignKey`` objects are hashable again. The 4.0 change from ``namedtuple`` to dataclass accidentally made them unhashable, breaking patterns like ``set(table.foreign_keys)`` that worked in 3.x. ``ForeignKey`` is now a frozen dataclass - immutable and hashable, like the namedtuple was. - Fixed a bug where compound primary key columns were returned in table column order instead of ``PRIMARY KEY`` declaration order. For a table declared as ``CREATE TABLE other (b TEXT, a TEXT, PRIMARY KEY (a, b))`` an implicit ``FOREIGN KEY (x, y) REFERENCES other`` was introspected as referencing ``(b, a)`` when SQLite resolves it as ``(a, b)`` - running ``transform()`` on such a table then rewrote the schema with the inverted column order, silently reversing the meaning of the constraint and causing foreign key errors on valid data. ``table.pks``, compound foreign key guessing and ``transform()`` now all use the primary key declaration order, and ``transform()`` no longer reorders a compound ``PRIMARY KEY (b, a)`` into table column order. .. _v4_0rc3: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 88e987d..92b582a 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -95,7 +95,7 @@ Python API changes for fk in db["courses"].foreign_keys: fk.table, fk.column, fk.other_table, fk.other_column -Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. +Attempting the old unpacking or ``fk[0]`` indexing now raises ``TypeError``, so any code using those patterns will fail loudly rather than silently misbehave. Like the old namedtuple, ``ForeignKey`` instances are immutable and hashable - they can be collected into sets and used as dictionary keys. Note that equality now includes the ``on_delete`` and ``on_update`` actions: a ``ForeignKey`` with ``ON DELETE CASCADE`` is not equal to one without. Compound foreign keys - previously returned as one ``ForeignKey`` per column, misleadingly suggesting several independent single-column keys - are now returned as a single ``ForeignKey`` with ``is_compound=True``. For these the scalar ``column`` and ``other_column`` fields are ``None``; use the ``columns`` and ``other_columns`` tuples instead. Single-column foreign keys are unaffected apart from the class change: ``column``/``other_column`` behave as before and ``columns``/``other_columns`` are one-item tuples. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 1e10b11..e94434a 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -193,7 +193,7 @@ Summary information about a column, see :ref:`python_api_analyze_column`. """ -@dataclass(order=True) +@dataclass(order=True, frozen=True) class ForeignKey: """ A foreign key defined on a table. @@ -208,6 +208,11 @@ class ForeignKey: ``on_delete`` and ``on_update`` hold the foreign key actions, e.g. ``"CASCADE"`` - ``"NO ACTION"`` if not set. + Instances are immutable and hashable, so they can be collected into + sets and used as dictionary keys. Equality covers every compared field, + including ``on_delete`` and ``on_update`` - two foreign keys differing + only in their actions are different constraints. + Prior to sqlite-utils 4.0 this was a ``namedtuple`` and could be unpacked or indexed as ``(table, column, other_table, other_column)``. It is now a dataclass - access its fields by name instead. @@ -227,16 +232,21 @@ class ForeignKey: def __post_init__(self): # Populate columns/other_columns for single-column foreign keys, - # normalizing any lists to tuples + # normalizing any lists to tuples. object.__setattr__ because the + # dataclass is frozen if self.columns: - self.columns = tuple(self.columns) + object.__setattr__(self, "columns", tuple(self.columns)) else: - self.columns = (self.column,) if self.column is not None else () + object.__setattr__( + self, "columns", (self.column,) if self.column is not None else () + ) if self.other_columns: - self.other_columns = tuple(self.other_columns) + object.__setattr__(self, "other_columns", tuple(self.other_columns)) else: - self.other_columns = ( - (self.other_column,) if self.other_column is not None else () + object.__setattr__( + self, + "other_columns", + (self.other_column,) if self.other_column is not None else (), ) diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 1672069..49d3352 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -578,3 +578,35 @@ def test_add_compound_foreign_key_guesses_pk_declaration_order(fresh_db): fresh_db["child"].insert({"id": 1, "x": "A", "y": "B"}, pk="id") fresh_db["child"].add_foreign_key(("x", "y"), "other") assert fresh_db["child"].foreign_keys[0].other_columns == ("a", "b") + + +def test_foreign_keys_are_hashable(fresh_db): + # set() over foreign_keys worked with the 3.x namedtuple and must + # keep working with the dataclass + fresh_db["p"].insert({"id": 1}, pk="id") + fresh_db["c"].insert( + {"id": 1, "pid": 1}, pk="id", foreign_keys=[("pid", "p", "id")] + ) + fks = set(fresh_db["c"].foreign_keys) + assert len(fks) == 1 + assert ForeignKey("c", "pid", "p", "id") in fks + # Usable as dict keys too + assert {fk: True for fk in fks} + + +def test_foreign_key_is_immutable(): + import dataclasses + + fk = ForeignKey("c", "pid", "p", "id") + with pytest.raises(dataclasses.FrozenInstanceError): + fk.table = "other" + + +def test_foreign_key_equality_and_hash_include_actions(): + # Two foreign keys differing only in ON DELETE behavior are different + # constraints - they compare unequal and hash separately + plain = ForeignKey("c", "pid", "p", "id") + cascade = ForeignKey("c", "pid", "p", "id", on_delete="CASCADE") + assert plain != cascade + assert len({plain, cascade}) == 2 + assert plain == ForeignKey("c", "pid", "p", "id") From 29ca9d27e24be818a26fff41450664ef771a03a0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:37:56 -0700 Subject: [PATCH 092/120] foreign_keys= accepts mixed ForeignKey objects, tuples and strings again resolve_foreign_keys() used all-or-nothing isinstance checks, so mixing ForeignKey objects with tuples raised "foreign_keys= should be a list of tuples" - a regression from 3.x, where ForeignKey was a namedtuple and passed the tuple check. Each item is now normalized individually, which also allows bare column-name strings in a mixed list. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 30 ++++++++++++++++-------------- tests/test_foreign_keys.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b1c6108..6e6fa30 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- The ``foreign_keys=`` argument to ``create()`` and ``insert()`` accepts a mixed list of ``ForeignKey`` objects, tuples and column name strings again. In 4.0 pre-releases mixing ``ForeignKey`` objects with tuples raised a ``ValueError`` - a regression from 3.x, where ``ForeignKey`` was a ``namedtuple`` and passed the tuple checks. - ``ForeignKey`` objects are hashable again. The 4.0 change from ``namedtuple`` to dataclass accidentally made them unhashable, breaking patterns like ``set(table.foreign_keys)`` that worked in 3.x. ``ForeignKey`` is now a frozen dataclass - immutable and hashable, like the namedtuple was. - Fixed a bug where compound primary key columns were returned in table column order instead of ``PRIMARY KEY`` declaration order. For a table declared as ``CREATE TABLE other (b TEXT, a TEXT, PRIMARY KEY (a, b))`` an implicit ``FOREIGN KEY (x, y) REFERENCES other`` was introspected as referencing ``(b, a)`` when SQLite resolves it as ``(a, b)`` - running ``transform()`` on such a table then rewrote the schema with the inverted column order, silently reversing the meaning of the constraint and causing foreign key errors on valid data. ``table.pks``, compound foreign key guessing and ``transform()`` now all use the primary key declaration order, and ``transform()`` no longer reorders a compound ``PRIMARY KEY (b, a)`` into table column order. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e94434a..6105a64 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1238,21 +1238,23 @@ class Database: (("campus_name", "dept_code"), "departments", ("campus_name", "dept_code")) """ table = self.table(name) - if all(isinstance(fk, ForeignKey) for fk in foreign_keys): - return cast(List[ForeignKey], foreign_keys) - if all(isinstance(fk, str) for fk in foreign_keys): - # It's a list of columns - fks = [] - for column in foreign_keys: - column = cast(str, column) - other_table = table.guess_foreign_table(column) - other_column = table.guess_foreign_column(other_table) - fks.append(ForeignKey(name, column, other_table, other_column)) - return fks - if not all(isinstance(fk, (tuple, list)) for fk in foreign_keys): - raise ValueError("foreign_keys= should be a list of tuples") fks = [] - for tuple_or_list in cast(Iterable[Sequence[Any]], foreign_keys): + for fk in foreign_keys: + if isinstance(fk, ForeignKey): + fks.append(fk) + continue + if isinstance(fk, str): + # A bare column name - guess the other table and column + other_table = table.guess_foreign_table(fk) + other_column = table.guess_foreign_column(other_table) + fks.append(ForeignKey(name, fk, other_table, other_column)) + continue + if not isinstance(fk, (tuple, list)): + raise ValueError( + "foreign_keys= should be a list of tuples, " + "ForeignKey objects or column name strings" + ) + tuple_or_list = cast(Sequence[Any], fk) if len(tuple_or_list) == 4: if tuple_or_list[0] != name: raise ValueError( diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 49d3352..d7c20f6 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -610,3 +610,36 @@ def test_foreign_key_equality_and_hash_include_actions(): assert plain != cascade assert len({plain, cascade}) == 2 assert plain == ForeignKey("c", "pid", "p", "id") + + +def test_create_table_mixed_foreign_keys_list(fresh_db): + # 3.x accepted a mix of ForeignKey objects, tuples and bare column + # strings in foreign_keys= (ForeignKey was a namedtuple, so it passed + # the tuple check) - keep accepting the mix + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["publishers"].insert({"id": 1}, pk="id") + fresh_db["books"].create( + {"id": int, "author_id": int, "publisher_id": int}, + pk="id", + foreign_keys=[ + ForeignKey("books", "author_id", "authors", "id"), + ("publisher_id", "publishers", "id"), + ], + ) + fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} + assert fks == {"author_id": "authors", "publisher_id": "publishers"} + + +def test_create_table_mixed_foreign_keys_with_string(fresh_db): + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["publishers"].insert({"id": 1}, pk="id") + fresh_db["books"].create( + {"id": int, "author_id": int, "publisher_id": int}, + pk="id", + foreign_keys=[ + "author_id", # bare column, table and column guessed + ("publisher_id", "publishers", "id"), + ], + ) + fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} + assert fks == {"author_id": "authors", "publisher_id": "publishers"} From 1ed95e4ad2676b7f2f0725919bba9c4b051456c4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:42:02 -0700 Subject: [PATCH 093/120] Fix pks_and_rows_where() on views The previous compound-pk-ordering commit switched pks_and_rows_where() to Table-only properties, but the method is defined on Queryable and views exposed it too - calling it on a View raised AttributeError. Restored Queryable-safe logic, and stopped double-quoting the synthesized rowid column: SQLite turns a double-quoted identifier that does not resolve into a string literal, so on a view the generated select "rowid" silently produced the string 'rowid' and a confusing KeyError, where 3.x's [rowid] quoting raised OperationalError cleanly. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 21 ++++++++++++++++----- tests/test_rows.py | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6e6fa30..5d6c901 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed ``pks_and_rows_where()`` raising ``AttributeError`` when called on a view, and no longer double-quotes the synthesized ``rowid`` column in its generated SQL - SQLite turns a double-quoted identifier that does not resolve into a string literal, which on a view produced a confusing ``KeyError`` instead of the ``OperationalError`` raised in 3.x. Compound primary keys returned by this method now follow ``PRIMARY KEY`` declaration order. - The ``foreign_keys=`` argument to ``create()`` and ``insert()`` accepts a mixed list of ``ForeignKey`` objects, tuples and column name strings again. In 4.0 pre-releases mixing ``ForeignKey`` objects with tuples raised a ``ValueError`` - a regression from 3.x, where ``ForeignKey`` was a ``namedtuple`` and passed the tuple checks. - ``ForeignKey`` objects are hashable again. The 4.0 change from ``namedtuple`` to dataclass accidentally made them unhashable, breaking patterns like ``set(table.foreign_keys)`` that worked in 3.x. ``ForeignKey`` is now a frozen dataclass - immutable and hashable, like the namedtuple was. - Fixed a bug where compound primary key columns were returned in table column order instead of ``PRIMARY KEY`` declaration order. For a table declared as ``CREATE TABLE other (b TEXT, a TEXT, PRIMARY KEY (a, b))`` an implicit ``FOREIGN KEY (x, y) REFERENCES other`` was introspected as referencing ``(b, a)`` when SQLite resolves it as ``(a, b)`` - running ``transform()`` on such a table then rewrote the schema with the inverted column order, silently reversing the meaning of the constraint and causing foreign key errors on valid data. ``table.pks``, compound foreign key guessing and ``transform()`` now all use the primary key declaration order, and ``transform()`` no longer reorders a compound ``PRIMARY KEY (b, a)`` into table column order. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6105a64..4c7ef52 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2023,11 +2023,22 @@ class Queryable: :param limit: Integer number of rows to limit to :param offset: Integer for SQL offset """ - column_names = [column.name for column in self.columns] - pks = self.pks - if self.use_rowid: - column_names.insert(0, "rowid") - select = ",".join(quote_identifier(column_name) for column_name in column_names) + # This method is defined on Queryable so it serves views too, which + # have no pks property - sort pk columns into declaration order here + pk_columns = sorted( + (column for column in self.columns if column.is_pk), + key=lambda column: column.is_pk, + ) + pks = [column.name for column in pk_columns] + select_parts = [quote_identifier(column.name) for column in self.columns] + if not pks: + # rowid is left unquoted: it is not a real column, and SQLite + # turns a double-quoted identifier that does not resolve into a + # string literal - on a view that would silently select the + # string 'rowid' instead of raising an error + select_parts.insert(0, "rowid") + pks = ["rowid"] + select = ",".join(select_parts) for row in self.rows_where( select=select, where=where, diff --git a/tests/test_rows.py b/tests/test_rows.py index f050d5a..46417ef 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -111,3 +111,24 @@ def test_rows_where_duplicate_select_columns_are_deduped(fresh_db): fresh_db["t"].insert({"id": 1, "name": "Cleo"}) rows = list(fresh_db["t"].rows_where(select="id, id, name")) assert rows == [{"id": 1, "id_2": 1, "name": "Cleo"}] + + +def test_pks_and_rows_where_view(fresh_db): + # pks_and_rows_where() lives on Queryable so views expose it, but + # SQLite views have no rowid - it has always failed with an + # OperationalError from the generated SQL. Guard against it failing + # earlier with an AttributeError from View lacking Table properties + from sqlite_utils.utils import sqlite3 + + fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db.create_view("dog_names", "select name from dogs") + with pytest.raises(sqlite3.OperationalError): + list(fresh_db["dog_names"].pks_and_rows_where()) + + +def test_pks_and_rows_where_compound_pk_declaration_order(fresh_db): + # Compound pks are returned in PRIMARY KEY declaration order + fresh_db.execute("create table t (b text, a text, primary key (a, b))") + fresh_db["t"].insert({"a": "A", "b": "B"}) + pks_and_rows = list(fresh_db["t"].pks_and_rows_where()) + assert pks_and_rows == [(("A", "B"), {"b": "B", "a": "A"})] From 884574685fde886863dcadb0d297960c88ba5446 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:44:56 -0700 Subject: [PATCH 094/120] insert/upsert --csv no longer rewrites column types of existing tables Type detection is the 4.0 default for CSV/TSV data, and the detected-type transform ran even when the target table already existed - inserting a CSV into a table with a TEXT zip column converted the column to INTEGER, corrupting values with leading zeros ('01234' became 1234) with no warning. Detected types now only apply to tables the command created. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/cli.rst | 2 ++ sqlite_utils/cli.py | 12 +++++++++- tests/test_cli_insert.py | 48 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d6c901..80bcfb7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed a bug where inserting CSV or TSV data into an existing table rewrote that table's column types to match the incoming file. Type detection is the default in 4.0, so ``sqlite-utils insert data.db places places.csv --csv`` against a table with a ``TEXT`` zip code column would convert the column to ``INTEGER`` and corrupt values with leading zeros - ``"01234"`` became ``1234``. Detected types are now only applied when the ``insert`` or ``upsert`` command creates the table. - Fixed ``pks_and_rows_where()`` raising ``AttributeError`` when called on a view, and no longer double-quotes the synthesized ``rowid`` column in its generated SQL - SQLite turns a double-quoted identifier that does not resolve into a string literal, which on a view produced a confusing ``KeyError`` instead of the ``OperationalError`` raised in 3.x. Compound primary keys returned by this method now follow ``PRIMARY KEY`` declaration order. - The ``foreign_keys=`` argument to ``create()`` and ``insert()`` accepts a mixed list of ``ForeignKey`` objects, tuples and column name strings again. In 4.0 pre-releases mixing ``ForeignKey`` objects with tuples raised a ``ValueError`` - a regression from 3.x, where ``ForeignKey`` was a ``namedtuple`` and passed the tuple checks. - ``ForeignKey`` objects are hashable again. The 4.0 change from ``namedtuple`` to dataclass accidentally made them unhashable, breaking patterns like ``set(table.foreign_keys)`` that worked in 3.x. ``ForeignKey`` is now a frozen dataclass - immutable and hashable, like the namedtuple was. diff --git a/docs/cli.rst b/docs/cli.rst index 1f95bbd..063e80f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1326,6 +1326,8 @@ A progress bar is displayed when inserting data from a file. You can hide the pr By default, column types are automatically detected for CSV or TSV files - resulting in a mix of ``TEXT``, ``INTEGER`` and ``REAL`` columns. To disable type detection and treat all columns as ``TEXT``, use the ``--no-detect-types`` option. +Detected types are only applied when the table is created by the command. Inserting CSV or TSV data into a table that already exists leaves the existing column types unchanged - values are inserted using the table's existing schema. + For example, given a ``creatures.csv`` file containing this: .. code-block:: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8ee2091..0ef7cc4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1165,6 +1165,9 @@ def insert_upsert_implementation( db.conn.cursor().executemany(bulk_sql, doc_chunk) return + # table_names() rather than db.table(), which raises NoTable for + # views before the error handling below can deal with them + table_existed_before_insert = table in db.table_names() try: db.table(table).insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs @@ -1194,7 +1197,14 @@ def insert_upsert_implementation( ) else: raise - if tracker is not None and db.table(table).exists(): + # Apply detected types only to a table this command created - + # transforming a pre-existing table would rewrite its column types + # and corrupt values such as TEXT zip codes with leading zeros + if ( + tracker is not None + and not table_existed_before_insert + and db.table(table).exists() + ): db.table(table).transform(types=tracker.types) # Clean up open file-like objects diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 2df1e0c..196590e 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -628,3 +628,51 @@ def test_insert_into_view_errors(tmpdir): ) assert result.exit_code == 1 assert result.output.strip() == "Error: Table v is actually a view" + + +def test_insert_csv_detect_types_leaves_existing_table_alone(db_path): + # Type detection is the default for CSV/TSV inserts, but it must only + # apply to tables created by this command - transforming a pre-existing + # table would rewrite its column types and corrupt data such as + # TEXT zip codes with leading zeros + db = Database(db_path) + db["places"].insert({"name": "Boston", "zip": "01234"}) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "places", "-", "--csv"], + catch_exceptions=False, + input="name,zip\nSF,94107", + ) + assert result.exit_code == 0, result.output + assert db["places"].columns_dict["zip"] is str + assert list(db["places"].rows) == [ + {"name": "Boston", "zip": "01234"}, + {"name": "SF", "zip": "94107"}, + ] + + +def test_insert_csv_detect_types_new_table(db_path): + # A table created by the insert still gets detected types + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "data", "-", "--csv"], + catch_exceptions=False, + input="name,age,weight\nCleo,5,12.5", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert db["data"].columns_dict == {"name": str, "age": int, "weight": float} + + +def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): + db = Database(db_path) + db["places"].insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id") + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "places", "-", "--csv", "--pk", "id"], + catch_exceptions=False, + input="id,name,zip\n2,SF,94107", + ) + assert result.exit_code == 0, result.output + assert db["places"].columns_dict["zip"] is str + assert db["places"].get(1)["zip"] == "01234" From 3de8507c6b7d28288d220d5627d4efe4f21b7abc Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:47:59 -0700 Subject: [PATCH 095/120] insert(pk=, alter=True) can add the pk column from records again The InvalidColumns check added for #732 fired before alter=True had a chance to add the missing pk column - a regression from 3.x, where insert({"id": 5, "a": 2}, pk="id", alter=True) against a table without an id column worked. With alter=True the check is now deferred until the first batch of record keys is known: a pk column supplied by the records passes, one found in neither the table nor the records still raises InvalidColumns before anything is inserted. An empty record iterator with alter=True returns without error, matching the 3.x no-op. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 19 ++++++++++++++++++- tests/test_create.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 80bcfb7..df41884 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed a regression where ``table.insert(..., pk=..., alter=True)`` raised ``InvalidColumns`` if the primary key column did not exist in the table yet. With ``alter=True`` the check now waits until the record keys are known, so a pk column supplied by the records is added by the alter as it was in 3.x. A pk column found in neither the table nor the records still raises ``InvalidColumns``. - Fixed a bug where inserting CSV or TSV data into an existing table rewrote that table's column types to match the incoming file. Type detection is the default in 4.0, so ``sqlite-utils insert data.db places places.csv --csv`` against a table with a ``TEXT`` zip code column would convert the column to ``INTEGER`` and corrupt values with leading zeros - ``"01234"`` became ``1234``. Detected types are now only applied when the ``insert`` or ``upsert`` command creates the table. - Fixed ``pks_and_rows_where()`` raising ``AttributeError`` when called on a view, and no longer double-quotes the synthesized ``rowid`` column in its generated SQL - SQLite turns a double-quoted identifier that does not resolve into a string literal, which on a view produced a confusing ``KeyError`` instead of the ``OperationalError`` raised in 3.x. Compound primary keys returned by this method now follow ``PRIMARY KEY`` declaration order. - The ``foreign_keys=`` argument to ``create()`` and ``insert()`` accepts a mixed list of ``ForeignKey`` objects, tuples and column name strings again. In 4.0 pre-releases mixing ``ForeignKey`` objects with tuples raised a ``ValueError`` - a regression from 3.x, where ``ForeignKey`` was a ``namedtuple`` and passed the tuple checks. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 4c7ef52..faf588f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -4282,6 +4282,10 @@ class Table(Queryable): if hash_id: pk = hash_id + # pk columns missing from an existing table are an error - unless + # alter=True, where a pk column supplied by the records will be + # added, so validation waits until the record keys are known + deferred_invalid_pk_check = None if pk and not hash_id and self.exists(): pk_cols = [pk] if isinstance(pk, str) else list(pk) existing_columns = self.columns_dict @@ -4291,7 +4295,7 @@ class Table(Queryable): if resolve_casing(col, existing_columns) not in existing_columns ] if missing_pk_cols: - raise InvalidColumns( + invalid_pk_error = InvalidColumns( "Invalid primary key column{} {} for table {} with columns {}".format( "s" if len(missing_pk_cols) > 1 else "", missing_pk_cols, @@ -4299,6 +4303,9 @@ class Table(Queryable): list(existing_columns), ) ) + if not alter: + raise invalid_pk_error + deferred_invalid_pk_check = (missing_pk_cols, invalid_pk_error) if ignore and replace: raise ValueError("Use either ignore=True or replace=True, not both") @@ -4409,6 +4416,16 @@ class Table(Queryable): all_columns = list(sorted(all_columns_set)) if hash_id: all_columns.insert(0, hash_id) + if deferred_invalid_pk_check is not None: + # alter=True - pk columns the table lacks are valid if + # the records supply them, otherwise raise the error + missing_pk_cols, invalid_pk_error = deferred_invalid_pk_check + record_columns = {column: True for column in all_columns} + if any( + resolve_casing(col, record_columns) not in record_columns + for col in missing_pk_cols + ): + raise invalid_pk_error else: if not list_mode: for record in cast(List[Dict[str, Any]], chunk): diff --git a/tests/test_create.py b/tests/test_create.py index 2788920..42fa1c4 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -932,6 +932,23 @@ def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + with pytest.raises(InvalidColumns) as ex: + fresh_db["t"].insert_all(rows, pk="not_a_column") + + assert ex.value.args == ( + "Invalid primary key column ['not_a_column'] for table t with columns ['a', 'b']", + ) + assert fresh_db["t"].count == 0 + + +@pytest.mark.parametrize("num_rows", (1, 2, 3, 10)) +def test_insert_all_pk_not_in_records_alter_raises(fresh_db, num_rows): + # With alter=True the check is deferred until the record keys are + # known - a pk column that is in neither the table nor the records + # still raises + fresh_db.conn.execute("CREATE TABLE t (a TEXT, b INT, PRIMARY KEY (a, b))") + rows = [{"a": "x{}".format(i), "b": i} for i in range(num_rows)] + with pytest.raises(InvalidColumns) as ex: fresh_db["t"].insert_all(rows, pk="not_a_column", alter=True) @@ -941,6 +958,26 @@ def test_insert_all_pk_not_in_records_raises(fresh_db, num_rows): assert fresh_db["t"].count == 0 +def test_insert_pk_in_records_with_alter_adds_column(fresh_db): + # 3.x allowed insert(pk=..., alter=True) to add the pk column from the + # records - the InvalidColumns check must not fire in that case + fresh_db["t"].insert({"a": 1}) + fresh_db["t"].insert({"id": 5, "a": 2}, pk="id", alter=True) + assert fresh_db["t"].columns_dict.keys() == {"a", "id"} + assert list(fresh_db.query("select * from t order by a")) == [ + {"a": 1, "id": None}, + {"a": 2, "id": 5}, + ] + + +def test_insert_all_invalid_pk_alter_empty_records_is_noop(fresh_db): + # With alter=True the pk check needs record keys, so an empty iterator + # returns without error - matching the 3.x no-op for empty inserts + fresh_db.conn.execute("CREATE TABLE t (a TEXT)") + fresh_db["t"].insert_all([], pk="not_a_column", alter=True) + assert fresh_db["t"].count == 0 + + def test_insert_ignore(fresh_db): fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") # Should raise an error if we try this again From b3aa3f47b717bba2c166f11e59a35bcd2f8e6b09 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:50:05 -0700 Subject: [PATCH 096/120] migrate --stop-before an already-applied migration is now an error The CLI validated --stop-before names against both pending and applied migrations, but Migrations.apply() only looked for the stop name among pending ones - naming an applied migration passed validation and then silently applied every migration after it, the exact outcome the option exists to prevent. apply() now raises ValueError before applying anything if a stop_before name matches an applied migration in its set; names not in the set are still ignored, since unqualified CLI values are offered to every set. The migrate command reports it as a clean error. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/migrations.rst | 2 +- sqlite_utils/cli.py | 5 ++++- sqlite_utils/migrations.py | 22 +++++++++++++++++++++- tests/test_cli_migrate.py | 22 ++++++++++++++++++++++ tests/test_migrations.py | 30 ++++++++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index df41884..4339e81 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``sqlite-utils migrate --stop-before`` now exits with an error if the named migration has already been applied. Previously the name passed validation but was only checked against pending migrations, so every migration after it was silently applied - the exact outcome ``--stop-before`` exists to prevent. ``Migrations.apply(db, stop_before=...)`` raises ``ValueError`` in the same situation, before applying anything. - Fixed a regression where ``table.insert(..., pk=..., alter=True)`` raised ``InvalidColumns`` if the primary key column did not exist in the table yet. With ``alter=True`` the check now waits until the record keys are known, so a pk column supplied by the records is added by the alter as it was in 3.x. A pk column found in neither the table nor the records still raises ``InvalidColumns``. - Fixed a bug where inserting CSV or TSV data into an existing table rewrote that table's column types to match the incoming file. Type detection is the default in 4.0, so ``sqlite-utils insert data.db places places.csv --csv`` against a table with a ``TEXT`` zip code column would convert the column to ``INTEGER`` and corrupt values with leading zeros - ``"01234"`` became ``1234``. Detected types are now only applied when the ``insert`` or ``upsert`` command creates the table. - Fixed ``pks_and_rows_where()`` raising ``AttributeError`` when called on a view, and no longer double-quotes the synthesized ``rowid`` column in its generated SQL - SQLite turns a double-quoted identifier that does not resolve into a string literal, which on a view produced a confusing ``KeyError`` instead of the ``OperationalError`` raised in 3.x. Compound primary keys returned by this method now follow ``PRIMARY KEY`` declaration order. diff --git a/docs/migrations.rst b/docs/migrations.rst index e685936..cfdbf13 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -157,7 +157,7 @@ You can also target a specific migration set using ``migration_set:migration_nam The ``--stop-before`` option can be passed more than once. -If a ``--stop-before`` value does not match any known migration the command exits with an error, rather than silently applying everything. +If a ``--stop-before`` value does not match any known migration the command exits with an error, rather than silently applying everything. Naming a migration that has already been applied is also an error - stopping before it is impossible to honor - and no pending migrations are applied. Verbose output ============== diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0ef7cc4..188eff6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3462,7 +3462,10 @@ def migrate(db_path, migrations, stop_before, list_, verbose): for migration_set in migration_sets: matches = _stop_before_for_migration_set(stop_before, migration_set.name) if isinstance(migration_set, sqlite_utils.Migrations): - migration_set.apply(db, stop_before=matches) + try: + migration_set.apply(db, stop_before=matches) + except ValueError as e: + raise click.ClickException(str(e)) else: # Legacy sqlite-migrate Migrations objects take a single string # for stop_before, not a list diff --git a/sqlite_utils/migrations.py b/sqlite_utils/migrations.py index 36e053d..00d0fa5 100644 --- a/sqlite_utils/migrations.py +++ b/sqlite_utils/migrations.py @@ -96,14 +96,34 @@ class Migrations: changes are rolled back, no record is written and the migration stays pending. Migrations registered with ``transactional=False`` run outside of a transaction. + + :raises ValueError: if a ``stop_before`` name matches a migration in + this set that has already been applied - stopping before it is + impossible to honor, and no pending migrations are applied """ - self.ensure_migrations_table(db) if stop_before is None: stop_before_names = set() elif isinstance(stop_before, str): stop_before_names = {stop_before} else: stop_before_names = set(stop_before) + # A stop_before naming an already-applied migration cannot be + # honored - error rather than applying everything after it. Names + # not in this set at all are ignored, because unqualified CLI + # values are offered to every migration set + already_applied = stop_before_names.intersection( + migration.name for migration in self.applied(db) + ) + if already_applied: + raise ValueError( + "Cannot stop before migration{} {} in set '{}' - already " + "been applied".format( + "s" if len(already_applied) > 1 else "", + ", ".join(sorted(already_applied)), + self.name, + ) + ) + self.ensure_migrations_table(db) for migration in self.pending(db): name = migration.name if name in stop_before_names: diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index 1cdf8e7..0f1c7ea 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -463,3 +463,25 @@ def test_list_does_not_upgrade_legacy_migrations_table(two_migrations): db2 = sqlite_utils.Database(db_path) assert db2["_sqlite_migrations"].pks == ["migration_set", "name"] db2.close() + + +def test_stop_before_applied_migration_errors(two_migrations): + path, _ = two_migrations + db_path = str(path / "test.db") + migrations_path = str(path / "foo" / "migrations.py") + # Apply everything first + first = CliRunner().invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, migrations_path, "--stop-before", "bar"], + ) + assert first.exit_code == 0 + # foo is now applied - stopping before it is an error, and bar + # must not be applied as a side effect + result = CliRunner().invoke( + sqlite_utils.cli.cli, + ["migrate", db_path, migrations_path, "--stop-before", "foo"], + ) + assert result.exit_code != 0 + assert "already been applied" in result.output + db = sqlite_utils.Database(db_path) + assert not db["bar"].exists() diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 5634d79..04185fc 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -214,3 +214,33 @@ def test_duplicate_migration_name_errors(): pass assert "m001" in str(ex.value) + + +def test_stop_before_applied_migration_errors(migrations): + # Stopping before a migration that has already been applied is + # impossible to honor - previously the stop name was only checked + # against pending migrations, so everything after it was applied + db = sqlite_utils.Database(memory=True) + migrations.apply(db, stop_before="m002") # applies m001 only + with pytest.raises(ValueError) as ex: + migrations.apply(db, stop_before="m001") + assert "m001" in str(ex.value) + assert "already been applied" in str(ex.value) + # Nothing else was applied + assert not db["cats"].exists() + + +def test_stop_before_applied_migration_errors_before_any_apply(migrations): + # The error fires before any pending migration runs, even those that + # come before the already-applied stop target in registration order + db = sqlite_utils.Database(memory=True) + only_second = Migrations("test") + + @only_second() + def m002(db): + db["cats"].create({"name": str}) + + only_second.apply(db) # m002 applied, m001 still pending + with pytest.raises(ValueError): + migrations.apply(db, stop_before="m002") + assert not db["dogs"].exists() From a0387791e511a17eb96e0ed667da222550a00412 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:51:31 -0700 Subject: [PATCH 097/120] ensure_autocommit_on() raises TransactionError inside a transaction Assigning conn.isolation_level commits any pending transaction as a side effect, so entering the context manager with a transaction open silently committed the caller's work and made a later rollback() a no-op. All internal callers already ensure no transaction is open; the public API now enforces it, matching enable_wal() and disable_wal(). Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 9 +++++++++ tests/test_wal.py | 17 +++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4339e81..87a1ab7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``db.ensure_autocommit_on()`` now raises ``TransactionError`` if called while a transaction is open. Assigning ``isolation_level`` commits any pending transaction as a side effect, so entering the block silently committed the caller's open transaction and made a later ``rollback()`` a no-op. - ``sqlite-utils migrate --stop-before`` now exits with an error if the named migration has already been applied. Previously the name passed validation but was only checked against pending migrations, so every migration after it was silently applied - the exact outcome ``--stop-before`` exists to prevent. ``Migrations.apply(db, stop_before=...)`` raises ``ValueError`` in the same situation, before applying anything. - Fixed a regression where ``table.insert(..., pk=..., alter=True)`` raised ``InvalidColumns`` if the primary key column did not exist in the table yet. With ``alter=True`` the check now waits until the record keys are known, so a pk column supplied by the records is added by the alter as it was in 3.x. A pk column found in neither the table nor the records still raises ``InvalidColumns``. - Fixed a bug where inserting CSV or TSV data into an existing table rewrote that table's column types to match the incoming file. Type detection is the default in 4.0, so ``sqlite-utils insert data.db places places.csv --csv`` against a table with a ``TEXT`` zip code column would convert the column to ``INTEGER`` and corrupt values with leading zeros - ``"01234"`` became ``1234``. Detected types are now only applied when the ``insert`` or ``upsert`` command creates the table. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index faf588f..90e7a83 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -665,7 +665,16 @@ class Database: # do stuff here The previous ``isolation_level`` is restored at the end of the block. + + :raises TransactionError: if a transaction is open - assigning + ``isolation_level`` would commit it as a side effect, silently + breaking the caller's ability to roll back """ + if self.conn.in_transaction: + raise TransactionError( + "ensure_autocommit_on() cannot be used inside a transaction - " + "changing isolation_level would commit the open transaction" + ) old_isolation_level = self.conn.isolation_level try: self.conn.isolation_level = None diff --git a/tests/test_wal.py b/tests/test_wal.py index c5a9c60..2ddcf54 100644 --- a/tests/test_wal.py +++ b/tests/test_wal.py @@ -69,3 +69,20 @@ def test_enable_wal_noop_inside_transaction_is_allowed(db_path_tmpdir): db["test"].insert({"id": 1}, pk="id") db.enable_wal() assert [r["id"] for r in db["test"].rows] == [1] + + +def test_ensure_autocommit_on_inside_transaction_raises(db_path_tmpdir): + # Setting isolation_level commits any pending transaction as a side + # effect, silently breaking the caller's rollback guarantee - so + # entering autocommit mode with a transaction open is an error + db, path, tmpdir = db_path_tmpdir + db["test"].insert({"id": 1}, pk="id") + db.begin() + db.execute("insert into test (id) values (2)") + with pytest.raises(TransactionError): + with db.ensure_autocommit_on(): + pass + # The transaction is still open and can still be rolled back + assert db.conn.in_transaction + db.rollback() + assert [r["id"] for r in db["test"].rows] == [1] From 16bbfb582d6acf33a8c05317b64d93c85e184a5c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:53:48 -0700 Subject: [PATCH 098/120] add_foreign_keys() errors instead of silently dropping action changes The existing-foreign-key dedup compared only columns and other_table, so requesting an FK that already exists with different ON DELETE/ON UPDATE actions was a silent no-op - and with add_foreign_key() raising "already exists", there was no signal that the requested actions were dropped. An exact match (including actions) is still skipped for idempotency; a mismatch now raises AlterError pointing at table.transform(). Also validates that compound foreign keys passed as 4-tuples have the same number of columns on both sides - extra other-columns were being silently discarded, e.g. ("t", ("a",), "other", ("a", "b")) created a single-column key referencing just "a". Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + docs/python-api.rst | 2 ++ sqlite_utils/db.py | 25 ++++++++++++++++++--- tests/test_foreign_keys.py | 46 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 87a1ab7..ff78738 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``db.add_foreign_keys()`` no longer silently ignores requested ``ON DELETE``/``ON UPDATE`` actions when a foreign key with the same columns already exists - it raises ``AlterError`` suggesting ``table.transform()``, since the actions of an existing foreign key cannot be changed in place. Exact duplicates, including actions, are still skipped so repeated calls stay idempotent. The method also now validates that compound foreign keys have the same number of columns on both sides, instead of silently discarding the extra columns. - ``db.ensure_autocommit_on()`` now raises ``TransactionError`` if called while a transaction is open. Assigning ``isolation_level`` commits any pending transaction as a side effect, so entering the block silently committed the caller's open transaction and made a later ``rollback()`` a no-op. - ``sqlite-utils migrate --stop-before`` now exits with an error if the named migration has already been applied. Previously the name passed validation but was only checked against pending migrations, so every migration after it was silently applied - the exact outcome ``--stop-before`` exists to prevent. ``Migrations.apply(db, stop_before=...)`` raises ``ValueError`` in the same situation, before applying anything. - Fixed a regression where ``table.insert(..., pk=..., alter=True)`` raised ``InvalidColumns`` if the primary key column did not exist in the table yet. With ``alter=True`` the check now waits until the record keys are known, so a pk column supplied by the records is added by the alter as it was in 3.x. A pk column found in neither the table nor the records still raises ``InvalidColumns``. diff --git a/docs/python-api.rst b/docs/python-api.rst index 516e2aa..11af1f4 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1636,6 +1636,8 @@ Here's an example adding two foreign keys at once: This method runs the same checks as ``.add_foreign_keys()`` and will raise ``sqlite_utils.db.AlterError`` if those checks fail. +Foreign keys that already exist are silently skipped, so repeated calls are idempotent - but only if they match exactly. Requesting a foreign key that exists with different ``ON DELETE``/``ON UPDATE`` actions raises ``AlterError``: use ``table.transform()`` to change the actions of an existing foreign key. + .. _python_api_index_foreign_keys: Adding indexes for all foreign keys diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 90e7a83..eb59c50 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1786,6 +1786,11 @@ class Database: if isinstance(other_column_or_columns, str) else tuple(other_column_or_columns) ) + if len(columns) != len(other_columns): + raise ValueError( + "Compound foreign key must have the same number of " + "columns on both sides" + ) if len(columns) == 1: fk_object = ForeignKey( table, columns[0], other_table, other_columns[0] @@ -1825,10 +1830,11 @@ class Database: other_column, other_table ) ) - # We will silently skip foreign keys that exist already + # Silently skip foreign keys that exist already - but only if + # they match exactly, including ON DELETE/ON UPDATE actions columns_folded = tuple(fold_identifier_case(c) for c in columns) other_columns_folded = tuple(fold_identifier_case(c) for c in other_columns) - if not any( + existing = [ fk for fk in table_obj.foreign_keys if tuple(fold_identifier_case(c) for c in fk.columns) == columns_folded @@ -1836,8 +1842,21 @@ class Database: == fold_identifier_case(other_table) and tuple(fold_identifier_case(c) for c in fk.other_columns) == other_columns_folded - ): + ] + if not existing: foreign_keys_to_create.append(fk_object) + elif any( + fk.on_delete != fk_object.on_delete + or fk.on_update != fk_object.on_update + for fk in existing + ): + raise AlterError( + "Foreign key already exists for {} => {}.{} but with " + "different ON DELETE/ON UPDATE actions - use " + "table.transform() to change them".format( + ", ".join(columns), other_table, ", ".join(other_columns) + ) + ) # Group them by table by_table: Dict[str, List[ForeignKey]] = {} diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index d7c20f6..b37d374 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -643,3 +643,49 @@ def test_create_table_mixed_foreign_keys_with_string(fresh_db): ) fks = {fk.column: fk.other_table for fk in fresh_db["books"].foreign_keys} assert fks == {"author_id": "authors", "publisher_id": "publishers"} + + +def test_add_foreign_keys_existing_with_different_actions_errors(fresh_db): + # Requesting an existing foreign key with different ON DELETE/ON UPDATE + # actions was silently skipped, dropping the requested change + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert( + {"id": 1, "author_id": 1}, + pk="id", + foreign_keys=[("author_id", "authors", "id")], + ) + with pytest.raises(AlterError) as ex: + fresh_db.add_foreign_keys( + [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] + ) + assert "ON DELETE" in str(ex.value) + assert fresh_db["books"].foreign_keys[0].on_delete == "NO ACTION" + + +def test_add_foreign_keys_identical_existing_is_noop(fresh_db): + # An exact match, including actions, is silently skipped so repeated + # calls stay idempotent + fresh_db["authors"].insert({"id": 1}, pk="id") + fresh_db["books"].insert({"id": 1, "author_id": 1}, pk="id") + fresh_db["books"].add_foreign_key("author_id", "authors", "id", on_delete="CASCADE") + fresh_db.add_foreign_keys( + [ForeignKey("books", "author_id", "authors", "id", on_delete="CASCADE")] + ) + fks = fresh_db["books"].foreign_keys + assert len(fks) == 1 + assert fks[0].on_delete == "CASCADE" + + +def test_add_foreign_keys_compound_column_count_mismatch_errors(fresh_db): + # Previously the extra other-column was silently discarded, creating + # a single-column foreign key to just ("id") + fresh_db["departments"].insert( + {"campus": "north", "code": "cs"}, pk=("campus", "code") + ) + fresh_db["courses"].insert({"id": 1, "campus": "north"}, pk="id") + with pytest.raises(ValueError) as ex: + fresh_db.add_foreign_keys( + [("courses", ("campus",), "departments", ("campus", "code"))] + ) + assert "same number of columns" in str(ex.value) + assert fresh_db["courses"].foreign_keys == [] From 8572d1e39c3c807bc0643249411fb33bd0881eb6 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:55:21 -0700 Subject: [PATCH 099/120] extract() no longer duplicates NULL-containing rows in shared lookups The lookup-table insert relied on INSERT OR IGNORE and the unique index to dedupe against existing rows, but SQLite unique indexes treat NULLs as distinct - extracting a second table into the same lookup table re-inserted every NULL-containing value, growing orphan rows on each extract. The insert now also has an IS-based NOT EXISTS guard, matching how the foreign keys themselves are resolved. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 18 +++++++++++++++++- tests/test_extract.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ff78738..d2b8a67 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed a bug where running ``table.extract()`` more than once against the same lookup table inserted duplicate rows for values containing ``null`` - SQLite unique indexes treat ``NULL`` values as distinct, so ``INSERT OR IGNORE`` alone could not dedupe them. Each repeat extract added another copy that nothing referenced. The insert now uses an ``IS``-based ``NOT EXISTS`` guard so ``null``-containing rows match existing lookup rows. - ``db.add_foreign_keys()`` no longer silently ignores requested ``ON DELETE``/``ON UPDATE`` actions when a foreign key with the same columns already exists - it raises ``AlterError`` suggesting ``table.transform()``, since the actions of an existing foreign key cannot be changed in place. Exact duplicates, including actions, are still skipped so repeated calls stay idempotent. The method also now validates that compound foreign keys have the same number of columns on both sides, instead of silently discarding the extra columns. - ``db.ensure_autocommit_on()`` now raises ``TransactionError`` if called while a transaction is open. Assigning ``isolation_level`` commits any pending transaction as a side effect, so entering the block silently committed the caller's open transaction and made a later ``rollback()`` a no-op. - ``sqlite-utils migrate --stop-before`` now exits with an error if the named migration has already been applied. Previously the name passed validation but was only checked against pending migrations, so every migration after it was silently applied - the exact outcome ``--stop-before`` exists to prevent. ``Migrations.apply(db, stop_before=...)`` raises ``ValueError`` in the same situation, before applying anything. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index eb59c50..e69f996 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2919,8 +2919,22 @@ class Table(Queryable): all_columns_are_null = " AND ".join( "{} IS NULL".format(quote_identifier(c)) for c in columns ) + # INSERT OR IGNORE dedupes against the unique index, but unique + # indexes treat NULLs as distinct - the NOT EXISTS guard uses IS + # comparison so NULL-containing rows match existing lookup rows + # instead of being inserted again + already_in_lookup = " AND ".join( + "{lookup}.{lookup_col} IS {source}.{source_col}".format( + lookup=quote_identifier(table), + lookup_col=quote_identifier(rename.get(column) or column), + source=quote_identifier(self.name), + source_col=quote_identifier(column), + ) + for column in columns + ) self.db.execute( - "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {} WHERE NOT ({all_null})".format( + "INSERT OR IGNORE INTO {} ({lookup_columns}) SELECT DISTINCT {table_cols} FROM {} " + "WHERE NOT ({all_null}) AND NOT EXISTS (SELECT 1 FROM {lookup} WHERE {already_in_lookup})".format( quote_identifier(table), quote_identifier(self.name), lookup_columns=", ".join( @@ -2928,6 +2942,8 @@ class Table(Queryable): ), table_cols=", ".join(quote_identifier(c) for c in columns), all_null=all_columns_are_null, + lookup=quote_identifier(table), + already_in_lookup=already_in_lookup, ) ) diff --git a/tests/test_extract.py b/tests/test_extract.py index 1c0fa01..c73ee7a 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -271,3 +271,34 @@ def test_extract_null_values_existing_lookup_table_with_null_row(fresh_db): {"id": 10, "name": "Terriana", "species_id": 2}, {"id": 11, "name": "Spenidorm", "species_id": None}, ] + + +def test_extract_repeated_into_shared_lookup_with_nulls(fresh_db): + # Unique indexes treat NULLs as distinct, so INSERT OR IGNORE alone + # cannot dedupe NULL-containing rows against the existing lookup + # table - extracting a second table into the same lookup previously + # inserted duplicate rows that nothing pointed to + fresh_db["t1"].insert_all( + [ + {"id": 1, "species": None, "common": "X"}, + {"id": 2, "species": "Oak", "common": "Oak"}, + ], + pk="id", + ) + fresh_db["t2"].insert_all([{"id": 1, "species": None, "common": "X"}], pk="id") + fresh_db["t1"].extract(["species", "common"], table="lk") + fresh_db["t2"].extract(["species", "common"], table="lk") + assert fresh_db["lk"].count == 2 + # Both tables point at the same lookup row + t1_fk = fresh_db.execute("select lk_id from t1 where id = 1").fetchone()[0] + t2_fk = fresh_db.execute("select lk_id from t2 where id = 1").fetchone()[0] + assert t1_fk == t2_fk + + +def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db): + # Non-NULL rows were already deduped by the unique index - keep it so + fresh_db["t1"].insert_all([{"id": 1, "species": "Oak"}], pk="id") + fresh_db["t2"].insert_all([{"id": 1, "species": "Oak"}], pk="id") + fresh_db["t1"].extract(["species"], table="lk") + fresh_db["t2"].extract(["species"], table="lk") + assert fresh_db["lk"].count == 1 From 548a886ca1d2ba6c259676da2c8e00f022bacc2c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:57:14 -0700 Subject: [PATCH 100/120] Clean CLI errors for InvalidColumns from insert --pk and extract sqlite-utils insert db t - --pk badcol and sqlite-utils extract db t nosuchcol dumped raw InvalidColumns tracebacks - the insert error handling caught NoTable and OperationalError but not the InvalidColumns introduced for #732, and the extract command had no handling at all (including for NoTable when pointed at a view). Both now exit with click-style Error: messages. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/cli.py | 8 ++++++-- tests/test_cli.py | 19 +++++++++++++++++++ tests/test_cli_insert.py | 15 +++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d2b8a67..8c5b3e6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``sqlite-utils insert ... --pk `` and ``sqlite-utils extract `` now show a clean ``Error:`` message instead of a raw Python traceback. The ``extract`` command also shows a clean error when pointed at a view. - Fixed a bug where running ``table.extract()`` more than once against the same lookup table inserted duplicate rows for values containing ``null`` - SQLite unique indexes treat ``NULL`` values as distinct, so ``INSERT OR IGNORE`` alone could not dedupe them. Each repeat extract added another copy that nothing referenced. The insert now uses an ``IS``-based ``NOT EXISTS`` guard so ``null``-containing rows match existing lookup rows. - ``db.add_foreign_keys()`` no longer silently ignores requested ``ON DELETE``/``ON UPDATE`` actions when a foreign key with the same columns already exists - it raises ``AlterError`` suggesting ``table.transform()``, since the actions of an existing foreign key cannot be changed in place. Exact duplicates, including actions, are still skipped so repeated calls stay idempotent. The method also now validates that compound foreign keys have the same number of columns on both sides, instead of silently discarding the extra columns. - ``db.ensure_autocommit_on()`` now raises ``TransactionError`` if called while a transaction is open. Assigning ``isolation_level`` commits any pending transaction as a side effect, so entering the block silently committed the caller's open transaction and made a later ``rollback()`` a no-op. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 188eff6..29dcb9d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -13,6 +13,7 @@ from sqlite_utils.db import ( BadMultiValues, DEFAULT, DescIndex, + InvalidColumns, NoTable, NoView, quote_identifier, @@ -1172,7 +1173,7 @@ def insert_upsert_implementation( db.table(table).insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs ) - except NoTable as e: + except (NoTable, InvalidColumns) as e: raise click.ClickException(str(e)) except Exception as e: if ( @@ -2734,7 +2735,10 @@ def extract( fk_column=fk_column, rename=dict(rename), ) - db.table(table).extract(**kwargs) + try: + db.table(table).extract(**kwargs) + except (NoTable, InvalidColumns) as e: + raise click.ClickException(str(e)) @cli.command(name="insert-files") diff --git a/tests/test_cli.py b/tests/test_cli.py index 9e17969..06e14ea 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2758,3 +2758,22 @@ def test_insert_upsert_strict(tmpdir, method, strict): assert result.exit_code == 0 db = Database(db_path) assert db["items"].strict == strict or not db.supports_strict + + +def test_extract_bad_column_clean_error(db_path): + db = Database(db_path) + db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") + result = CliRunner().invoke(cli.cli, ["extract", db_path, "trees", "nope"]) + assert result.exit_code == 1 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.startswith("Error: Invalid columns") + + +def test_extract_view_clean_error(db_path): + db = Database(db_path) + db["trees"].insert({"id": 1, "species": "Palm"}, pk="id") + db.create_view("v", "select * from trees") + result = CliRunner().invoke(cli.cli, ["extract", db_path, "v", "species"]) + assert result.exit_code == 1 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.startswith("Error:") diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 196590e..5af8a2f 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -676,3 +676,18 @@ def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): assert result.exit_code == 0, result.output assert db["places"].columns_dict["zip"] is str assert db["places"].get(1)["zip"] == "01234" + + +def test_insert_invalid_pk_clean_error(db_path): + # An invalid --pk against an existing table should be a clean CLI + # error, not a raw InvalidColumns traceback + db = Database(db_path) + db["t"].insert({"a": 1}) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "t", "-", "--pk", "badcol"], + input='{"a": 2}', + ) + assert result.exit_code == 1 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.startswith("Error: Invalid primary key column") From 93640a7ddedf008036427a3e5bbd033d17cbe9df Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 21:58:52 -0700 Subject: [PATCH 101/120] migrate --list is read-only with legacy sqlite-migrate classes too The docs promise --list will not create the database file or the _sqlite_migrations table, but legacy sqlite_migrate.Migrations classes create the table (in the legacy schema) from their pending()/applied() methods. The listing now runs inside a transaction that is rolled back, keeping --list read-only regardless of what the migration class does. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/cli.py | 9 ++++++++- tests/test_cli_migrate.py | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8c5b3e6..881423c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- ``sqlite-utils migrate --list`` is now read-only even when the migrations file uses the legacy ``sqlite_migrate.Migrations`` class, whose listing methods create the ``_sqlite_migrations`` table as a side effect. The listing now runs inside a transaction that is rolled back. - ``sqlite-utils insert ... --pk `` and ``sqlite-utils extract `` now show a clean ``Error:`` message instead of a raw Python traceback. The ``extract`` command also shows a clean error when pointed at a view. - Fixed a bug where running ``table.extract()`` more than once against the same lookup table inserted duplicate rows for values containing ``null`` - SQLite unique indexes treat ``NULL`` values as distinct, so ``INSERT OR IGNORE`` alone could not dedupe them. Each repeat extract added another copy that nothing referenced. The insert now uses an ``IS``-based ``NOT EXISTS`` guard so ``null``-containing rows match existing lookup rows. - ``db.add_foreign_keys()`` no longer silently ignores requested ``ON DELETE``/``ON UPDATE`` actions when a foreign key with the same columns already exists - it raises ``AlterError`` suggesting ``table.transform()``, since the actions of an existing foreign key cannot be changed in place. Exact duplicates, including actions, are still skipped so repeated calls stay idempotent. The method also now validates that compound foreign keys have the same number of columns on both sides, instead of silently discarding the extra columns. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 29dcb9d..fe7dbe4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3434,7 +3434,14 @@ def migrate(db_path, migrations, stop_before, list_, verbose): # Listing is read-only - don't create the database file db = sqlite_utils.Database(memory=True) _register_db_for_cleanup(db) - _display_migration_list(db, migration_sets) + # Legacy sqlite-migrate classes create the migrations table from + # their pending()/applied() methods - run the listing inside a + # transaction and roll it back so --list stays read-only + db.begin() + try: + _display_migration_list(db, migration_sets) + finally: + db.rollback() return db = sqlite_utils.Database(db_path) diff --git a/tests/test_cli_migrate.py b/tests/test_cli_migrate.py index 0f1c7ea..0f29e36 100644 --- a/tests/test_cli_migrate.py +++ b/tests/test_cli_migrate.py @@ -485,3 +485,23 @@ def test_stop_before_applied_migration_errors(two_migrations): assert "already been applied" in result.output db = sqlite_utils.Database(db_path) assert not db["bar"].exists() + + +def test_list_with_legacy_class_is_read_only(tmpdir): + # Legacy sqlite-migrate classes create the _sqlite_migrations table + # from their pending()/applied() methods - --list must roll that + # back so it stays a read-only operation as documented + path = pathlib.Path(tmpdir) + (path / "migrations.py").write_text(LEGACY_MIGRATIONS, "utf-8") + db_path = str(path / "test.db") + db = sqlite_utils.Database(db_path) + db["existing"].insert({"id": 1}) + db.close() + result = CliRunner().invoke( + sqlite_utils.cli.cli, ["migrate", db_path, str(path), "--list"] + ) + assert result.exit_code == 0, result.output + assert "first" in result.output + db2 = sqlite_utils.Database(db_path) + assert "_sqlite_migrations" not in db2.table_names() + db2.close() From c2a17744095553c7fcc4706cc5da1d20045a299e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:02:41 -0700 Subject: [PATCH 102/120] Fix two tests that assumed modern SQLite behavior SQLite 3.23.1 rejects a UTF-8 byte order mark before the first token, so the BOM variant of the execute()-prefixed-BEGIN test now skips when the SQLite version does not accept a leading BOM. And versions before 3.36 allowed selecting rowid from a view, returning NULL, rather than raising an error - the pks_and_rows_where() view test now accepts either behavior. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- tests/test_atomic.py | 53 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_rows.py | 16 ++++++++----- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/tests/test_atomic.py b/tests/test_atomic.py index 0d25b84..c3fd02f 100644 --- a/tests/test_atomic.py +++ b/tests/test_atomic.py @@ -258,11 +258,21 @@ def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db): assert [r["id"] for r in fresh_db["t"].rows] == [1] +def _sqlite_accepts_bom(): + try: + sqlite3.connect(":memory:").execute("\ufeffselect 1") + return True + except sqlite3.OperationalError: + return False + + @pytest.mark.parametrize("begin_sql", ["; begin", "\ufeffbegin"]) def test_execute_prefixed_begin_leaves_transaction_open(fresh_db, begin_sql): # sqlite3 tolerates empty statements and a UTF-8 BOM before the first # real token, so a BEGIN behind either must not be auto-committed # out from under the caller + if begin_sql.startswith("\ufeff") and not _sqlite_accepts_bom(): + pytest.skip("This SQLite version rejects a leading byte order mark") fresh_db["t"].insert({"id": 1}, pk="id") fresh_db.execute(begin_sql) assert fresh_db.conn.in_transaction @@ -327,3 +337,46 @@ def test_query_returning_commits_after_iteration(tmpdir): assert other.execute("select count(*) from t").fetchone()[0] == 2 other.close() db.close() + + +TRIGGER_SQL = """ +create trigger no_bad before insert on t +when new.v = 'bad' +begin + select raise(rollback, 'trigger says no'); +end +""" + + +def test_atomic_preserves_error_from_transaction_destroying_trigger(fresh_db): + # RAISE(ROLLBACK) rolls back the whole transaction and destroys every + # savepoint - atomic()'s cleanup must not mask the IntegrityError + # with "cannot rollback - no transaction is active" + fresh_db.execute("create table t (id integer primary key, v text)") + fresh_db.execute(TRIGGER_SQL) + with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): + with fresh_db.atomic(): + fresh_db.execute("insert into t (v) values ('bad')") + assert not fresh_db.conn.in_transaction + + +def test_nested_atomic_preserves_error_from_transaction_destroying_trigger( + fresh_db, +): + # The nested savepoint branch previously raised + # "no such savepoint" from ROLLBACK TO SAVEPOINT + fresh_db.execute("create table t (id integer primary key, v text)") + fresh_db.execute(TRIGGER_SQL) + with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): + with fresh_db.atomic(): + with fresh_db.atomic(): + fresh_db.execute("insert into t (v) values ('bad')") + assert not fresh_db.conn.in_transaction + + +def test_atomic_preserves_error_from_insert_or_rollback(fresh_db): + fresh_db["t"].insert({"id": 1}, pk="id") + with pytest.raises(sqlite3.IntegrityError): + with fresh_db.atomic(): + fresh_db.execute("insert or rollback into t (id) values (1)") + assert not fresh_db.conn.in_transaction diff --git a/tests/test_rows.py b/tests/test_rows.py index 46417ef..46d4f53 100644 --- a/tests/test_rows.py +++ b/tests/test_rows.py @@ -115,15 +115,21 @@ def test_rows_where_duplicate_select_columns_are_deduped(fresh_db): def test_pks_and_rows_where_view(fresh_db): # pks_and_rows_where() lives on Queryable so views expose it, but - # SQLite views have no rowid - it has always failed with an - # OperationalError from the generated SQL. Guard against it failing - # earlier with an AttributeError from View lacking Table properties + # SQLite views have no rowid. Modern SQLite (3.36+) raises an + # OperationalError from the generated SQL; older versions returned + # NULL for a view's rowid. Either way it must not fail earlier with + # an AttributeError from View lacking Table-only properties from sqlite_utils.utils import sqlite3 fresh_db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id") fresh_db.create_view("dog_names", "select name from dogs") - with pytest.raises(sqlite3.OperationalError): - list(fresh_db["dog_names"].pks_and_rows_where()) + try: + result = list(fresh_db["dog_names"].pks_and_rows_where()) + except sqlite3.OperationalError: + pass # SQLite 3.36+: no such column: rowid + else: + # Older SQLite returns NULL rowids for views + assert result == [(None, {"rowid": None, "name": "Cleo"})] def test_pks_and_rows_where_compound_pk_declaration_order(fresh_db): From d9a0fd26e09a7e1c07b7356d0cd25a22695f89c5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:04:00 -0700 Subject: [PATCH 103/120] Transaction cleanup no longer masks transaction-destroying errors A RAISE(ROLLBACK) trigger or INSERT OR ROLLBACK conflict rolls back the entire transaction and destroys every savepoint. The cleanup paths in atomic() and query() then raised OperationalError ("no such savepoint" / "cannot rollback - no transaction is active"), masking the original IntegrityError - breaking user code that catches sqlite3.IntegrityError. Cleanup now checks conn.in_transaction first: if the error already destroyed the transaction there is nothing left to undo, and the original exception propagates. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- docs/changelog.rst | 1 + sqlite_utils/db.py | 22 ++++++++++++++++------ tests/test_query.py | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 881423c..9228fb6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Unreleased - Fixed a bug where a failed write statement executed with ``db.execute()`` left the driver's implicit transaction open. Every subsequent write then joined that phantom transaction, which nothing committed, so their work was silently rolled back when the connection was closed. The implicit transaction opened by a failed statement is now rolled back before the exception is raised. A failed write inside a transaction opened with ``db.begin()`` or ``db.atomic()`` leaves that transaction open and untouched, as before. - Fixed a bug where transaction-control statements prefixed with an empty statement - ``db.query("; COMMIT")`` - or a UTF-8 byte order mark slipped past the check that rejects them, committing the caller's open transaction before raising a confusing ``OperationalError``. The keyword scanner used by ``db.query()`` and ``db.execute()`` now skips leading ``;`` and byte order marks, matching what the ``sqlite3`` driver tolerates before the first token, so these statements are rejected with a ``ValueError`` without being executed. The same fix means ``db.execute("; BEGIN")`` no longer auto-commits the transaction it just opened. - Documented a limitation of ``db.query()``: a ``PRAGMA`` statement that returns no rows raises a ``ValueError`` but still takes effect, because PRAGMA statements run outside the savepoint guard used to roll back other rejected statements. Use ``db.execute()`` for row-less PRAGMA statements. +- Fixed exception masking when a statement destroys the enclosing transaction. An error such as a ``RAISE(ROLLBACK)`` trigger or ``INSERT OR ROLLBACK`` conflict rolls back the whole transaction, destroying every savepoint - the cleanup in ``db.atomic()`` and ``db.query()`` then failed with ``OperationalError: no such savepoint`` (or ``cannot rollback - no transaction is active``), hiding the original ``IntegrityError`` from code that tried to catch it. Cleanup now checks whether a transaction is still open first, so the original exception propagates. - ``sqlite-utils migrate --list`` is now read-only even when the migrations file uses the legacy ``sqlite_migrate.Migrations`` class, whose listing methods create the ``_sqlite_migrations`` table as a side effect. The listing now runs inside a transaction that is rolled back. - ``sqlite-utils insert ... --pk `` and ``sqlite-utils extract `` now show a clean ``Error:`` message instead of a raw Python traceback. The ``extract`` command also shows a clean error when pointed at a view. - Fixed a bug where running ``table.extract()`` more than once against the same lookup table inserted duplicate rows for values containing ``null`` - SQLite unique indexes treat ``NULL`` values as distinct, so ``INSERT OR IGNORE`` alone could not dedupe them. Each repeat extract added another copy that nothing referenced. The insert now uses an ``IS``-based ``NOT EXISTS`` guard so ``null``-containing rows match existing lookup rows. diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e69f996..bec68fc 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -602,8 +602,13 @@ class Database: try: yield self except BaseException: - self.conn.execute("ROLLBACK TO SAVEPOINT {};".format(savepoint)) - self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) + # An error such as a RAISE(ROLLBACK) trigger can destroy + # the whole transaction, savepoints included - cleaning up + # anyway would mask the original exception with + # "no such savepoint" + if self.conn.in_transaction: + self.conn.execute("ROLLBACK TO SAVEPOINT {};".format(savepoint)) + self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) raise else: self.conn.execute("RELEASE SAVEPOINT {};".format(savepoint)) @@ -612,13 +617,15 @@ class Database: try: yield self except BaseException: - self.conn.execute("ROLLBACK") + # rollback() is a no-op if the error already destroyed the + # transaction, so the original exception propagates + self.rollback() raise else: try: self.conn.execute("COMMIT") except BaseException: - self.conn.execute("ROLLBACK") + self.rollback() raise def begin(self) -> None: @@ -870,8 +877,11 @@ class Database: return (dict(zip(keys, row)) for row in fetched) return (dict(zip(keys, row)) for row in cursor) finally: - if not released: - # An error occurred - undo anything the statement changed + if not released and self.conn.in_transaction: + # An error occurred - undo anything the statement changed. + # If the error itself destroyed the transaction (such as a + # RAISE(ROLLBACK) trigger) the savepoint is already gone + # and there is nothing left to undo self.conn.execute('ROLLBACK TO "sqlite_utils_query"') self.conn.execute('RELEASE "sqlite_utils_query"') diff --git a/tests/test_query.py b/tests/test_query.py index f4aa336..aac5a29 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -280,3 +280,20 @@ def test_execute_returning_dicts(fresh_db): assert fresh_db.execute_returning_dicts("select * from test") == [ {"id": 1, "bar": 2} ] + + +def test_query_preserves_error_from_transaction_destroying_trigger(fresh_db): + # RAISE(ROLLBACK) destroys the savepoint guard - the original + # IntegrityError must propagate, not "no such savepoint" + fresh_db.execute("create table t (id integer primary key, v text)") + fresh_db.execute(""" + create trigger no_bad before insert on t + when new.v = 'bad' + begin + select raise(rollback, 'trigger says no'); + end + """) + with pytest.raises(sqlite3.IntegrityError, match="trigger says no"): + fresh_db.query("insert into t (id, v) values (1, 'bad') returning id") + assert not fresh_db.conn.in_transaction + assert fresh_db.execute("select count(*) from t").fetchone()[0] == 0 From 25824467846ca75db3bd737cb19b5a8e0d01b1b2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:06:51 -0700 Subject: [PATCH 104/120] Skip RETURNING-based trigger test on SQLite older than 3.35 The query() exception-masking test uses INSERT ... RETURNING, which is a syntax error on SQLite 3.23.1 - skip it there like the other RETURNING tests. Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900034150 Co-Authored-By: Claude Fable 5 --- tests/test_query.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_query.py b/tests/test_query.py index aac5a29..06847da 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -282,6 +282,10 @@ def test_execute_returning_dicts(fresh_db): ] +@pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 35, 0), + reason="RETURNING requires SQLite 3.35.0 or higher", +) def test_query_preserves_error_from_transaction_destroying_trigger(fresh_db): # RAISE(ROLLBACK) destroys the savepoint guard - the original # IntegrityError must propagate, not "no such savepoint" From 9a2c58246528109c63afa48af3c0c5b5f943f2b0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:26:58 -0700 Subject: [PATCH 105/120] README tweak ready for v4, refs #769 --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e425461..c444c64 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,12 @@ Python CLI utility and library for manipulating SQLite databases. - [Configure SQLite full-text search](https://sqlite-utils.datasette.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance - Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as changing the type of a column - [Extract columns](https://sqlite-utils.datasette.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data +- [Manage database migrations](https://sqlite-utils.datasette.io/en/stable/migrations.html) using Python migration files and the `sqlite-utils migrate` command - [Install plugins](https://sqlite-utils.datasette.io/en/stable/plugins.html) to add custom SQL functions and additional features -Read more on my blog, in this series of posts on [New features in sqlite-utils](https://simonwillison.net/series/sqlite-utils-features/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/). +Upgrading from sqlite-utils 3.x? See the [4.0 upgrade guide](https://sqlite-utils.datasette.io/en/stable/upgrading.html#upgrading-from-3-x-to-4-0). + +Read more on my blog, in this series of posts on [New features in sqlite-utils](https://simonwillison.net/series/sqlite-utils-features/) and other [entries tagged sqlite-utils](https://simonwillison.net/tags/sqlite-utils/). ## Installation From d34f1bea0b34b7004a56b75b5dd083f89f8cc970 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:33:50 -0700 Subject: [PATCH 106/120] Release 4.0rc4 Refs #186, #554, #566, #625, #732, #769 --- docs/changelog.rst | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9228fb6..0943dca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,10 +4,10 @@ Changelog =========== -.. _unreleased: +.. _v4_0rc4: -Unreleased ----------- +4.0rc4 (2026-07-06) +------------------- - **Breaking change**: ``table.extract()`` - and the ``sqlite-utils extract`` command - no longer extract rows where every extracted column is ``null``. Those rows now keep a ``null`` value in the new foreign key column instead of pointing at an all-``null`` record in the lookup table. When extracting multiple columns, rows are still extracted if at least one of the columns has a value. (:issue:`186`) - The ``extracts=`` option to ``table.insert()`` and friends no longer creates a lookup table record for ``None`` values - the column value stays ``null``. Previously every batch of inserted rows containing a ``None`` value would add a duplicate ``null`` record to the lookup table. diff --git a/pyproject.toml b/pyproject.toml index 01f1735..c1024c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.0rc3" +version = "4.0rc4" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From d314d04215f7337d42c847214861ec7ffe0bf757 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 6 Jul 2026 22:40:37 -0700 Subject: [PATCH 107/120] Bump GitHub Actions versions --- .github/workflows/publish.yml | 8 ++++---- .github/workflows/test-coverage.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 396e2b8..23b23bd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,9 +12,9 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] os: [ubuntu-latest, windows-latest, macos-latest] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip @@ -29,9 +29,9 @@ jobs: runs-on: ubuntu-latest needs: [test] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.14' cache: pip diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 1f55f4e..7668f1b 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -12,9 +12,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repo - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" cache: pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d85cbf8..923de2e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,9 +14,9 @@ jobs: numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest, macos-14] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} allow-prereleases: true From 60811e730509667f702bc08f9bf5fc3fe13b7f45 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 08:24:24 -0700 Subject: [PATCH 108/120] Fix rowid pk and last_rowid regressions in insert/upsert Closes #781, #783 Refs https://github.com/simonw/sqlite-utils/issues/769#issuecomment-4900618685 * Fix rowid pk and last_rowid regressions in insert/upsert Two behaviour regressions in the 4.0 insert/upsert rewrite broke callers (notably Datasette's write API) that operate on tables without an explicit primary key. Both are fixed here with regression tests. 1. rowid (and its aliases _rowid_/oid) were rejected as a primary key. Table.pks already reports ["rowid"] for a rowid table, but the new pk validation raised InvalidColumns because rowid is not listed among the table's columns, and the insert success path then raised KeyError when looking up the pk value. rowid aliases are now accepted for rowid tables and resolve directly to the rowid. 2. An ignored insert (INSERT OR IGNORE that matched an existing row) no longer populated last_rowid, and only set last_pk when an explicit pk= was passed. It now locates the existing conflicting row by its primary key values and reports that row's rowid and pk, rather than relying on the connection's last inserted rowid. Add a shared ROWID_ALIASES constant for the rowid alias names. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01E7af8SxFZqiCerJB6MqKnY --- sqlite_utils/db.py | 97 +++++++++++++++++++++++++++++++------------- tests/test_create.py | 90 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 28 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index bec68fc..3033a36 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -53,6 +53,11 @@ except ImportError: SQLITE_MAX_VARS = 999 +# Names that refer to a rowid table's implicit integer primary key. These are +# valid primary key targets even though they are not listed among a table's +# columns. See https://www.sqlite.org/lang_createtable.html#rowid +ROWID_ALIASES = frozenset({"rowid", "_rowid_", "oid"}) + _quote_fts_re = re.compile(r'\s+|(".*?")') _virtual_table_using_re = re.compile( @@ -4343,10 +4348,14 @@ class Table(Queryable): if pk and not hash_id and self.exists(): pk_cols = [pk] if isinstance(pk, str) else list(pk) existing_columns = self.columns_dict + # rowid and its aliases are valid primary keys for a rowid table + # even though they are not listed among the table's columns + rowid_aliases = ROWID_ALIASES if self.use_rowid else frozenset() missing_pk_cols = [ col for col in pk_cols - if resolve_casing(col, existing_columns) not in existing_columns + if col.lower() not in rowid_aliases + and resolve_casing(col, existing_columns) not in existing_columns ] if missing_pk_cols: invalid_pk_error = InvalidColumns( @@ -4512,40 +4521,72 @@ class Table(Queryable): if not upsert and result is not None: ignored_insert = ignore and result.rowcount == 0 if ignored_insert: + # The row was not inserted because it conflicts with an + # existing row. Point last_pk / last_rowid at that existing + # row when we can identify it from the record's primary key + # values, rather than leaving them stale or unset. if list_mode: - first_record_list = cast(Sequence[Any], first_record) - if hash_id: - pass - elif isinstance(pk, str): - pk_index = column_names.index( - resolve_casing(pk, column_names) - ) - self.last_pk = first_record_list[pk_index] - elif pk: - self.last_pk = tuple( - first_record_list[ - column_names.index(resolve_casing(p, column_names)) - ] - for p in pk - ) + first_record_dict = dict( + zip(column_names, cast(Sequence[Any], first_record)) + ) else: first_record_dict = cast(Dict[str, Any], first_record) - if hash_id: - self.last_pk = hash_record( - first_record_dict, hash_id_columns - ) - elif isinstance(pk, str): - self.last_pk = first_record_dict[ - resolve_casing(pk, first_record_dict) + if hash_id: + self.last_pk = hash_record(first_record_dict, hash_id_columns) + elif isinstance(pk, str): + self.last_pk = first_record_dict[ + resolve_casing(pk, first_record_dict) + ] + elif pk: + self.last_pk = tuple( + first_record_dict[resolve_casing(p, first_record_dict)] + for p in pk + ) + # Locate the existing conflicting row using its primary key + # columns so we can report its rowid (and pk if not already + # known). Falls back to leaving them unset if the conflict + # cannot be resolved to a pk lookup (e.g. a UNIQUE column). + key_cols: Optional[List[str]] = None + if isinstance(pk, str): + key_cols = [pk] + elif pk: + key_cols = list(pk) + elif not hash_id and not self.use_rowid: + key_cols = self.pks + if key_cols: + try: + key_values = [ + first_record_dict[resolve_casing(c, first_record_dict)] + for c in key_cols ] - elif pk: - self.last_pk = tuple( - first_record_dict[resolve_casing(p, first_record_dict)] - for p in pk + except KeyError: + key_values = None + if key_values is not None: + where = " and ".join( + "{} = ?".format(quote_identifier(c)) for c in key_cols ) + existing = self.db.execute( + "select rowid from {} where {} limit 1".format( + quote_identifier(self.name), where + ), + key_values, + ).fetchone() + if existing is not None: + self.last_rowid = existing[0] + # On a primary key conflict the record's pk + # values identify the existing row + if self.last_pk is None: + self.last_pk = ( + key_values[0] + if len(key_cols) == 1 + else tuple(key_values) + ) else: self.last_rowid = result.lastrowid - if (hash_id or pk) and self.last_rowid: + # A rowid-alias pk resolves directly to the rowid, so there + # is no separate pk column to look up + rowid_pk = isinstance(pk, str) and pk.lower() in ROWID_ALIASES + if (hash_id or (pk and not rowid_pk)) and self.last_rowid: # Set self.last_pk to the pk(s) for that rowid row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] if hash_id: diff --git a/tests/test_create.py b/tests/test_create.py index 42fa1c4..7fab5e6 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -990,6 +990,96 @@ def test_insert_ignore(fresh_db): assert rows == [{"id": 1, "bar": 2}] +def test_insert_ignore_reports_existing_row(fresh_db): + # An ignored insert (row already exists) should point last_rowid and + # last_pk at the existing conflicting row - see the Datasette insert API + fresh_db["docs"].insert({"id": 1, "title": "Exists"}, pk="id") + # Insert a conflicting row with ignore=True and no explicit pk= + table = fresh_db["docs"].insert({"id": 1, "title": "One"}, ignore=True) + assert table.last_rowid == 1 + assert table.last_pk == 1 + assert list(fresh_db["docs"].rows_where("rowid = ?", [table.last_rowid])) == [ + {"id": 1, "title": "Exists"} + ] + + +@pytest.mark.parametrize("rowid_alias", ("rowid", "_rowid_", "oid")) +@pytest.mark.parametrize("method", ("upsert", "insert_replace", "insert_ignore")) +def test_pk_rowid_alias_on_rowid_table(fresh_db, rowid_alias, method): + # rowid and its aliases are valid primary keys for a rowid table even + # though they are not listed among the table's columns - see the Datasette + # upsert API against tables without an explicit primary key + fresh_db["t"].insert({"title": "Hello"}) + assert fresh_db["t"].pks == ["rowid"] + record = {rowid_alias: 1, "title": "Updated"} + if method == "upsert": + table = fresh_db["t"].upsert(record, pk=rowid_alias) + elif method == "insert_replace": + table = fresh_db["t"].insert(record, pk=rowid_alias, replace=True) + else: + table = fresh_db["t"].insert(record, pk=rowid_alias, ignore=True) + assert table.last_pk == 1 + expected_title = "Hello" if method == "insert_ignore" else "Updated" + assert list(fresh_db["t"].rows) == [{"title": expected_title}] + + +def test_insert_ignore_reports_existing_row_compound_pk(fresh_db): + # Compound primary key variant of the ignored-insert lookup + fresh_db["t"].insert_all([{"a": 1, "b": 2, "note": "first"}], pk=("a", "b")) + table = fresh_db["t"].insert( + {"a": 1, "b": 2, "note": "second"}, pk=("a", "b"), ignore=True + ) + assert table.last_pk == (1, 2) + assert list(fresh_db["t"].rows_where("rowid = ?", [table.last_rowid])) == [ + {"a": 1, "b": 2, "note": "first"} + ] + + +def test_insert_ignore_reports_existing_row_list_mode(fresh_db): + # List-based iteration variant of the ignored-insert lookup + fresh_db["t"].insert_all([["id", "title"], [1, "first"]], pk="id") + table = fresh_db["t"].insert_all( + [["id", "title"], [1, "second"]], pk="id", ignore=True + ) + assert table.last_pk == 1 + assert table.last_rowid == 1 + assert list(fresh_db["t"].rows) == [{"id": 1, "title": "first"}] + + +def test_insert_ignore_hash_id_reports_pk(fresh_db): + # With hash_id the pk is the computed hash; the original record has no id + # column to look up so last_rowid is left unset + first = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id") + table = fresh_db["dogs"].insert({"name": "Cleo"}, hash_id="id", ignore=True) + assert table.last_pk == first.last_pk + assert table.last_rowid is None + assert fresh_db["dogs"].count == 1 + + +def test_insert_ignore_unresolvable_conflict_leaves_pk_unset(fresh_db): + # When the conflict cannot be resolved to a primary key lookup, last_pk and + # last_rowid are left unset rather than reporting a misleading value + + # rowid table with a UNIQUE column and no primary key: no pk to look up + fresh_db["u"].db.execute("create table u (title text unique)") + fresh_db["u"].insert({"title": "x"}) + table = fresh_db["u"].insert({"title": "x"}, ignore=True) + assert table.last_pk is None + assert table.last_rowid is None + assert fresh_db["u"].count == 1 + + # Conflict on a UNIQUE column other than the primary key: the pk value from + # the record does not match the existing row, so the lookup finds nothing + fresh_db["docs"].db.execute( + "create table docs (id integer primary key, email text unique)" + ) + fresh_db["docs"].insert({"id": 1, "email": "a"}, pk="id") + table = fresh_db["docs"].insert({"id": 2, "email": "a"}, ignore=True) + assert table.last_pk is None + assert table.last_rowid is None + assert fresh_db["docs"].count == 1 + + def test_insert_ignore_with_pk_after_other_table_insert(fresh_db): # https://github.com/simonw/sqlite-utils/issues/554 user = {"id": "abc", "name": "david"} From 8bc9213a8e9c52a2d46ffc14da72fcd5af276840 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 08:40:31 -0700 Subject: [PATCH 109/120] Release 4.0 Refs #781, #783, #769 --- docs/changelog.rst | 31 +++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0943dca..9623ec2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,37 @@ Changelog =========== +.. _v4_0: + +4.0 (2026-07-07) +---------------- + +The 4.0 release includes some minor backwards-incompatible fixes (hence the major version number bump) and introduces three major new features: + +- :ref:`Database migrations `, providing a structured mechanism for evolving a project's schema over time. (:issue:`752`) +- :ref:`Nested transaction support ` via ``db.atomic()``, plus numerous improvements to how transactions work across the library. (:issue:`755`) +- Support for :ref:`compound foreign keys `, including creation, transformation and introspection through :ref:`table.foreign_keys `. (:issue:`594`) + +Other notable changes include: + +- Upserts now use SQLite's ``INSERT ... ON CONFLICT ... DO UPDATE SET`` syntax, detect existing table primary keys automatically and reject records that are missing required primary key values. (:issue:`652`) +- ``db.query()`` now executes immediately and rejects statements that do not return rows; use ``db.execute()`` for writes and DDL. +- CSV and TSV imports now detect column types by default, while inserts into existing tables preserve those tables' column types. (:issue:`679`) +- Foreign key handling now preserves ``ON DELETE``/``ON UPDATE`` actions during transforms and resolves referenced primary keys more accurately. (:issue:`530`) +- Column names passed to Python API methods are now matched case-insensitively, mirroring SQLite's own identifier behavior. (:issue:`760`) +- The command-line tool now emits UTF-8 JSON output by default, with ``--ascii`` available to restore escaped output. (:issue:`625`) +- ``table.extract()`` and ``extracts=`` no longer create lookup table records for all-``null`` values. (:issue:`186`) + +See :ref:`upgrading_3_to_4` for details on backwards-incompatible changes. + +The detailed release notes for the features and fixes shipped during the 4.0 pre-release cycle are available in :ref:`4.0a0 `, :ref:`4.0a1 `, :ref:`4.0rc1 `, :ref:`4.0rc2 `, :ref:`4.0rc3 ` and :ref:`4.0rc4 `. + +Bug fixes since 4.0rc4 +~~~~~~~~~~~~~~~~~~~~~~ + +- Fixed 4.0 regressions in ``insert``/``upsert`` against tables that use SQLite's implicit ``rowid`` primary key. Passing ``pk="rowid"``, ``pk="_rowid_"`` or ``pk="oid"`` now works again for rowid tables, and ``last_pk`` is set correctly. (:issue:`781`) +- Fixed ``insert(..., ignore=True)`` and ``insert_all(..., ignore=True)`` so an ignored insert that conflicts with an existing primary key row now reports that existing row in ``last_rowid`` and ``last_pk`` where possible. This also works for compound primary keys and list-mode inserts. (:issue:`783`) + .. _v4_0rc4: 4.0rc4 (2026-07-06) diff --git a/pyproject.toml b/pyproject.toml index c1024c2..a3a42a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlite-utils" -version = "4.0rc4" +version = "4.0" description = "CLI tool and Python library for manipulating SQLite databases" readme = { file = "README.md", content-type = "text/markdown" } authors = [ From 353baf280d3765b9680a49276945a0e96bc9e238 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 11:51:53 -0700 Subject: [PATCH 110/120] Corrected imports in migrations docs --- docs/migrations.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/migrations.rst b/docs/migrations.rst index cfdbf13..23aa9d8 100644 --- a/docs/migrations.rst +++ b/docs/migrations.rst @@ -27,7 +27,7 @@ Here is a simple example of a ``migrations.py`` file which creates a table, then .. code-block:: python - from sqlite_utils import Database, Migrations + from sqlite_utils import Migrations migrations = Migrations("creatures") @@ -51,6 +51,8 @@ Once you have a ``Migrations(name)`` collection with one or more migrations regi .. code-block:: python + from sqlite_utils import Database + db = Database("creatures.db") migrations.apply(db) From 619770bf427e47657099f1a4fdaa8777ec4c68d8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 17:52:26 -0700 Subject: [PATCH 111/120] sqlite-utils query - to read SQL from stdin, closes #765 --- docs/changelog.rst | 7 +++++++ docs/cli-reference.rst | 4 ++++ docs/cli.rst | 10 ++++++++++ sqlite_utils/cli.py | 8 ++++++++ tests/test_cli.py | 19 +++++++++++++++++++ 5 files changed, 48 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9623ec2..cf9486a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog =========== +.. _v_unreleased: + +Unreleased +---------- + +- ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) + .. _v4_0: 4.0 (2026-07-07) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 49eba52..000f88f 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -109,6 +109,10 @@ See :ref:`cli_query`. "select * from chickens where age > :age" \ -p age 1 + Pass "-" as the SQL to read the query from standard input: + + echo "select * from chickens" | sqlite-utils data.db - + Options: --attach ... Additional databases to attach - specify alias and filepath diff --git a/docs/cli.rst b/docs/cli.rst index 063e80f..bbbbf03 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -29,6 +29,16 @@ The ``sqlite-utils query`` command lets you run queries directly against a SQLit .. note:: In Python: :ref:`db.query() ` CLI reference: :ref:`sqlite-utils query ` +Pass ``-`` as the SQL query to read the query from standard input. This is useful for longer queries that would otherwise require careful shell escaping, or for piping in SQL generated by another tool: + +.. code-block:: bash + + echo "select * from dogs" | sqlite-utils query dogs.db - + +.. code-block:: bash + + sqlite-utils query dogs.db - < query.sql + .. _cli_query_json: Returning JSON diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index fe7dbe4..c6496d4 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1893,7 +1893,15 @@ def query( sqlite-utils data.db \\ "select * from chickens where age > :age" \\ -p age 1 + + Pass "-" as the SQL to read the query from standard input: + + \b + echo "select * from chickens" | sqlite-utils data.db - """ + if sql == "-": + # Read SQL from standard input + sql = sys.stdin.read() db = sqlite_utils.Database(path) _register_db_for_cleanup(db) for alias, attach_path in attach: diff --git a/tests/test_cli.py b/tests/test_cli.py index 06e14ea..8196a08 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -782,6 +782,25 @@ def test_query_json(db_path, sql, args, expected): assert expected == result.output.strip() +def test_query_sql_from_stdin(db_path): + # https://github.com/simonw/sqlite-utils/issues/765 + db = Database(db_path) + with db.conn: + db["dogs"].insert_all( + [ + {"id": 1, "age": 4, "name": "Cleo"}, + {"id": 2, "age": 2, "name": "Pancakes"}, + ] + ) + result = CliRunner().invoke( + cli.cli, + ["query", db_path, "-"], + input="select name from dogs order by name", + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output) == [{"name": "Cleo"}, {"name": "Pancakes"}] + + def test_query_json_empty(db_path): result = CliRunner().invoke( cli.cli, From fa5d66bf5377b00e976ebb3bf92d806ed8f54270 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 17:56:57 -0700 Subject: [PATCH 112/120] Update create-table --help to mention real --- docs/cli-reference.rst | 4 ++-- sqlite_utils/cli.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 000f88f..fd1199f 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -930,10 +930,10 @@ See :ref:`cli_create_table`. sqlite-utils create-table my.db people \ id integer \ name text \ - height float \ + height real \ photo blob --pk id - Valid column types are text, integer, float and blob. + Valid column types are text, integer, real, float and blob. Options: --pk TEXT Column to use as primary key diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c6496d4..905963f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1634,10 +1634,10 @@ def create_table( sqlite-utils create-table my.db people \\ id integer \\ name text \\ - height float \\ + height real \\ photo blob --pk id - Valid column types are text, integer, float and blob. + Valid column types are text, integer, real, float and blob. """ db = sqlite_utils.Database(path) _register_db_for_cleanup(db) From 8ee0b7c65cfa34f550672872bffa06183bea9963 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 17:57:29 -0700 Subject: [PATCH 113/120] Fix for rogue quote in enable-fts --help --- docs/cli-reference.rst | 2 +- sqlite_utils/cli.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index fd1199f..e57f4b3 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -1028,7 +1028,7 @@ See :ref:`cli_fts`. Usage: sqlite-utils enable-fts [OPTIONS] PATH TABLE COLUMN... - Enable full-text search for specific table and columns" + Enable full-text search for specific table and columns Example: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 905963f..99a8c64 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -706,7 +706,7 @@ def create_index( def enable_fts( path, table, column, fts4, fts5, tokenize, create_triggers, replace, load_extension ): - """Enable full-text search for specific table and columns" + """Enable full-text search for specific table and columns Example: From cf3373e7b7fb9342d705d7b5b8d52fe09ac46a34 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 17:59:58 -0700 Subject: [PATCH 114/120] Refactor --functions option, improve help --- docs/cli-reference.rst | 11 ++++++----- sqlite_utils/cli.py | 29 ++++++++++++++--------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index e57f4b3..c232881 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -138,8 +138,8 @@ See :ref:`cli_query`. -r, --raw Raw output, first column of first row --raw-lines Raw output, first column of each row -p, --param ... Named :parameters for SQL query - --functions TEXT Python code or file path defining custom SQL - functions + --functions TEXT Python code or a file path defining custom SQL + functions; can be used multiple times --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. @@ -181,8 +181,8 @@ See :ref:`cli_memory`. sqlite-utils memory animals.csv --schema Options: - --functions TEXT Python code or file path defining custom SQL - functions + --functions TEXT Python code or a file path defining custom SQL + functions; can be used multiple times --attach ... Additional databases to attach - specify alias and filepath --flatten Flatten nested JSON objects, so {"foo": {"bar": @@ -383,7 +383,8 @@ See :ref:`cli_bulk`. Options: --batch-size INTEGER Commit every X records - --functions TEXT Python code or file path defining custom SQL functions + --functions TEXT Python code or a file path defining custom SQL + functions; can be used multiple times --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 99a8c64..edf1634 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -153,6 +153,17 @@ def load_extension_option(fn): )(fn) +def functions_option(fn): + return click.option( + "--functions", + help=( + "Python code or a file path defining custom SQL functions; " + "can be used multiple times" + ), + multiple=True, + )(fn) + + @click.group( cls=DefaultGroup, default="query", @@ -1450,11 +1461,7 @@ def upsert( @click.argument("sql") @click.argument("file", type=click.File("rb"), required=True) @click.option("--batch-size", type=int, default=100, help="Commit every X records") -@click.option( - "--functions", - help="Python code or file path defining custom SQL functions", - multiple=True, -) +@functions_option @import_options @load_extension_option def bulk( @@ -1860,11 +1867,7 @@ def drop_view(path, view, ignore, load_extension): type=(str, str), help="Named :parameters for SQL query", ) -@click.option( - "--functions", - help="Python code or file path defining custom SQL functions", - multiple=True, -) +@functions_option @load_extension_option def query( path, @@ -1937,11 +1940,7 @@ def query( nargs=-1, ) @click.argument("sql") -@click.option( - "--functions", - help="Python code or file path defining custom SQL functions", - multiple=True, -) +@functions_option @click.option( "--attach", type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), From aa300942bfcba0bd788951db0b9070cf291c193a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 18:03:25 -0700 Subject: [PATCH 115/120] Update sqlite-utils convert --help, refs #686 --- docs/cli-reference.rst | 7 +++++-- sqlite_utils/cli.py | 9 ++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index c232881..105b19a 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -621,6 +621,11 @@ See :ref:`cli_convert`. "value" is a variable with the column value to be converted. + CODE can also be a reference to a callable that takes the value, for example: + + sqlite-utils convert my.db mytable date r.parsedate + sqlite-utils convert my.db mytable data json.loads --import json + Use "-" for CODE to read Python code from standard input. The following common operations are available as recipe functions: @@ -634,7 +639,6 @@ See :ref:`cli_convert`. errors: 'Optional[object]' = None) -> 'Optional[str]' Parse a date and convert it to ISO date format: yyyy-mm-dd - - dayfirst=True: treat xx as the day in xx/yy/zz - yearfirst=True: treat xx as the year in xx/yy/zz - errors=r.IGNORE to ignore values that cannot be parsed @@ -644,7 +648,6 @@ See :ref:`cli_convert`. False, errors: 'Optional[object]' = None) -> 'Optional[str]' Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS - - dayfirst=True: treat xx as the day in xx/yy/zz - yearfirst=True: treat xx as the year in xx/yy/zz - errors=r.IGNORE to ignore values that cannot be parsed diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index edf1634..318b208 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3054,6 +3054,12 @@ def _generate_convert_help(): "value" is a variable with the column value to be converted. + CODE can also be a reference to a callable that takes the value, for example: + + \b + sqlite-utils convert my.db mytable date r.parsedate + sqlite-utils convert my.db mytable data json.loads --import json + Use "-" for CODE to read Python code from standard input. The following common operations are available as recipe functions: @@ -3067,8 +3073,9 @@ def _generate_convert_help(): ] for name in recipe_names: fn = getattr(recipes, name) + doc = textwrap.dedent(fn.__doc__.rstrip()).replace("\b\n", "") help += "\n\nr.{}{}\n\n\b{}".format( - name, str(inspect.signature(fn)), textwrap.dedent(fn.__doc__.rstrip()) + name, str(inspect.signature(fn)), doc ) help += "\n\n" help += textwrap.dedent(""" From ebafb84c93bd3666e0e6acd24bfafe070f25cdee Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 18:09:12 -0700 Subject: [PATCH 116/120] Allow sqlite-utils upsert to infer --pk from existing table --- docs/cli-reference.rst | 3 ++- docs/cli.rst | 2 ++ sqlite_utils/cli.py | 7 +++++-- tests/test_cli.py | 24 +++++++++++++++++++++--- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 105b19a..5ea9200 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -321,6 +321,8 @@ See :ref:`cli_upsert`. incoming record has a primary key that matches an existing record the existing record will be updated. + If the table already exists and has a primary key, --pk can be omitted. + Example: echo '[ @@ -330,7 +332,6 @@ See :ref:`cli_upsert`. Options: --pk TEXT Columns to use as the primary key, e.g. id - [required] --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON diff --git a/docs/cli.rst b/docs/cli.rst index bbbbf03..e373704 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1593,6 +1593,8 @@ For example: This will update the dog with an ID of 2 to have an age of 4, creating a new record (with a null name) if one does not exist. If a row DOES exist the name will be left as-is. +If the table already exists and has a primary key, you can omit the ``--pk`` option and ``sqlite-utils`` will use that existing primary key. + The command will fail if you reference columns that do not exist on the table. To automatically create missing columns, use the ``--alter`` option. .. note:: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 318b208..5e21f92 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -16,6 +16,7 @@ from sqlite_utils.db import ( InvalidColumns, NoTable, NoView, + PrimaryKeyRequired, quote_identifier, ) from sqlite_utils.plugins import ensure_plugins_loaded, pm, get_plugins @@ -1184,7 +1185,7 @@ def insert_upsert_implementation( db.table(table).insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs ) - except (NoTable, InvalidColumns) as e: + except (NoTable, InvalidColumns, PrimaryKeyRequired) as e: raise click.ClickException(str(e)) except Exception as e: if ( @@ -1372,7 +1373,7 @@ def insert( @cli.command() -@insert_upsert_options(require_pk=True) +@insert_upsert_options() def upsert( path, table, @@ -1408,6 +1409,8 @@ def upsert( an incoming record has a primary key that matches an existing record the existing record will be updated. + If the table already exists and has a primary key, --pk can be omitted. + Example: \b diff --git a/tests/test_cli.py b/tests/test_cli.py index 8196a08..bc5d492 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1235,20 +1235,38 @@ def test_upsert(db_path, tmpdir): ] -def test_upsert_pk_required(db_path, tmpdir): +def test_upsert_pk_inferred_from_existing_table(db_path, tmpdir): json_path = str(tmpdir / "dogs.json") + db = Database(db_path) insert_dogs = [ {"id": 1, "name": "Cleo", "age": 4}, {"id": 2, "name": "Nixie", "age": 4}, ] write_json(json_path, insert_dogs) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", json_path, "--pk", "id"], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output + + write_json( + json_path, + [ + {"id": 1, "age": 5}, + {"id": 2, "age": 5}, + ], + ) result = CliRunner().invoke( cli.cli, ["upsert", db_path, "dogs", json_path], catch_exceptions=False, ) - assert result.exit_code == 2 - assert "Error: Missing option '--pk'" in result.output + assert result.exit_code == 0, result.output + assert list(db.query("select * from dogs order by id")) == [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Nixie", "age": 5}, + ] def test_upsert_analyze(db_path, tmpdir): From 23a21c1d6bc8aa1c17a49b37181aadffd7abb8be Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 18:11:34 -0700 Subject: [PATCH 117/120] Ran Black --- sqlite_utils/cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 5e21f92..b99dcf1 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -3077,9 +3077,7 @@ def _generate_convert_help(): for name in recipe_names: fn = getattr(recipes, name) doc = textwrap.dedent(fn.__doc__.rstrip()).replace("\b\n", "") - help += "\n\nr.{}{}\n\n\b{}".format( - name, str(inspect.signature(fn)), doc - ) + help += "\n\nr.{}{}\n\n\b{}".format(name, str(inspect.signature(fn)), doc) help += "\n\n" help += textwrap.dedent(""" You can use these recipes like so: From 569608e40f28893262d727adadddbf3d077112da Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 18:48:57 -0700 Subject: [PATCH 118/120] sqlite-utils insert --code option, closes #684 --- docs/changelog.rst | 1 + docs/cli-reference.rst | 18 ++- docs/cli.rst | 21 ++++ sqlite_utils/cli.py | 232 +++++++++++++++++++++++++++------------ tests/test_cli_insert.py | 150 +++++++++++++++++++++++++ 5 files changed, 347 insertions(+), 75 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cf9486a..a9dd4a3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) +- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for providing a block of Python code (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) .. _v4_0: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5ea9200..5798809 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -230,7 +230,7 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr :: - Usage: sqlite-utils insert [OPTIONS] PATH TABLE FILE + Usage: sqlite-utils insert [OPTIONS] PATH TABLE [FILE] Insert records from FILE into a table, creating the table if it does not already exist. @@ -272,8 +272,20 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr echo 'A bunch of words' | sqlite-utils insert words.db words - \ --text --convert '({"word": w} for w in text.split())' + Instead of a FILE you can use --code to provide a block of Python code that + defines the rows to insert, as either a rows() function that yields + dictionaries or a "rows" iterable. --code can also be a path to a .py file: + + sqlite-utils insert data.db creatures --code ' + def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} + ' --pk id + Options: --pk TEXT Columns to use as the primary key, e.g. id + --code TEXT Python code defining a rows() function or iterable + of rows to insert --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON @@ -315,7 +327,7 @@ See :ref:`cli_upsert`. :: - Usage: sqlite-utils upsert [OPTIONS] PATH TABLE FILE + Usage: sqlite-utils upsert [OPTIONS] PATH TABLE [FILE] Upsert records based on their primary key. Works like 'insert' but if an incoming record has a primary key that matches an existing record the existing @@ -332,6 +344,8 @@ See :ref:`cli_upsert`. Options: --pk TEXT Columns to use as the primary key, e.g. id + --code TEXT Python code defining a rows() function or iterable + of rows to insert --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON diff --git a/docs/cli.rst b/docs/cli.rst index e373704..4ed92ed 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1561,6 +1561,27 @@ The result looks like this: COMMIT; +.. _cli_insert_code: + +Inserting rows generated by Python code +======================================= + +Instead of providing a ``FILE`` to import, you can use the ``--code`` option to pass a block of Python code that generates the rows to insert. This is the command-line equivalent of calling ``db["creatures"].insert_all(rows())`` from the :ref:`Python API `. + +Your code should define either a ``rows()`` function that returns or yields dictionaries, or a ``rows`` iterable such as a list of dictionaries: + +.. code-block:: bash + + sqlite-utils insert data.db creatures --code ' + def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} + ' --pk id + +``--code`` can also be given a path to a Python ``.py`` file. + +The ``--code`` option works with both ``sqlite-utils insert`` and ``sqlite-utils upsert``, and composes with table options such as ``--pk``, ``--replace``, ``--alter``, ``--not-null`` and ``--default``. It cannot be combined with a ``FILE`` argument or with input format options such as ``--csv`` or ``--convert``. + .. _cli_insert_replace: Insert-replacing data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index b99dcf1..4eefc70 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -944,13 +944,19 @@ def insert_upsert_options(*, require_pk=False): required=True, ), click.argument("table"), - click.argument("file", type=click.File("rb", lazy=True), required=True), + click.argument( + "file", type=click.File("rb", lazy=True), required=False + ), click.option( "--pk", help="Columns to use as the primary key, e.g. id", multiple=True, required=require_pk, ), + click.option( + "--code", + help="Python code defining a rows() function or iterable of rows to insert", + ), ) + _import_options + ( @@ -1035,11 +1041,118 @@ def insert_upsert_implementation( bulk_sql=None, functions=None, strict=False, + code=None, ): db = sqlite_utils.Database(path) _register_db_for_cleanup(db) _load_extensions(db, load_extension) _maybe_register_functions(db, functions) + + def _insert_docs(docs, tracker=None): + extra_kwargs = { + "ignore": ignore, + "replace": replace, + "truncate": truncate, + "analyze": analyze, + "strict": strict, + } + if not_null: + extra_kwargs["not_null"] = set(not_null) + if default: + extra_kwargs["defaults"] = dict(default) + if upsert: + extra_kwargs["upsert"] = upsert + + # docs should all be dictionaries + docs = (verify_is_dict(doc) for doc in docs) + + # Apply {"$base64": true, ...} decoding, if needed + docs = (decode_base64_values(doc) for doc in docs) + + # For bulk_sql= we use cursor.executemany() instead + if bulk_sql: + if batch_size: + doc_chunks = chunks(docs, batch_size) + else: + doc_chunks = [docs] + for doc_chunk in doc_chunks: + with db.atomic(): + db.conn.cursor().executemany(bulk_sql, doc_chunk) + return + + # table_names() rather than db.table(), which raises NoTable for + # views before the error handling below can deal with them + table_existed_before_insert = table in db.table_names() + try: + db.table(table).insert_all( + docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs + ) + except (NoTable, InvalidColumns, PrimaryKeyRequired) as e: + raise click.ClickException(str(e)) + except Exception as e: + if ( + isinstance(e, OperationalError) + and e.args + and ( + "has no column named" in e.args[0] or "no such column" in e.args[0] + ) + ): + raise click.ClickException( + "{}\n\nTry using --alter to add additional columns".format( + e.args[0] + ) + ) + # If we can find sql= and parameters= arguments, show those + variables = _find_variables(e.__traceback__, ["sql", "parameters"]) + if "sql" in variables and "parameters" in variables: + raise click.ClickException( + "{}\n\nsql = {}\nparameters = {}".format( + str(e), variables["sql"], variables["parameters"] + ) + ) + else: + raise + # Apply detected types only to a table this command created - + # transforming a pre-existing table would rewrite its column types + # and corrupt values such as TEXT zip codes with leading zeros + if ( + tracker is not None + and not table_existed_before_insert + and db.table(table).exists() + ): + db.table(table).transform(types=tracker.types) + + if code is not None: + if file is not None: + raise click.ClickException("--code cannot be used with a FILE argument") + if any( + [ + flatten, + nl, + csv, + tsv, + empty_null, + lines, + text, + convert, + sniff, + no_headers, + delimiter, + quotechar, + encoding, + ] + ): + raise click.ClickException( + "--code cannot be used with input format options" + ) + _insert_docs(_rows_from_code(code)) + return + + if file is None: + raise click.ClickException( + "Provide either a FILE argument or --code to specify rows to insert" + ) + if (delimiter or quotechar or sniff or no_headers) and not tsv: csv = True if (nl + csv + tsv) >= 2: @@ -1147,78 +1260,7 @@ def insert_upsert_implementation( else: docs = (fn(doc) or doc for doc in docs) - extra_kwargs = { - "ignore": ignore, - "replace": replace, - "truncate": truncate, - "analyze": analyze, - "strict": strict, - } - if not_null: - extra_kwargs["not_null"] = set(not_null) - if default: - extra_kwargs["defaults"] = dict(default) - if upsert: - extra_kwargs["upsert"] = upsert - - # docs should all be dictionaries - docs = (verify_is_dict(doc) for doc in docs) - - # Apply {"$base64": true, ...} decoding, if needed - docs = (decode_base64_values(doc) for doc in docs) - - # For bulk_sql= we use cursor.executemany() instead - if bulk_sql: - if batch_size: - doc_chunks = chunks(docs, batch_size) - else: - doc_chunks = [docs] - for doc_chunk in doc_chunks: - with db.atomic(): - db.conn.cursor().executemany(bulk_sql, doc_chunk) - return - - # table_names() rather than db.table(), which raises NoTable for - # views before the error handling below can deal with them - table_existed_before_insert = table in db.table_names() - try: - db.table(table).insert_all( - docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs - ) - except (NoTable, InvalidColumns, PrimaryKeyRequired) as e: - raise click.ClickException(str(e)) - except Exception as e: - if ( - isinstance(e, OperationalError) - and e.args - and ( - "has no column named" in e.args[0] or "no such column" in e.args[0] - ) - ): - raise click.ClickException( - "{}\n\nTry using --alter to add additional columns".format( - e.args[0] - ) - ) - # If we can find sql= and parameters= arguments, show those - variables = _find_variables(e.__traceback__, ["sql", "parameters"]) - if "sql" in variables and "parameters" in variables: - raise click.ClickException( - "{}\n\nsql = {}\nparameters = {}".format( - str(e), variables["sql"], variables["parameters"] - ) - ) - else: - raise - # Apply detected types only to a table this command created - - # transforming a pre-existing table would rewrite its column types - # and corrupt values such as TEXT zip codes with leading zeros - if ( - tracker is not None - and not table_existed_before_insert - and db.table(table).exists() - ): - db.table(table).transform(types=tracker.types) + _insert_docs(docs, tracker=tracker) # Clean up open file-like objects if sniff_buffer: @@ -1261,6 +1303,7 @@ def insert( table, file, pk, + code, flatten, nl, csv, @@ -1332,6 +1375,17 @@ def insert( \b echo 'A bunch of words' | sqlite-utils insert words.db words - \\ --text --convert '({"word": w} for w in text.split())' + + Instead of a FILE you can use --code to provide a block of Python code + that defines the rows to insert, as either a rows() function that yields + dictionaries or a "rows" iterable. --code can also be a path to a .py file: + + \b + sqlite-utils insert data.db creatures --code ' + def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} + ' --pk id """ try: insert_upsert_implementation( @@ -1367,6 +1421,7 @@ def insert( not_null=not_null, default=default, strict=strict, + code=code, ) except UnicodeDecodeError as ex: raise click.ClickException(UNICODE_ERROR.format(ex)) @@ -1379,6 +1434,7 @@ def upsert( table, file, pk, + code, flatten, nl, csv, @@ -1450,6 +1506,7 @@ def upsert( load_extension=load_extension, silent=silent, strict=strict, + code=code, ) except UnicodeDecodeError as ex: raise click.ClickException(UNICODE_ERROR.format(ex)) @@ -3669,3 +3726,32 @@ def _maybe_register_functions(db, functions_list): for functions in functions_list: if isinstance(functions, str) and functions.strip(): _register_functions(db, functions) + + +def _rows_from_code(code): + # code may be a path to a .py file + if "\n" not in code and code.endswith(".py"): + try: + code = pathlib.Path(code).read_text() + except FileNotFoundError: + raise click.ClickException("File not found: {}".format(code)) + namespace = {} + try: + exec(code, namespace) + except SyntaxError as ex: + raise click.ClickException("Error in --code: {}".format(ex)) + rows = namespace.get("rows") + if callable(rows): + rows = rows() + if isinstance(rows, dict): + rows = [rows] + error = click.ClickException( + "--code must define a 'rows' function or iterable of rows to insert" + ) + if rows is None or isinstance(rows, (str, bytes)): + raise error + try: + iter(rows) + except TypeError: + raise error + return rows diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 5af8a2f..e290f97 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -691,3 +691,153 @@ def test_insert_invalid_pk_clean_error(db_path): assert result.exit_code == 1 assert result.exception is None or isinstance(result.exception, SystemExit) assert result.output.startswith("Error: Invalid primary key column") + + +# --code tests, see https://github.com/simonw/sqlite-utils/issues/684 +CODE_ROWS_FUNCTION = """ +def rows(): + yield {"id": 1, "name": "Cleo"} + yield {"id": 2, "name": "Suna"} +""" + +CODE_ROWS_ITERABLE = """ +rows = [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, +] +""" + + +@pytest.mark.parametrize("code", (CODE_ROWS_FUNCTION, CODE_ROWS_ITERABLE)) +def test_insert_code(tmpdir, code): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", code, "--pk", "id"], + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert db["creatures"].pks == ["id"] + assert list(db["creatures"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, + ] + + +def test_insert_code_from_file(tmpdir): + db_path = str(tmpdir / "dogs.db") + code_path = str(tmpdir / "gen.py") + with open(code_path, "w") as fp: + fp.write(CODE_ROWS_FUNCTION) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", code_path], + ) + assert result.exit_code == 0, result.output + assert list(Database(db_path)["creatures"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, + ] + + +def test_upsert_code(tmpdir): + db_path = str(tmpdir / "dogs.db") + db = Database(db_path) + db["creatures"].insert_all( + [{"id": 1, "name": "old"}, {"id": 2, "name": "Suna"}], pk="id" + ) + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--pk", "id"], + ) + assert result.exit_code == 0, result.output + assert list(db["creatures"].rows) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, + ] + + +def test_insert_code_requires_file_or_code(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke(cli.cli, ["insert", db_path, "creatures"]) + assert result.exit_code == 1 + assert "Provide either a FILE argument or --code" in result.output + + +def test_insert_code_mutually_exclusive_with_file(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "-", "--code", CODE_ROWS_FUNCTION], + input="{}", + ) + assert result.exit_code == 1 + assert "--code cannot be used with a FILE argument" in result.output + + +def test_insert_code_rejects_input_format_options(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", CODE_ROWS_FUNCTION, "--csv"], + ) + assert result.exit_code == 1 + assert "--code cannot be used with input format options" in result.output + + +def test_insert_code_missing_rows(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "x = 1"], + ) + assert result.exit_code == 1 + assert "must define a 'rows' function or iterable" in result.output + + +def test_insert_code_single_dict(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "creatures", + "--code", + 'rows = {"id": 1, "name": "Cleo"}', + "--pk", + "id", + ], + ) + assert result.exit_code == 0, result.output + assert list(Database(db_path)["creatures"].rows) == [{"id": 1, "name": "Cleo"}] + + +def test_insert_code_not_iterable(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "rows = 5"], + ) + assert result.exit_code == 1 + assert "must define a 'rows' function or iterable" in result.output + + +def test_insert_code_syntax_error(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "def rows(:"], + ) + assert result.exit_code == 1 + assert "Error in --code" in result.output + + +def test_insert_code_file_not_found(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "creatures", "--code", "missing.py"], + ) + assert result.exit_code == 1 + assert "File not found: missing.py" in result.output From 092f0919c3c254de0801dda60e7bbf8b73818c6c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 18:57:27 -0700 Subject: [PATCH 119/120] Changelog tweak refs #684 --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a9dd4a3..5d1816a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ Unreleased ---------- - ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) -- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for providing a block of Python code (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) +- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) .. _v4_0: From d2ac3765ed9f0516bb0cbc2508a5c3907fb6a71a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 19:17:29 -0700 Subject: [PATCH 120/120] sqlite-utils insert/upsert --type colunm-name type option, closes #131 --- docs/changelog.rst | 1 + docs/cli-reference.rst | 8 +++++++ docs/cli.rst | 19 ++++++++++++++++ sqlite_utils/cli.py | 28 +++++++++++++++++++++++- tests/test_cli_insert.py | 47 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5d1816a..9e03f39 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased - ``sqlite-utils query`` can now read the SQL query from standard input by passing ``-`` in place of the query, for example ``echo "select * from dogs" | sqlite-utils query dogs.db -``. (:issue:`765`) - ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept a ``--code`` option for :ref:`providing a block of Python code ` (or a path to a ``.py`` file) that defines a ``rows()`` function or ``rows`` iterable of rows to insert, as an alternative to importing from a file. (:issue:`684`) +- ``sqlite-utils insert`` and ``sqlite-utils upsert`` now accept ``--type column-name type`` to :ref:`override the type automatically chosen when the table is created `. This is useful for CSV or TSV columns such as ZIP codes that look like integers but should be stored as ``TEXT`` to preserve leading zeros. (:issue:`131`) .. _v4_0: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 5798809..39226ac 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -246,6 +246,9 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr - Use --lines to write each incoming line to a column called "line" - Use --text to write the entire input to a column called "text" + Use --type column-name type to override the type automatically chosen when the + table is created. + You can also use --convert to pass a fragment of Python code that will be used to convert each input. @@ -306,6 +309,7 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr --alter Alter existing table to add any missing columns --not-null TEXT Columns that should be created as NOT NULL --default ... Default value that should be set for a column + --type ... Column types to use when creating the table --no-detect-types Treat all CSV/TSV columns as TEXT --analyze Run ANALYZE at the end of this operation --load-extension TEXT Path to SQLite extension, with optional :entrypoint @@ -335,6 +339,9 @@ See :ref:`cli_upsert`. If the table already exists and has a primary key, --pk can be omitted. + Use --type column-name type to override the type automatically chosen when the + table is created. + Example: echo '[ @@ -366,6 +373,7 @@ See :ref:`cli_upsert`. --alter Alter existing table to add any missing columns --not-null TEXT Columns that should be created as NOT NULL --default ... Default value that should be set for a column + --type ... Column types to use when creating the table --no-detect-types Treat all CSV/TSV columns as TEXT --analyze Run ANALYZE at the end of this operation --load-extension TEXT Path to SQLite extension, with optional :entrypoint diff --git a/docs/cli.rst b/docs/cli.rst index 4ed92ed..731e93b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1366,6 +1366,25 @@ Will produce this schema with automatically detected types: "weight" REAL ); +.. _cli_insert_csv_tsv_column_types: + +Overriding column types +----------------------- + +Use ``--type column-name type`` to override the type automatically chosen when the table is created. This option can be used more than once, and works with both ``insert`` and ``upsert``: + +.. code-block:: bash + + sqlite-utils insert places.db places places.csv --csv \ + --type zipcode text \ + --type score real + +This is useful for values such as ZIP codes, which may look like integers but should be stored as ``TEXT`` to preserve leading zeros. + +The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL`` or ``BLOB``. Column types are matched case-insensitively. + +As with detected column types, ``--type`` only affects tables created by the command. If the table already exists, its existing column types are left unchanged. + To disable type detection and treat all columns as TEXT, use ``--no-detect-types``: .. code-block:: bash diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 4eefc70..cf39ff8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -980,6 +980,16 @@ def insert_upsert_options(*, require_pk=False): type=(str, str), help="Default value that should be set for a column", ), + click.option( + "--type", + "types", + type=( + str, + click.Choice(list(VALID_COLUMN_TYPES), case_sensitive=False), + ), + multiple=True, + help="Column types to use when creating the table", + ), click.option( "--no-detect-types", is_flag=True, @@ -1034,6 +1044,7 @@ def insert_upsert_implementation( truncate=False, not_null=None, default=None, + types=None, no_detect_types=False, analyze=False, load_extension=None, @@ -1047,6 +1058,7 @@ def insert_upsert_implementation( _register_db_for_cleanup(db) _load_extensions(db, load_extension) _maybe_register_functions(db, functions) + column_type_overrides = {column: ctype.upper() for column, ctype in (types or [])} def _insert_docs(docs, tracker=None): extra_kwargs = { @@ -1060,6 +1072,8 @@ def insert_upsert_implementation( extra_kwargs["not_null"] = set(not_null) if default: extra_kwargs["defaults"] = dict(default) + if column_type_overrides: + extra_kwargs["columns"] = column_type_overrides if upsert: extra_kwargs["upsert"] = upsert @@ -1120,7 +1134,9 @@ def insert_upsert_implementation( and not table_existed_before_insert and db.table(table).exists() ): - db.table(table).transform(types=tracker.types) + detected_types = tracker.types + detected_types.update(column_type_overrides) + db.table(table).transform(types=detected_types) if code is not None: if file is not None: @@ -1330,6 +1346,7 @@ def insert( truncate, not_null, default, + types, strict, ): """ @@ -1348,6 +1365,9 @@ def insert( - Use --lines to write each incoming line to a column called "line" - Use --text to write the entire input to a column called "text" + Use --type column-name type to override the type automatically chosen + when the table is created. + You can also use --convert to pass a fragment of Python code that will be used to convert each input. @@ -1420,6 +1440,7 @@ def insert( silent=silent, not_null=not_null, default=default, + types=types, strict=strict, code=code, ) @@ -1454,6 +1475,7 @@ def upsert( alter, not_null, default, + types, no_detect_types, analyze, load_extension, @@ -1467,6 +1489,9 @@ def upsert( If the table already exists and has a primary key, --pk can be omitted. + Use --type column-name type to override the type automatically chosen + when the table is created. + Example: \b @@ -1501,6 +1526,7 @@ def upsert( upsert=True, not_null=not_null, default=default, + types=types, no_detect_types=no_detect_types, analyze=analyze, load_extension=load_extension, diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index e290f97..df6f80c 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -664,6 +664,53 @@ def test_insert_csv_detect_types_new_table(db_path): assert db["data"].columns_dict == {"name": str, "age": int, "weight": float} +@pytest.mark.parametrize( + "command,extra_args,input_text,expected_row", + ( + ( + "insert", + [], + "zipcode,score\n01234,9.5\n", + {"zipcode": "01234", "score": 9.5}, + ), + ( + "upsert", + ["--pk", "id"], + "id,zipcode,score\n1,01234,9.5\n", + {"id": 1, "zipcode": "01234", "score": 9.5}, + ), + ), +) +def test_insert_upsert_csv_type_overrides_detected_types( + db_path, command, extra_args, input_text, expected_row +): + result = CliRunner().invoke( + cli.cli, + [ + command, + db_path, + "places", + "-", + "--csv", + ] + + extra_args + + [ + "--type", + "zipcode", + "text", + ], + catch_exceptions=False, + input=input_text, + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + expected_columns = {"zipcode": str, "score": float} + if command == "upsert": + expected_columns = {"id": int, **expected_columns} + assert db["places"].columns_dict == expected_columns + assert list(db["places"].rows) == [expected_row] + + def test_upsert_csv_detect_types_leaves_existing_table_alone(db_path): db = Database(db_path) db["places"].insert({"id": 1, "name": "Boston", "zip": "01234"}, pk="id")