From fcfccea8132e4aa6167a14f9afec5a690de7485c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 12 Aug 2026 16:43:33 -0700 Subject: [PATCH] Support ANY column types for strict tables Closes #790, #820 --- docs/changelog.rst | 1 + docs/cli-reference.rst | 6 +-- docs/cli.rst | 17 +++++++- docs/python-api.rst | 23 +++++++++- sqlite_utils/__init__.py | 11 ++++- sqlite_utils/cli.py | 22 ++++++---- sqlite_utils/db.py | 14 +++++++ sqlite_utils/utils.py | 6 +++ tests/test_cli.py | 79 ++++++++++++++++++++++++++++++++++- tests/test_column_affinity.py | 3 ++ tests/test_create.py | 40 ++++++++++++++++++ tests/test_extract.py | 36 ++++++++++++++++ tests/test_transform.py | 50 ++++++++++++++++++++++ 13 files changed, 292 insertions(+), 16 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 31b4ea3..0ef85ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,6 +10,7 @@ Unreleased ---------- - New ``table.checks``, ``table.column_checks`` and ``table.table_checks`` introspection properties expose column-level and table-level ``CHECK`` constraints. (:issue:`834`) +- New ``sqlite_utils.ANY`` marker type for creating and introspecting SQLite ``ANY`` columns. The Python API and CLI can create, add and transform these columns, and ``table.transform()`` and ``table.extract()`` now preserve ``ANY`` columns and their values in ``STRICT`` tables. (:issue:`790`) - ``table.transform()`` now preserves ``CHECK`` constraints, including comments within their expressions. Renaming a column rewrites identifier references in checks without changing string literals or function names. Dropping a column drops a check owned by that column, and raises ``TransformError`` if a remaining check depends on it. (:issue:`762`) - ``table.transform()`` now preserves comments immediately before or after column definitions. These comments move with the column if it is renamed or reordered, and are removed if the column is dropped. (:issue:`762`) - ``table.transform(rename=...)`` now preserves explicit indexes on renamed columns by dropping and recreating those indexes against the new column names. Previously this raised a ``TransformError``. (:issue:`822`) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index a4ec402..c53d642 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -494,7 +494,7 @@ See :ref:`cli_transform_table`. Options: --type ... Change column type to INTEGER, TEXT, FLOAT, - REAL or BLOB + REAL, BLOB or ANY --drop TEXT Drop this column --rename ... Rename this column to X -o, --column-order TEXT Reorder columns @@ -963,7 +963,7 @@ See :ref:`cli_create_table`. height real \ photo blob --pk id - Valid column types are text, integer, real, float and blob. + Valid column types are text, integer, real, float, blob and any. Options: --pk TEXT Column to use as primary key @@ -1257,7 +1257,7 @@ See :ref:`cli_add_column`. :: Usage: sqlite-utils add-column [OPTIONS] PATH TABLE COL_NAME - [integer|int|float|real|text|str|blob|bytes] + [integer|int|float|real|text|str|blob|bytes|any] Add a column to the specified table diff --git a/docs/cli.rst b/docs/cli.rst index cf241aa..417911a 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1390,7 +1390,14 @@ Use ``--type column-name type`` to override the type automatically chosen when t This is useful for values such as ZIP codes, which may look like integers but should be stored as ``TEXT`` to preserve leading zeros. -The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL`` or ``BLOB``. Column types are matched case-insensitively. +The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL``, ``BLOB`` or ``ANY``. Column types are matched case-insensitively. + +``ANY`` is especially useful with ``--strict``. An ``ANY`` column in a strict table preserves values without coercion, so text such as ``000123`` remains text instead of being converted to an integer: + +.. code-block:: bash + + sqlite-utils insert events.db events events.csv --csv --strict \ + --type payload any As with detected column types, ``--type`` only affects tables created by the command. If the table already exists, its existing column types are left unchanged. @@ -2141,6 +2148,12 @@ You can create a table in `SQLite STRICT mode ` @@ -1569,7 +1582,7 @@ You can specify the ``col_type`` argument either using a SQLite type as a string The ``col_type`` is optional - if you omit it the type of ``TEXT`` will be used. -SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"``, ``"REAL"`` or ``"BLOB"``. +SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"``, ``"REAL"``, ``"BLOB"`` or ``"ANY"``. You can use the ``sqlite_utils.ANY`` marker instead of the ``"ANY"`` string. If you pass a Python type, it will be mapped to SQLite types as shown here:: @@ -1582,6 +1595,7 @@ If you pass a Python type, it will be mapped to SQLite types as shown here:: datetime.date: "TEXT" datetime.time: "TEXT" datetime.timedelta: "TEXT" + sqlite_utils.ANY: "ANY" # If numpy is installed np.int8: "INTEGER" @@ -1831,6 +1845,8 @@ Pass ``strict=False`` to convert a strict table back to a regular non-strict tab table.transform(strict=False) +If the table has ``ANY`` columns, converting it to non-strict mode can coerce text values that look numeric. For example, SQLite converts ``"000123"`` to the integer ``123`` when copying it into an ordinary ``ANY`` column. This is SQLite's documented distinction between `STRICT and ordinary ANY columns `__. + The default is ``strict=None``, which preserves the table's existing strict mode. Passing ``strict=True`` raises ``sqlite_utils.db.TransformError`` if the available SQLite version does not support strict tables. @@ -2458,6 +2474,11 @@ The ``.columns_dict`` property returns a dictionary version of the columns with >>> db.table("PlantType").columns_dict {'id': , 'value': } +SQLite ``ANY`` columns are represented by the ``sqlite_utils.ANY`` marker type:: + + >>> db.table("events").columns_dict + {'id': , 'payload': } + .. _python_api_introspection_default_values: .default_values diff --git a/sqlite_utils/__init__.py b/sqlite_utils/__init__.py index 0d25716..3f350e1 100644 --- a/sqlite_utils/__init__.py +++ b/sqlite_utils/__init__.py @@ -1,6 +1,13 @@ from .db import Database from .hookspecs import hookimpl, hookspec from .migrations import Migrations -from .utils import suggest_column_types +from .utils import ANY, suggest_column_types -__all__ = ["Database", "Migrations", "hookimpl", "hookspec", "suggest_column_types"] +__all__ = [ + "ANY", + "Database", + "Migrations", + "hookimpl", + "hookspec", + "suggest_column_types", +] diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index a8baff8..c230902 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -76,7 +76,7 @@ def _close_databases(ctx): pass -VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "REAL", "BLOB") +VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "REAL", "BLOB", "ANY") UNICODE_ERROR = """ {} @@ -489,7 +489,17 @@ def dump(path, load_extension): @click.argument( "col_type", type=click.Choice( - ["integer", "int", "float", "real", "text", "str", "blob", "bytes"], + [ + "integer", + "int", + "float", + "real", + "text", + "str", + "blob", + "bytes", + "any", + ], case_sensitive=False, ), required=False, @@ -1758,7 +1768,7 @@ def create_table( height real \\ photo blob --pk id - Valid column types are text, integer, real, float and blob. + Valid column types are text, integer, real, float, blob and any. """ db = sqlite_utils.Database(path) _register_db_for_cleanup(db) @@ -2668,12 +2678,10 @@ def schema( "--type", type=( str, - click.Choice( - ["INTEGER", "TEXT", "FLOAT", "REAL", "BLOB"], case_sensitive=False - ), + click.Choice(list(VALID_COLUMN_TYPES), case_sensitive=False), ), multiple=True, - help="Change column type to INTEGER, TEXT, FLOAT, REAL or BLOB", + help="Change column type to INTEGER, TEXT, FLOAT, REAL, BLOB or ANY", ) @click.option("--drop", type=str, multiple=True, help="Drop this column") @click.option( diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index edefbab..9c0b402 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -39,6 +39,7 @@ from .create_table_parser import ( sql_ends_in_line_comment, ) from .utils import ( + ANY, OperationalError, chunks, column_affinity, @@ -366,6 +367,7 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = { decimal.Decimal: "REAL", None.__class__: "TEXT", uuid.UUID: "TEXT", + ANY: "ANY", # SQLite explicit types "TEXT": "TEXT", "INTEGER": "INTEGER", @@ -380,6 +382,8 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = { "real": "REAL", "blob": "BLOB", "bytes": "BLOB", + "ANY": "ANY", + "any": "ANY", } # If numpy is available, add more types if np: @@ -3092,6 +3096,15 @@ class Table(Queryable): if col in columns } if lookup_table.exists(): + if ( + self.strict + and ANY in lookup_columns_definition.values() + and not lookup_table.strict + ): + raise InvalidColumns( + f"Lookup table {table} already exists but is not STRICT, " + "so it cannot preserve ANY column values" + ) if not set(lookup_columns_definition.items()).issubset( lookup_table.columns_dict.items() ): @@ -3105,6 +3118,7 @@ class Table(Queryable): **lookup_columns_definition, }, pk="id", + strict=self.strict, ) lookup_columns = [(rename.get(col) or col) for col in columns] lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 06404eb..ee6695b 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -59,6 +59,10 @@ Row = dict[str, RowValue] T = TypeVar("T") +class ANY: + """Marker type for an SQLite ``ANY`` column.""" + + class _CloseableIterator(Iterator[Row]): """Iterator wrapper that closes a file when iteration is complete.""" @@ -178,6 +182,8 @@ def column_affinity(column_type: str) -> type: return bytes if "REAL" in column_type or "FLOA" in column_type or "DOUB" in column_type: return float + if column_type == "ANY": + return ANY # Default is 'NUMERIC', which we currently also treat as float return float diff --git a/tests/test_cli.py b/tests/test_cli.py index f60c7d5..064026a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,7 +9,7 @@ from pathlib import Path import pytest from click.testing import CliRunner -from sqlite_utils import Database, cli +from sqlite_utils import ANY, Database, cli from sqlite_utils.db import ForeignKey, Index @@ -355,6 +355,7 @@ def test_create_index_desc(db_path): ("blob", "BLOB", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), ("blob", "bytes", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), ("blob", "BYTES", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), + ("anything", "any", 'CREATE TABLE "dogs" (\n "name" TEXT\n, "anything" ANY)'), ("default", None, 'CREATE TABLE "dogs" (\n "name" TEXT\n, "default" TEXT)'), ), ) @@ -2007,6 +2008,25 @@ def test_transform_strict_option_with_invalid_data(db_path): assert not any(name.startswith("dogs_new_") for name in db.table_names()) +def test_transform_column_to_any(db_path): + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db.table("items").create({"data": str}, strict=True) + db.table("items").insert({"data": "000123"}) + + result = CliRunner().invoke( + cli.cli, ["transform", db_path, "items", "--type", "data", "any"] + ) + + assert result.exit_code == 0, result.output + assert db.table("items").columns_dict == {"data": ANY} + assert db.execute("select typeof(data), data from items").fetchone() == ( + "text", + "000123", + ) + + @pytest.mark.parametrize( "extra_args,expected_schema", ( @@ -2872,6 +2892,30 @@ def test_create_table_strict(strict): assert db.table("items").columns_dict == {"id": int, "w": float} +def test_create_table_strict_any(): + runner = CliRunner() + with runner.isolated_filesystem(): + db = Database("test.db") + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + result = runner.invoke( + cli.cli, + [ + "create-table", + "test.db", + "items", + "id", + "integer", + "data", + "any", + "--strict", + ], + ) + assert result.exit_code == 0, result.output + assert db.table("items").strict is True + assert db.table("items").columns_dict == {"id": int, "data": ANY} + + @pytest.mark.parametrize("method", ("insert", "upsert")) @pytest.mark.parametrize("strict", (False, True)) def test_insert_upsert_strict(tmpdir, method, strict): @@ -2887,6 +2931,39 @@ def test_insert_upsert_strict(tmpdir, method, strict): assert db.table("items").strict == strict or not db.supports_strict +@pytest.mark.parametrize("method", ("insert", "upsert")) +def test_insert_upsert_strict_any(tmpdir, method): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + if not db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + db.close() + result = CliRunner().invoke( + cli.cli, + [ + method, + db_path, + "items", + "-", + "--csv", + "--pk", + "id", + "--type", + "data", + "any", + "--strict", + ], + input="id,data\n1,000123", + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert db.table("items").columns_dict == {"id": int, "data": ANY} + assert db.execute("select typeof(data), data from items").fetchone() == ( + "text", + "000123", + ) + + def test_extract_bad_column_clean_error(db_path): db = Database(db_path) db.table("trees").insert({"id": 1, "species": "Palm"}, pk="id") diff --git a/tests/test_column_affinity.py b/tests/test_column_affinity.py index 8c619e1..2d7846e 100644 --- a/tests/test_column_affinity.py +++ b/tests/test_column_affinity.py @@ -1,5 +1,6 @@ import pytest +from sqlite_utils import ANY from sqlite_utils.utils import column_affinity EXAMPLES = [ @@ -26,6 +27,8 @@ EXAMPLES = [ ("DOUBLE", float), ("DOUBLE PRECISION", float), ("FLOAT", float), + ("ANY", ANY), + ("any", ANY), # Numeric, treated as float: ("NUMERIC", float), ("DECIMAL(10,5)", float), diff --git a/tests/test_create.py b/tests/test_create.py index e900aee..b738df9 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -7,6 +7,7 @@ import uuid import pytest +from sqlite_utils import ANY from sqlite_utils.db import ( AlterError, Database, @@ -1366,6 +1367,18 @@ def test_quote(fresh_db, input, expected): {"col": list}, '"col" TEXT', ), + ( + {"col": ANY}, + '"col" ANY', + ), + ( + {"col": "ANY"}, + '"col" ANY', + ), + ( + {"col": "any"}, + '"col" ANY', + ), ), ) def test_create_table_sql(fresh_db, columns, expected_sql_middle): @@ -1589,6 +1602,33 @@ def test_create_strict(fresh_db, strict): assert table.strict == strict or not fresh_db.supports_strict +def test_create_strict_with_any(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + table = fresh_db.table("items").create( + {"id": int, "data": ANY}, pk="id", strict=True + ) + table.insert_all( + [ + {"id": 1, "data": 42}, + {"id": 2, "data": "000123"}, + {"id": 3, "data": 3.14}, + {"id": 4, "data": b"bytes"}, + {"id": 5, "data": None}, + ] + ) + assert table.columns_dict == {"id": int, "data": ANY} + assert fresh_db.execute( + "select typeof(data), data from items order by id" + ).fetchall() == [ + ("integer", 42), + ("text", "000123"), + ("real", 3.14), + ("blob", b"bytes"), + ("null", None), + ] + + def test_bad_table_and_view_exceptions(fresh_db): fresh_db.table("t").insert({"id": 1}, pk="id") fresh_db.create_view("v", "select * from t") diff --git a/tests/test_extract.py b/tests/test_extract.py index 72579c4..f855041 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -2,6 +2,7 @@ import itertools import pytest +from sqlite_utils import ANY from sqlite_utils.db import InvalidColumns @@ -305,3 +306,38 @@ def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db): fresh_db.table("t1").extract(["species"], table="lk") fresh_db.table("t2").extract(["species"], table="lk") assert fresh_db.table("lk").count == 1 + + +def test_extract_preserves_strict_any(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (id integer primary key, data any) strict") + fresh_db.execute("insert into items values (1, ?)", ("000123",)) + + fresh_db["items"].extract("data", table="data_values") + + lookup = fresh_db["data_values"] + assert lookup.strict is True + assert lookup.columns_dict == {"id": int, "data": ANY} + assert fresh_db.execute( + "select typeof(data), data from data_values" + ).fetchone() == ("text", "000123") + + +def test_extract_strict_any_rejects_non_strict_lookup(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (data any) strict") + fresh_db.execute("insert into items values (?)", ("000123",)) + fresh_db.execute("create table data_values (id integer primary key, data any)") + + with pytest.raises( + InvalidColumns, + match="is not STRICT, so it cannot preserve ANY column values", + ): + fresh_db["items"].extract("data", table="data_values") + + assert fresh_db.execute("select typeof(data), data from items").fetchone() == ( + "text", + "000123", + ) diff --git a/tests/test_transform.py b/tests/test_transform.py index 5793f10..6a8a143 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -2,6 +2,7 @@ import sqlite3 import pytest +from sqlite_utils import ANY from sqlite_utils.db import Check, ForeignKey, TransactionError, TransformError from sqlite_utils.utils import OperationalError @@ -823,6 +824,55 @@ def test_transform_to_strict_not_supported(fresh_db, method_name): assert table.strict is False +def test_transform_preserves_any_column_in_strict_table(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (id integer primary key, data any) strict") + fresh_db.conn.executemany( + "insert into items values (?, ?)", + [ + (1, 42), + (2, "000123"), + (3, 3.14), + (4, b"bytes"), + (5, None), + ], + ) + table = fresh_db["items"] + + table.transform() + + assert table.strict is True + assert table.columns_dict == {"id": int, "data": ANY} + assert fresh_db.execute( + "select typeof(data), data from items order by id" + ).fetchall() == [ + ("integer", 42), + ("text", "000123"), + ("real", 3.14), + ("blob", b"bytes"), + ("null", None), + ] + + +def test_transform_any_column_from_strict_to_non_strict(fresh_db): + if not fresh_db.supports_strict: + pytest.skip("SQLite version does not support strict tables") + fresh_db.execute("create table items (data any) strict") + fresh_db.execute("insert into items values (?)", ("000123",)) + table = fresh_db["items"] + + table.transform(strict=False) + + assert table.strict is False + assert table.columns_dict == {"data": ANY} + # Ordinary non-STRICT ANY columns apply NUMERIC affinity + assert fresh_db.execute("select typeof(data), data from items").fetchone() == ( + "integer", + 123, + ) + + @pytest.mark.parametrize( "indexes, transform_params", [