Support ANY column types for strict tables

Closes #790, #820

tests/test_cli.py

@@ -9,7 +9,7 @@ from pathlib import Path
 import pytest
 from click.testing import CliRunner
 
-from sqlite_utils import Database, cli
+from sqlite_utils import ANY, Database, cli
 from sqlite_utils.db import ForeignKey, Index
 
 
@@ -355,6 +355,7 @@ def test_create_index_desc(db_path):
         ("blob", "BLOB", 'CREATE TABLE "dogs" (\n   "name" TEXT\n, "blob" BLOB)'),
         ("blob", "bytes", 'CREATE TABLE "dogs" (\n   "name" TEXT\n, "blob" BLOB)'),
         ("blob", "BYTES", 'CREATE TABLE "dogs" (\n   "name" TEXT\n, "blob" BLOB)'),
+        ("anything", "any", 'CREATE TABLE "dogs" (\n   "name" TEXT\n, "anything" ANY)'),
         ("default", None, 'CREATE TABLE "dogs" (\n   "name" TEXT\n, "default" TEXT)'),
     ),
 )
@@ -2007,6 +2008,25 @@ def test_transform_strict_option_with_invalid_data(db_path):
     assert not any(name.startswith("dogs_new_") for name in db.table_names())
 
 
+def test_transform_column_to_any(db_path):
+    db = Database(db_path)
+    if not db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    db.table("items").create({"data": str}, strict=True)
+    db.table("items").insert({"data": "000123"})
+
+    result = CliRunner().invoke(
+        cli.cli, ["transform", db_path, "items", "--type", "data", "any"]
+    )
+
+    assert result.exit_code == 0, result.output
+    assert db.table("items").columns_dict == {"data": ANY}
+    assert db.execute("select typeof(data), data from items").fetchone() == (
+        "text",
+        "000123",
+    )
+
+
 @pytest.mark.parametrize(
     "extra_args,expected_schema",
     (
@@ -2872,6 +2892,30 @@ def test_create_table_strict(strict):
         assert db.table("items").columns_dict == {"id": int, "w": float}
 
 
+def test_create_table_strict_any():
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        db = Database("test.db")
+        if not db.supports_strict:
+            pytest.skip("SQLite version does not support strict tables")
+        result = runner.invoke(
+            cli.cli,
+            [
+                "create-table",
+                "test.db",
+                "items",
+                "id",
+                "integer",
+                "data",
+                "any",
+                "--strict",
+            ],
+        )
+        assert result.exit_code == 0, result.output
+        assert db.table("items").strict is True
+        assert db.table("items").columns_dict == {"id": int, "data": ANY}
+
+
 @pytest.mark.parametrize("method", ("insert", "upsert"))
 @pytest.mark.parametrize("strict", (False, True))
 def test_insert_upsert_strict(tmpdir, method, strict):
@@ -2887,6 +2931,39 @@ def test_insert_upsert_strict(tmpdir, method, strict):
     assert db.table("items").strict == strict or not db.supports_strict
 
 
+@pytest.mark.parametrize("method", ("insert", "upsert"))
+def test_insert_upsert_strict_any(tmpdir, method):
+    db_path = str(tmpdir / "test.db")
+    db = Database(db_path)
+    if not db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    db.close()
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            method,
+            db_path,
+            "items",
+            "-",
+            "--csv",
+            "--pk",
+            "id",
+            "--type",
+            "data",
+            "any",
+            "--strict",
+        ],
+        input="id,data\n1,000123",
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert db.table("items").columns_dict == {"id": int, "data": ANY}
+    assert db.execute("select typeof(data), data from items").fetchone() == (
+        "text",
+        "000123",
+    )
+
+
 def test_extract_bad_column_clean_error(db_path):
     db = Database(db_path)
     db.table("trees").insert({"id": 1, "species": "Palm"}, pk="id")

tests/test_column_affinity.py

@@ -1,5 +1,6 @@
 import pytest
 
+from sqlite_utils import ANY
 from sqlite_utils.utils import column_affinity
 
 EXAMPLES = [
@@ -26,6 +27,8 @@ EXAMPLES = [
     ("DOUBLE", float),
     ("DOUBLE PRECISION", float),
     ("FLOAT", float),
+    ("ANY", ANY),
+    ("any", ANY),
     # Numeric, treated as float:
     ("NUMERIC", float),
     ("DECIMAL(10,5)", float),

tests/test_create.py

@@ -7,6 +7,7 @@ import uuid
 
 import pytest
 
+from sqlite_utils import ANY
 from sqlite_utils.db import (
     AlterError,
     Database,
@@ -1366,6 +1367,18 @@ def test_quote(fresh_db, input, expected):
             {"col": list},
             '"col" TEXT',
         ),
+        (
+            {"col": ANY},
+            '"col" ANY',
+        ),
+        (
+            {"col": "ANY"},
+            '"col" ANY',
+        ),
+        (
+            {"col": "any"},
+            '"col" ANY',
+        ),
     ),
 )
 def test_create_table_sql(fresh_db, columns, expected_sql_middle):
@@ -1589,6 +1602,33 @@ def test_create_strict(fresh_db, strict):
     assert table.strict == strict or not fresh_db.supports_strict
 
 
+def test_create_strict_with_any(fresh_db):
+    if not fresh_db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    table = fresh_db.table("items").create(
+        {"id": int, "data": ANY}, pk="id", strict=True
+    )
+    table.insert_all(
+        [
+            {"id": 1, "data": 42},
+            {"id": 2, "data": "000123"},
+            {"id": 3, "data": 3.14},
+            {"id": 4, "data": b"bytes"},
+            {"id": 5, "data": None},
+        ]
+    )
+    assert table.columns_dict == {"id": int, "data": ANY}
+    assert fresh_db.execute(
+        "select typeof(data), data from items order by id"
+    ).fetchall() == [
+        ("integer", 42),
+        ("text", "000123"),
+        ("real", 3.14),
+        ("blob", b"bytes"),
+        ("null", None),
+    ]
+
+
 def test_bad_table_and_view_exceptions(fresh_db):
     fresh_db.table("t").insert({"id": 1}, pk="id")
     fresh_db.create_view("v", "select * from t")

tests/test_extract.py

@@ -2,6 +2,7 @@ import itertools
 
 import pytest
 
+from sqlite_utils import ANY
 from sqlite_utils.db import InvalidColumns
 
 
@@ -305,3 +306,38 @@ def test_extract_repeated_into_shared_lookup_no_nulls(fresh_db):
     fresh_db.table("t1").extract(["species"], table="lk")
     fresh_db.table("t2").extract(["species"], table="lk")
     assert fresh_db.table("lk").count == 1
+
+
+def test_extract_preserves_strict_any(fresh_db):
+    if not fresh_db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    fresh_db.execute("create table items (id integer primary key, data any) strict")
+    fresh_db.execute("insert into items values (1, ?)", ("000123",))
+
+    fresh_db["items"].extract("data", table="data_values")
+
+    lookup = fresh_db["data_values"]
+    assert lookup.strict is True
+    assert lookup.columns_dict == {"id": int, "data": ANY}
+    assert fresh_db.execute(
+        "select typeof(data), data from data_values"
+    ).fetchone() == ("text", "000123")
+
+
+def test_extract_strict_any_rejects_non_strict_lookup(fresh_db):
+    if not fresh_db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    fresh_db.execute("create table items (data any) strict")
+    fresh_db.execute("insert into items values (?)", ("000123",))
+    fresh_db.execute("create table data_values (id integer primary key, data any)")
+
+    with pytest.raises(
+        InvalidColumns,
+        match="is not STRICT, so it cannot preserve ANY column values",
+    ):
+        fresh_db["items"].extract("data", table="data_values")
+
+    assert fresh_db.execute("select typeof(data), data from items").fetchone() == (
+        "text",
+        "000123",
+    )

tests/test_transform.py

@@ -2,6 +2,7 @@ import sqlite3
 
 import pytest
 
+from sqlite_utils import ANY
 from sqlite_utils.db import Check, ForeignKey, TransactionError, TransformError
 from sqlite_utils.utils import OperationalError
 
@@ -823,6 +824,55 @@ def test_transform_to_strict_not_supported(fresh_db, method_name):
     assert table.strict is False
 
 
+def test_transform_preserves_any_column_in_strict_table(fresh_db):
+    if not fresh_db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    fresh_db.execute("create table items (id integer primary key, data any) strict")
+    fresh_db.conn.executemany(
+        "insert into items values (?, ?)",
+        [
+            (1, 42),
+            (2, "000123"),
+            (3, 3.14),
+            (4, b"bytes"),
+            (5, None),
+        ],
+    )
+    table = fresh_db["items"]
+
+    table.transform()
+
+    assert table.strict is True
+    assert table.columns_dict == {"id": int, "data": ANY}
+    assert fresh_db.execute(
+        "select typeof(data), data from items order by id"
+    ).fetchall() == [
+        ("integer", 42),
+        ("text", "000123"),
+        ("real", 3.14),
+        ("blob", b"bytes"),
+        ("null", None),
+    ]
+
+
+def test_transform_any_column_from_strict_to_non_strict(fresh_db):
+    if not fresh_db.supports_strict:
+        pytest.skip("SQLite version does not support strict tables")
+    fresh_db.execute("create table items (data any) strict")
+    fresh_db.execute("insert into items values (?)", ("000123",))
+    table = fresh_db["items"]
+
+    table.transform(strict=False)
+
+    assert table.strict is False
+    assert table.columns_dict == {"data": ANY}
+    # Ordinary non-STRICT ANY columns apply NUMERIC affinity
+    assert fresh_db.execute("select typeof(data), data from items").fetchone() == (
+        "integer",
+        123,
+    )
+
+
 @pytest.mark.parametrize(
     "indexes, transform_params",
     [