Support ANY column types for strict tables

Closes #790, #820

sqlite_utils/__init__.py

@@ -1,6 +1,13 @@
 from .db import Database
 from .hookspecs import hookimpl, hookspec
 from .migrations import Migrations
-from .utils import suggest_column_types
+from .utils import ANY, suggest_column_types
 
-__all__ = ["Database", "Migrations", "hookimpl", "hookspec", "suggest_column_types"]
+__all__ = [
+    "ANY",
+    "Database",
+    "Migrations",
+    "hookimpl",
+    "hookspec",
+    "suggest_column_types",
+]

sqlite_utils/cli.py

@@ -76,7 +76,7 @@ def _close_databases(ctx):
             pass
 
 
-VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "REAL", "BLOB")
+VALID_COLUMN_TYPES = ("INTEGER", "TEXT", "FLOAT", "REAL", "BLOB", "ANY")
 
 UNICODE_ERROR = """
 {}
@@ -489,7 +489,17 @@ def dump(path, load_extension):
 @click.argument(
     "col_type",
     type=click.Choice(
-        ["integer", "int", "float", "real", "text", "str", "blob", "bytes"],
+        [
+            "integer",
+            "int",
+            "float",
+            "real",
+            "text",
+            "str",
+            "blob",
+            "bytes",
+            "any",
+        ],
         case_sensitive=False,
     ),
     required=False,
@@ -1758,7 +1768,7 @@ def create_table(
             height real \\
             photo blob --pk id
 
-    Valid column types are text, integer, real, float and blob.
+    Valid column types are text, integer, real, float, blob and any.
     """
     db = sqlite_utils.Database(path)
     _register_db_for_cleanup(db)
@@ -2668,12 +2678,10 @@ def schema(
     "--type",
     type=(
         str,
-        click.Choice(
-            ["INTEGER", "TEXT", "FLOAT", "REAL", "BLOB"], case_sensitive=False
-        ),
+        click.Choice(list(VALID_COLUMN_TYPES), case_sensitive=False),
     ),
     multiple=True,
-    help="Change column type to INTEGER, TEXT, FLOAT, REAL or BLOB",
+    help="Change column type to INTEGER, TEXT, FLOAT, REAL, BLOB or ANY",
 )
 @click.option("--drop", type=str, multiple=True, help="Drop this column")
 @click.option(

sqlite_utils/db.py

@@ -39,6 +39,7 @@ from .create_table_parser import (
     sql_ends_in_line_comment,
 )
 from .utils import (
+    ANY,
     OperationalError,
     chunks,
     column_affinity,
@@ -366,6 +367,7 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = {
     decimal.Decimal: "REAL",
     None.__class__: "TEXT",
     uuid.UUID: "TEXT",
+    ANY: "ANY",
     # SQLite explicit types
     "TEXT": "TEXT",
     "INTEGER": "INTEGER",
@@ -380,6 +382,8 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = {
     "real": "REAL",
     "blob": "BLOB",
     "bytes": "BLOB",
+    "ANY": "ANY",
+    "any": "ANY",
 }
 # If numpy is available, add more types
 if np:
@@ -3092,6 +3096,15 @@ class Table(Queryable):
                 if col in columns
             }
             if lookup_table.exists():
+                if (
+                    self.strict
+                    and ANY in lookup_columns_definition.values()
+                    and not lookup_table.strict
+                ):
+                    raise InvalidColumns(
+                        f"Lookup table {table} already exists but is not STRICT, "
+                        "so it cannot preserve ANY column values"
+                    )
                 if not set(lookup_columns_definition.items()).issubset(
                     lookup_table.columns_dict.items()
                 ):
@@ -3105,6 +3118,7 @@ class Table(Queryable):
                         **lookup_columns_definition,
                     },
                     pk="id",
+                    strict=self.strict,
                 )
             lookup_columns = [(rename.get(col) or col) for col in columns]
             lookup_table.create_index(lookup_columns, unique=True, if_not_exists=True)

sqlite_utils/utils.py

@@ -59,6 +59,10 @@ Row = dict[str, RowValue]
 T = TypeVar("T")
 
 
+class ANY:
+    """Marker type for an SQLite ``ANY`` column."""
+
+
 class _CloseableIterator(Iterator[Row]):
     """Iterator wrapper that closes a file when iteration is complete."""
 
@@ -178,6 +182,8 @@ def column_affinity(column_type: str) -> type:
         return bytes
     if "REAL" in column_type or "FLOA" in column_type or "DOUB" in column_type:
         return float
+    if column_type == "ANY":
+        return ANY
     # Default is 'NUMERIC', which we currently also treat as float
     return float