test_extract_works_with_null_values, refs #423, #455

Refs #468

docs/cli-reference.rst

@@ -865,6 +865,7 @@ See :ref:`cli_create_table`.
                                 foreign key
       --ignore                  If table already exists, do nothing
       --replace                 If table already exists, replace it
+      --transform               If table already exists, try to transform the schema
       --load-extension TEXT     Path to SQLite extension, with optional :entrypoint
       -h, --help                Show this message and exit.
 

docs/cli.rst

@@ -1520,6 +1520,8 @@ You can specify foreign key relationships between the tables you are creating us
 
 If a table with the same name already exists, you will get an error. You can choose to silently ignore this error with ``--ignore``, or you can replace the existing table with a new, empty table using ``--replace``.
 
+You can also pass ``--transform`` to transform the existing table to match the new schema. See :ref:`python_api_explicit_create` in the Python library documentation for details of how this option works.
+
 .. _cli_duplicate_table:
 
 Duplicating tables

docs/python-api.rst

@@ -554,6 +554,25 @@ To do nothing if the table already exists, add ``if_not_exists=True``:
         "name": str,
     }, pk="id", if_not_exists=True)
 
+You can also pass ``transform=True`` to have any existing tables :ref:`transformed <python_api_transform>` to match your new table specification. This is a **dangerous operation** as it will drop columns that are no longer listed in your call to ``.create()``, so be careful when running this.
+
+.. code-block:: python
+
+    db["cats"].create({
+        "id": int,
+        "name": str,
+        "weight": float,
+    }, pk="id", transform=True)
+
+The ``transform=True`` option will update the table schema if any of the following have changed:
+
+- The specified columns or their types
+- The specified primary key
+- The order of the columns, defined using ``column_order=``
+- The ``not_null=`` or ``defaults=`` arguments
+
+Changes to ``foreign_keys=`` are not currently detected and applied by ``transform=True``.
+
 .. _python_api_compound_primary_keys:
 
 Compound primary keys
@@ -1802,6 +1821,16 @@ The ``.columns_dict`` property returns a dictionary version of the columns with
     >>> db["PlantType"].columns_dict
     {'id': <class 'int'>, 'value': <class 'str'>}
 
+.. _python_api_introspection_default_values:
+
+.default_values
+---------------
+
+The ``.default_values`` property returns a dictionary of default values for each column that has a default::
+
+    >>> db["table_with_defaults"].default_values
+    {'score': 5}
+
 .. _python_api_introspection_pks:
 
 .pks

sqlite_utils/cli.py

@@ -1453,9 +1453,24 @@ def create_database(path, enable_wal, init_spatialite, load_extension):
     is_flag=True,
     help="If table already exists, replace it",
 )
+@click.option(
+    "--transform",
+    is_flag=True,
+    help="If table already exists, try to transform the schema",
+)
 @load_extension_option
 def create_table(
-    path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension
+    path,
+    table,
+    columns,
+    pk,
+    not_null,
+    default,
+    fk,
+    ignore,
+    replace,
+    transform,
+    load_extension,
 ):
     """
     Add a table with the specified columns. Columns should be specified using
@@ -1490,6 +1505,8 @@ def create_table(
             return
         elif replace:
             db[table].drop()
+        elif transform:
+            pass
         else:
             raise click.ClickException(
                 'Table "{}" already exists. Use --replace to delete and replace it.'.format(
@@ -1497,7 +1514,12 @@ def create_table(
                 )
             )
     db[table].create(
-        coltypes, pk=pk, not_null=not_null, defaults=dict(default), foreign_keys=fk
+        coltypes,
+        pk=pk,
+        not_null=not_null,
+        defaults=dict(default),
+        foreign_keys=fk,
+        transform=transform,
     )
 
 

sqlite_utils/db.py

@@ -9,6 +9,7 @@ from .utils import (
     progressbar,
     find_spatialite,
 )
+import binascii
 from collections import namedtuple
 from collections.abc import Mapping
 import contextlib
@@ -33,7 +34,6 @@ from typing import (
     Union,
     Optional,
     List,
-    Set,
     Tuple,
 )
 import uuid
@@ -875,6 +875,7 @@ class Database:
         hash_id_columns: Optional[Iterable[str]] = None,
         extracts: Optional[Union[Dict[str, str], List[str]]] = None,
         if_not_exists: bool = False,
+        transform: bool = False,
     ) -> "Table":
         """
         Create a table with the specified name and the specified ``{column_name: type}`` columns.
@@ -892,7 +893,61 @@ class Database:
         :param hash_id_columns: List of columns to be used when calculating the hash ID for a row
         :param extracts: List or dictionary of columns to be extracted during inserts, see :ref:`python_api_extracts`
         :param if_not_exists: Use ``CREATE TABLE IF NOT EXISTS``
+        :param transform: If table already exists transform it to fit the specified schema
         """
+        # Transform table to match the new definition if table already exists:
+        if transform and self[name].exists():
+            table = cast(Table, self[name])
+            should_transform = False
+            # First add missing columns and figure out columns to drop
+            existing_columns = table.columns_dict
+            missing_columns = dict(
+                (col_name, col_type)
+                for col_name, col_type in columns.items()
+                if col_name not in existing_columns
+            )
+            columns_to_drop = [
+                column for column in existing_columns if column not in columns
+            ]
+            if missing_columns:
+                for col_name, col_type in missing_columns.items():
+                    table.add_column(col_name, col_type)
+            if missing_columns or columns_to_drop or columns != existing_columns:
+                should_transform = True
+            # Do we need to change the column order?
+            if (
+                column_order
+                and list(existing_columns)[: len(column_order)] != column_order
+            ):
+                should_transform = True
+            # Has the primary key changed?
+            current_pks = table.pks
+            desired_pk = None
+            if isinstance(pk, str):
+                desired_pk = [pk]
+            elif pk:
+                desired_pk = list(pk)
+            if desired_pk and current_pks != desired_pk:
+                should_transform = True
+            # Any not-null changes?
+            current_not_null = {c.name for c in table.columns if c.notnull}
+            desired_not_null = set(not_null) if not_null else set()
+            if current_not_null != desired_not_null:
+                should_transform = True
+            # How about defaults?
+            if defaults and defaults != table.default_values:
+                should_transform = True
+            # Only run .transform() if there is something to do
+            if should_transform:
+                table.transform(
+                    types=columns,
+                    drop=columns_to_drop,
+                    column_order=column_order,
+                    not_null=not_null,
+                    defaults=defaults,
+                    pk=pk,
+                )
+            return table
         sql = self.create_table_sql(
             name=name,
             columns=columns,
@@ -907,7 +962,7 @@ class Database:
             if_not_exists=if_not_exists,
         )
         self.execute(sql)
-        table = self.table(
+        created_table = self.table(
             name,
             pk=pk,
             foreign_keys=foreign_keys,
@@ -917,7 +972,7 @@ class Database:
             hash_id=hash_id,
             hash_id_columns=hash_id_columns,
         )
-        return cast(Table, table)
+        return cast(Table, created_table)
 
     def create_view(
         self, name: str, sql: str, ignore: bool = False, replace: bool = False
@@ -1458,6 +1513,15 @@ class Table(Queryable):
         "``{trigger_name: sql}`` dictionary of triggers defined on this table."
         return {trigger.name: trigger.sql for trigger in self.triggers}
 
+    @property
+    def default_values(self) -> Dict[str, Any]:
+        "``{column_name: default_value}`` dictionary of default values for columns in this table."
+        return {
+            column.name: _decode_default_value(column.default_value)
+            for column in self.columns
+            if column.default_value is not None
+        }
+
     @property
     def strict(self) -> bool:
         "Is this a STRICT table?"
@@ -1477,6 +1541,7 @@ class Table(Queryable):
         hash_id_columns: Optional[Iterable[str]] = None,
         extracts: Optional[Union[Dict[str, str], List[str]]] = None,
         if_not_exists: bool = False,
+        transform: bool = False,
     ) -> "Table":
         """
         Create a table with the specified columns.
@@ -1508,6 +1573,7 @@ class Table(Queryable):
                 hash_id_columns=hash_id_columns,
                 extracts=extracts,
                 if_not_exists=if_not_exists,
+                transform=transform,
             )
         return self
 
@@ -1534,7 +1600,7 @@ class Table(Queryable):
         rename: Optional[dict] = None,
         drop: Optional[Iterable] = None,
         pk: Optional[Any] = DEFAULT,
-        not_null: Optional[Set[str]] = None,
+        not_null: Optional[Iterable[str]] = None,
         defaults: Optional[Dict[str, Any]] = None,
         drop_foreign_keys: Optional[Iterable] = None,
         column_order: Optional[List[str]] = None,
@@ -1654,10 +1720,12 @@ class Table(Queryable):
                     create_table_not_null.add(key)
         elif isinstance(not_null, set):
             create_table_not_null.update((rename.get(k) or k) for k in not_null)
-        elif not_null is None:
+        elif not not_null:
             pass
         else:
-            assert False, "not_null must be a dict or a set or None"
+            assert False, "not_null must be a dict or a set or None, it was {}".format(
+                repr(not_null)
+            )
         # defaults=
         create_table_defaults = {
             (rename.get(c.name) or c.name): c.default_value
@@ -2851,7 +2919,7 @@ class Table(Queryable):
         pk=DEFAULT,
         foreign_keys=DEFAULT,
         column_order: Optional[Union[List[str], Default]] = DEFAULT,
-        not_null: Optional[Union[Set[str], Default]] = DEFAULT,
+        not_null: Optional[Union[Iterable[str], Default]] = DEFAULT,
         defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT,
         hash_id: Optional[Union[str, Default]] = DEFAULT,
         hash_id_columns: Optional[Union[Iterable[str], Default]] = DEFAULT,
@@ -3131,7 +3199,7 @@ class Table(Queryable):
         pk: Optional[str] = "id",
         foreign_keys: Optional[ForeignKeysType] = None,
         column_order: Optional[List[str]] = None,
-        not_null: Optional[Set[str]] = None,
+        not_null: Optional[Iterable[str]] = None,
         defaults: Optional[Dict[str, Any]] = None,
         extracts: Optional[Union[Dict[str, str], List[str]]] = None,
         conversions: Optional[Dict[str, str]] = None,
@@ -3527,3 +3595,22 @@ def fix_square_braces(records: Iterable[Dict[str, Any]]):
             }
         else:
             yield record
+
+
+def _decode_default_value(value):
+    if value.startswith("'") and value.endswith("'"):
+        # It's a string
+        return value[1:-1]
+    if value.isdigit():
+        # It's an integer
+        return int(value)
+    if value.startswith("X'") and value.endswith("'"):
+        # It's a binary string, stored as hex
+        to_decode = value[2:-1]
+        return binascii.unhexlify(to_decode)
+    # If it is a string containing a floating point number:
+    try:
+        return float(value)
+    except ValueError:
+        pass
+    return value

tests/test_create.py

@@ -1155,3 +1155,79 @@ def test_create_if_no_columns(fresh_db):
     with pytest.raises(AssertionError) as error:
         fresh_db["t"].create({})
     assert error.value.args[0] == "Tables must have at least one column"
+
+
+@pytest.mark.parametrize(
+    "cols,kwargs,expected_schema,should_transform",
+    (
+        # Change nothing
+        (
+            {"id": int, "name": str},
+            {"pk": "id"},
+            "CREATE TABLE [demo] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT\n)",
+            False,
+        ),
+        # Drop name column, remove primary key
+        ({"id": int}, {}, 'CREATE TABLE "demo" (\n   [id] INTEGER\n)', True),
+        # Add a new column
+        (
+            {"id": int, "name": str, "age": int},
+            {"pk": "id"},
+            'CREATE TABLE "demo" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT,\n   [age] INTEGER\n)',
+            True,
+        ),
+        # Change a column type
+        (
+            {"id": int, "name": bytes},
+            {"pk": "id"},
+            'CREATE TABLE "demo" (\n   [id] INTEGER PRIMARY KEY,\n   [name] BLOB\n)',
+            True,
+        ),
+        # Change the primary key
+        (
+            {"id": int, "name": str},
+            {"pk": "name"},
+            'CREATE TABLE "demo" (\n   [id] INTEGER,\n   [name] TEXT PRIMARY KEY\n)',
+            True,
+        ),
+        # Change in column order
+        (
+            {"id": int, "name": str},
+            {"pk": "id", "column_order": ["name"]},
+            'CREATE TABLE "demo" (\n   [name] TEXT,\n   [id] INTEGER PRIMARY KEY\n)',
+            True,
+        ),
+        # Same column order is ignored
+        (
+            {"id": int, "name": str},
+            {"pk": "id", "column_order": ["id", "name"]},
+            "CREATE TABLE [demo] (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT\n)",
+            False,
+        ),
+        # Change not null
+        (
+            {"id": int, "name": str},
+            {"pk": "id", "not_null": {"name"}},
+            'CREATE TABLE "demo" (\n   [id] INTEGER PRIMARY KEY,\n   [name] TEXT NOT NULL\n)',
+            True,
+        ),
+        # Change default values
+        (
+            {"id": int, "name": str},
+            {"pk": "id", "defaults": {"id": 0, "name": "Bob"}},
+            "CREATE TABLE \"demo\" (\n   [id] INTEGER PRIMARY KEY DEFAULT 0,\n   [name] TEXT DEFAULT 'Bob'\n)",
+            True,
+        ),
+    ),
+)
+def test_create_transform(fresh_db, cols, kwargs, expected_schema, should_transform):
+    fresh_db.create_table("demo", {"id": int, "name": str}, pk="id")
+    fresh_db["demo"].insert({"id": 1, "name": "Cleo"})
+    traces = []
+    with fresh_db.tracer(lambda sql, parameters: traces.append((sql, parameters))):
+        fresh_db["demo"].create(cols, **kwargs, transform=True)
+    at_least_one_create_table = any(sql.startswith("CREATE TABLE") for sql, _ in traces)
+    assert should_transform == at_least_one_create_table
+    new_schema = fresh_db["demo"].schema
+    assert new_schema == expected_schema, repr(new_schema)
+    assert fresh_db["demo"].count == 1

tests/test_extract.py

@@ -186,3 +186,24 @@ def test_extract_error_on_incompatible_existing_lookup_table(fresh_db):
     fresh_db["species2"].insert({"id": 1, "common_name": 3.5})
     with pytest.raises(InvalidColumns):
         fresh_db["tree"].extract("common_name", table="species2")
+
+
+def test_extract_works_with_null_values(fresh_db):
+    fresh_db["listens"].insert_all(
+        [
+            {"id": 1, "track_title": "foo", "album_title": "bar"},
+            {"id": 2, "track_title": "baz", "album_title": None},
+        ],
+        pk="id",
+    )
+    fresh_db["listens"].extract(
+        columns=["album_title"], table="albums", fk_column="album_id"
+    )
+    assert list(fresh_db["listens"].rows) == [
+        {"id": 1, "track_title": "foo", "album_id": 1},
+        {"id": 2, "track_title": "baz", "album_id": 2},
+    ]
+    assert list(fresh_db["albums"].rows) == [
+        {"id": 1, "album_title": "bar"},
+        {"id": 2, "album_title": None},
+    ]

tests/test_introspect.py

@@ -299,3 +299,21 @@ def test_table_strict(fresh_db, create_table, expected_strict):
     fresh_db.execute(create_table)
     table = fresh_db["t"]
     assert table.strict == expected_strict
+
+
+@pytest.mark.parametrize(
+    "value",
+    (
+        1,
+        1.3,
+        "foo",
+        True,
+        b"binary",
+    ),
+)
+def test_table_default_values(fresh_db, value):
+    fresh_db["default_values"].insert(
+        {"nodefault": 1, "value": value}, defaults={"value": value}
+    )
+    default_values = fresh_db["default_values"].default_values
+    assert default_values == {"value": value}