diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index d3951fd..d74855e 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -865,6 +865,7 @@ See :ref:`cli_create_table`. foreign key --ignore If table already exists, do nothing --replace If table already exists, replace it + --transform If table already exists, try to transform the schema --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. diff --git a/docs/cli.rst b/docs/cli.rst index 3c5c595..d071158 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1520,6 +1520,8 @@ You can specify foreign key relationships between the tables you are creating us If a table with the same name already exists, you will get an error. You can choose to silently ignore this error with ``--ignore``, or you can replace the existing table with a new, empty table using ``--replace``. +You can also pass ``--transform`` to transform the existing table to match the new schema. See :ref:`python_api_explicit_create` in the Python library documentation for details of how this option works. + .. _cli_duplicate_table: Duplicating tables diff --git a/docs/python-api.rst b/docs/python-api.rst index c68611c..b8794f2 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -554,6 +554,25 @@ To do nothing if the table already exists, add ``if_not_exists=True``: "name": str, }, pk="id", if_not_exists=True) +You can also pass ``transform=True`` to have any existing tables :ref:`transformed ` to match your new table specification. This is a **dangerous operation** as it may drop columns that are no longer listed in your call to ``.create()``, so be careful when running this. + +.. code-block:: python + + db["cats"].create({ + "id": int, + "name": str, + "weight": float, + }, pk="id", transform=True) + +The ``transform=True`` option will update the table schema if any of the following have changed: + +- The specified columns or their types +- The specified primary key +- The order of the columns, defined using ``column_order=`` +- The ``not_null=`` or ``defaults=`` arguments + +Changes to ``foreign_keys=`` are not currently detected and applied by ``transform=True``. + .. _python_api_compound_primary_keys: Compound primary keys diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c09273d..c51b101 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1453,9 +1453,24 @@ def create_database(path, enable_wal, init_spatialite, load_extension): is_flag=True, help="If table already exists, replace it", ) +@click.option( + "--transform", + is_flag=True, + help="If table already exists, try to transform the schema", +) @load_extension_option def create_table( - path, table, columns, pk, not_null, default, fk, ignore, replace, load_extension + path, + table, + columns, + pk, + not_null, + default, + fk, + ignore, + replace, + transform, + load_extension, ): """ Add a table with the specified columns. Columns should be specified using @@ -1490,6 +1505,8 @@ def create_table( return elif replace: db[table].drop() + elif transform: + pass else: raise click.ClickException( 'Table "{}" already exists. Use --replace to delete and replace it.'.format( @@ -1497,7 +1514,12 @@ def create_table( ) ) db[table].create( - coltypes, pk=pk, not_null=not_null, defaults=dict(default), foreign_keys=fk + coltypes, + pk=pk, + not_null=not_null, + defaults=dict(default), + foreign_keys=fk, + transform=transform, ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 3f9d400..b653cc4 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -34,7 +34,6 @@ from typing import ( Union, Optional, List, - Set, Tuple, ) import uuid @@ -876,6 +875,7 @@ class Database: hash_id_columns: Optional[Iterable[str]] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, if_not_exists: bool = False, + transform: bool = False, ) -> "Table": """ Create a table with the specified name and the specified ``{column_name: type}`` columns. @@ -893,7 +893,61 @@ class Database: :param hash_id_columns: List of columns to be used when calculating the hash ID for a row :param extracts: List or dictionary of columns to be extracted during inserts, see :ref:`python_api_extracts` :param if_not_exists: Use ``CREATE TABLE IF NOT EXISTS`` + :param transform: If table already exists transform it to fit the specified schema """ + # Transform table to match the new definition if table already exists: + if transform and self[name].exists(): + table = cast(Table, self[name]) + should_transform = False + # First add missing columns and figure out columns to drop + existing_columns = table.columns_dict + missing_columns = dict( + (col_name, col_type) + for col_name, col_type in columns.items() + if col_name not in existing_columns + ) + columns_to_drop = [ + column for column in existing_columns if column not in columns + ] + if missing_columns: + for col_name, col_type in missing_columns.items(): + table.add_column(col_name, col_type) + if missing_columns or columns_to_drop or columns != existing_columns: + should_transform = True + # Do we need to change the column order? + if ( + column_order + and list(existing_columns)[: len(column_order)] != column_order + ): + should_transform = True + # Has the primary key changed? + current_pks = table.pks + desired_pk = None + if isinstance(pk, str): + desired_pk = [pk] + elif pk: + desired_pk = list(pk) + if desired_pk and current_pks != desired_pk: + should_transform = True + # Any not-null changes? + current_not_null = {c.name for c in table.columns if c.notnull} + desired_not_null = set(not_null) if not_null else set() + if current_not_null != desired_not_null: + should_transform = True + # How about defaults? + if defaults and defaults != table.default_values: + should_transform = True + # Only run .transform() if there is something to do + if should_transform: + table.transform( + types=columns, + drop=columns_to_drop, + column_order=column_order, + not_null=not_null, + defaults=defaults, + pk=pk, + ) + return table sql = self.create_table_sql( name=name, columns=columns, @@ -908,7 +962,7 @@ class Database: if_not_exists=if_not_exists, ) self.execute(sql) - table = self.table( + created_table = self.table( name, pk=pk, foreign_keys=foreign_keys, @@ -918,7 +972,7 @@ class Database: hash_id=hash_id, hash_id_columns=hash_id_columns, ) - return cast(Table, table) + return cast(Table, created_table) def create_view( self, name: str, sql: str, ignore: bool = False, replace: bool = False @@ -1487,6 +1541,7 @@ class Table(Queryable): hash_id_columns: Optional[Iterable[str]] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, if_not_exists: bool = False, + transform: bool = False, ) -> "Table": """ Create a table with the specified columns. @@ -1518,6 +1573,7 @@ class Table(Queryable): hash_id_columns=hash_id_columns, extracts=extracts, if_not_exists=if_not_exists, + transform=transform, ) return self @@ -1544,7 +1600,7 @@ class Table(Queryable): rename: Optional[dict] = None, drop: Optional[Iterable] = None, pk: Optional[Any] = DEFAULT, - not_null: Optional[Set[str]] = None, + not_null: Optional[Iterable[str]] = None, defaults: Optional[Dict[str, Any]] = None, drop_foreign_keys: Optional[Iterable] = None, column_order: Optional[List[str]] = None, @@ -1664,10 +1720,12 @@ class Table(Queryable): create_table_not_null.add(key) elif isinstance(not_null, set): create_table_not_null.update((rename.get(k) or k) for k in not_null) - elif not_null is None: + elif not not_null: pass else: - assert False, "not_null must be a dict or a set or None" + assert False, "not_null must be a dict or a set or None, it was {}".format( + repr(not_null) + ) # defaults= create_table_defaults = { (rename.get(c.name) or c.name): c.default_value @@ -2861,7 +2919,7 @@ class Table(Queryable): pk=DEFAULT, foreign_keys=DEFAULT, column_order: Optional[Union[List[str], Default]] = DEFAULT, - not_null: Optional[Union[Set[str], Default]] = DEFAULT, + not_null: Optional[Union[Iterable[str], Default]] = DEFAULT, defaults: Optional[Union[Dict[str, Any], Default]] = DEFAULT, hash_id: Optional[Union[str, Default]] = DEFAULT, hash_id_columns: Optional[Union[Iterable[str], Default]] = DEFAULT, @@ -3141,7 +3199,7 @@ class Table(Queryable): pk: Optional[str] = "id", foreign_keys: Optional[ForeignKeysType] = None, column_order: Optional[List[str]] = None, - not_null: Optional[Set[str]] = None, + not_null: Optional[Iterable[str]] = None, defaults: Optional[Dict[str, Any]] = None, extracts: Optional[Union[Dict[str, str], List[str]]] = None, conversions: Optional[Dict[str, str]] = None, diff --git a/tests/test_create.py b/tests/test_create.py index 60181de..7252e48 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1155,3 +1155,79 @@ def test_create_if_no_columns(fresh_db): with pytest.raises(AssertionError) as error: fresh_db["t"].create({}) assert error.value.args[0] == "Tables must have at least one column" + + +@pytest.mark.parametrize( + "cols,kwargs,expected_schema,should_transform", + ( + # Change nothing + ( + {"id": int, "name": str}, + {"pk": "id"}, + "CREATE TABLE [demo] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)", + False, + ), + # Drop name column, remove primary key + ({"id": int}, {}, 'CREATE TABLE "demo" (\n [id] INTEGER\n)', True), + # Add a new column + ( + {"id": int, "name": str, "age": int}, + {"pk": "id"}, + 'CREATE TABLE "demo" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [age] INTEGER\n)', + True, + ), + # Change a column type + ( + {"id": int, "name": bytes}, + {"pk": "id"}, + 'CREATE TABLE "demo" (\n [id] INTEGER PRIMARY KEY,\n [name] BLOB\n)', + True, + ), + # Change the primary key + ( + {"id": int, "name": str}, + {"pk": "name"}, + 'CREATE TABLE "demo" (\n [id] INTEGER,\n [name] TEXT PRIMARY KEY\n)', + True, + ), + # Change in column order + ( + {"id": int, "name": str}, + {"pk": "id", "column_order": ["name"]}, + 'CREATE TABLE "demo" (\n [name] TEXT,\n [id] INTEGER PRIMARY KEY\n)', + True, + ), + # Same column order is ignored + ( + {"id": int, "name": str}, + {"pk": "id", "column_order": ["id", "name"]}, + "CREATE TABLE [demo] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)", + False, + ), + # Change not null + ( + {"id": int, "name": str}, + {"pk": "id", "not_null": {"name"}}, + 'CREATE TABLE "demo" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT NOT NULL\n)', + True, + ), + # Change default values + ( + {"id": int, "name": str}, + {"pk": "id", "defaults": {"id": 0, "name": "Bob"}}, + "CREATE TABLE \"demo\" (\n [id] INTEGER PRIMARY KEY DEFAULT 0,\n [name] TEXT DEFAULT 'Bob'\n)", + True, + ), + ), +) +def test_create_transform(fresh_db, cols, kwargs, expected_schema, should_transform): + fresh_db.create_table("demo", {"id": int, "name": str}, pk="id") + fresh_db["demo"].insert({"id": 1, "name": "Cleo"}) + traces = [] + with fresh_db.tracer(lambda sql, parameters: traces.append((sql, parameters))): + fresh_db["demo"].create(cols, **kwargs, transform=True) + at_least_one_create_table = any(sql.startswith("CREATE TABLE") for sql, _ in traces) + assert should_transform == at_least_one_create_table + new_schema = fresh_db["demo"].schema + assert new_schema == expected_schema, repr(new_schema) + assert fresh_db["demo"].count == 1