transform() now works for tables referenced by views (#832)

Closes #831

docs/changelog.rst

@@ -4,6 +4,13 @@
  Changelog
 ===========
 
+.. _unreleased:
+
+Unreleased
+----------
+
+- ``table.transform()`` now works for tables that are referenced by views. Previously the ``ALTER TABLE ... RENAME TO`` step raised ``no such table`` if a view referenced the table being transformed. View definitions are left unchanged - see :ref:`python_api_transform_views`. This also fixes a bug where ``transform(keep_table=...)`` silently rewrote dependent views to point at the frozen backup table instead of the live one. (:issue:`831`)
+
 .. _v3_39_1:
 
 3.39.1 (2026-07-25)
@@ -18,6 +25,7 @@
 
 - ``table.transform()`` now raises a ``TransactionError`` if called while a transaction is open with ``PRAGMA foreign_keys`` enabled and the table is referenced by foreign keys with destructive ``ON DELETE`` actions - ``CASCADE``, ``SET NULL`` or ``SET DEFAULT``. The pragma cannot be changed inside a transaction, so previously dropping the old table as part of the transform could fire those actions and silently delete or modify referencing rows. See :ref:`python_api_transform_foreign_keys_transactions` for details and workarounds. (:issue:`794`)
 - The :ref:`CLI <cli>` and :ref:`Python API <python_api>` documentation now cross-reference each other: CLI sections link to the equivalent Python API functionality and Python API sections link back to the corresponding CLI command. (:issue:`791`)
+
 .. _v4_1:
 
 4.1 (2026-07-11)

docs/cli.rst

@@ -2288,7 +2288,11 @@ If you want to see the SQL that will be executed to make the change without actu
     INSERT INTO "roadside_attractions_new_4033a60276b9" ("longitude", "latitude", "id", "name")
        SELECT "longitude", "latitude", "pk", "name" FROM "roadside_attractions";
     DROP TABLE "roadside_attractions";
+    PRAGMA legacy_alter_table=ON;
     ALTER TABLE "roadside_attractions_new_4033a60276b9" RENAME TO "roadside_attractions";
+    PRAGMA legacy_alter_table=OFF;
+
+Tables that are referenced by views can be transformed - the view definitions are left unchanged, see :ref:`python_api_transform_views` for details.
 
 .. note::
     In Python: :ref:`table.transform() <python_api_transform>`  CLI reference: :ref:`sqlite-utils transform <cli_ref_transform>`

docs/python-api.rst

@@ -1986,6 +1986,17 @@ A bare column name drops any foreign key that column participates in, including
 
 Renaming a column with ``rename=`` updates any foreign keys that use it, and dropping a column with ``drop=`` also drops any foreign keys it participates in - for a compound foreign key this removes the whole constraint.
 
+.. _python_api_transform_views:
+
+Tables referenced by views
+--------------------------
+
+Tables that are referenced by views can be safely transformed - the view definitions are left byte-for-byte unchanged, and views continue to read from the live table even when ``keep_table=`` is used to keep a copy of the original around.
+
+A view that references a column which the transform renamed or dropped will remain defined but will raise a ``no such column`` error when it is next queried. This is inherent to SQLite views, whose SQL is stored as text - if you rename or drop columns that a view depends on you should update that view definition yourself.
+
+To achieve this, the SQL produced by ``transform_sql()`` turns on ``PRAGMA legacy_alter_table`` for its ``ALTER TABLE ... RENAME TO`` statements, then restores the pragma to the value it had when the SQL was generated - without this, SQLite would attempt to rewrite references to the renamed table in every view definition, which fails when a view references the table that was just dropped.
+
 .. _python_api_transform_sql:
 
 Custom transformations with .transform_sql()

sqlite_utils/db.py

@@ -2824,17 +2824,34 @@ class Table(Queryable):
             new_cols=", ".join(quote_identifier(col) for col in new_cols),
         )
         sqls.append(copy_sql)
-        # Drop (or keep) the old table
+        # Drop (or keep) the old table, then rename the new one into place.
+        # Since SQLite 3.25 ALTER TABLE ... RENAME TO rewrites references to
+        # the renamed table in every view definition, which fails if a view
+        # references the table that was just dropped - and with keep_table=
+        # would silently repoint views at the backup table. These renames are
+        # an implementation detail of transform(), so use legacy_alter_table
+        # to leave view definitions untouched, restoring the connection's
+        # current value afterwards.
+        legacy_alter_table_row = self.db.execute("PRAGMA legacy_alter_table").fetchone()
+        legacy_alter_table_was_on = bool(
+            legacy_alter_table_row and legacy_alter_table_row[0]
+        )
         if keep_table:
+            sqls.append("PRAGMA legacy_alter_table=ON;")
             sqls.append(
                 f"ALTER TABLE {quote_identifier(self.name)} RENAME TO {quote_identifier(keep_table)};"
             )
         else:
             sqls.append(f"DROP TABLE {quote_identifier(self.name)};")
-        # Rename the new one
+            sqls.append("PRAGMA legacy_alter_table=ON;")
         sqls.append(
             f"ALTER TABLE {quote_identifier(new_table_name)} RENAME TO {quote_identifier(self.name)};"
         )
+        sqls.append(
+            "PRAGMA legacy_alter_table={};".format(
+                "ON" if legacy_alter_table_was_on else "OFF"
+            )
+        )
         # Re-add existing indexes
         for index in self.indexes:
             if index.origin != "pk":

tests/test_transform.py

@@ -16,7 +16,9 @@ from sqlite_utils.utils import OperationalError
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
         # Change column type
@@ -26,7 +28,9 @@ from sqlite_utils.utils import OperationalError
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" INTEGER\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
         # Rename a column
@@ -36,7 +40,9 @@ from sqlite_utils.utils import OperationalError
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "dog_age" TEXT\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
         # Drop a column
@@ -46,7 +52,9 @@ from sqlite_utils.utils import OperationalError
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n   SELECT "rowid", "id", "name" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
         # Convert type AND rename column
@@ -56,7 +64,9 @@ from sqlite_utils.utils import OperationalError
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "dog_age" INTEGER\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
         # Change primary key
@@ -66,7 +76,9 @@ from sqlite_utils.utils import OperationalError
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT PRIMARY KEY\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
         # Change primary key to a compound pk
@@ -76,7 +88,9 @@ from sqlite_utils.utils import OperationalError
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT,\n   PRIMARY KEY ("age", "name")\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
         # Remove primary key, creating a rowid table
@@ -86,7 +100,9 @@ from sqlite_utils.utils import OperationalError
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
         # Keeping the table
@@ -95,8 +111,10 @@ from sqlite_utils.utils import OperationalError
             [
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name")\n   SELECT "rowid", "id", "name" FROM "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs" RENAME TO "kept_table";',
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
     ],
@@ -139,7 +157,9 @@ def test_transform_sql_table_with_primary_key(
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" TEXT\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
         # Change column type
@@ -149,7 +169,9 @@ def test_transform_sql_table_with_primary_key(
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "age" INTEGER\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
         # Rename a column
@@ -159,7 +181,9 @@ def test_transform_sql_table_with_primary_key(
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER,\n   "name" TEXT,\n   "dog_age" TEXT\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
         # Make ID a primary key
@@ -169,7 +193,9 @@ def test_transform_sql_table_with_primary_key(
                 'CREATE TABLE "dogs_new_suffix" (\n   "id" INTEGER PRIMARY KEY,\n   "name" TEXT,\n   "age" TEXT\n);',
                 'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n   SELECT "rowid", "id", "name", "age" FROM "dogs";',
                 'DROP TABLE "dogs";',
+                "PRAGMA legacy_alter_table=ON;",
                 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
+                "PRAGMA legacy_alter_table=OFF;",
             ],
         ),
     ],
@@ -903,3 +929,139 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db):
         "You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation."
         in str(excinfo.value)
     )
+
+
+def test_transform_preserves_view(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/831
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo"}, pk="id")
+    fresh_db.execute("create view dogs_view as select id, name from dogs")
+    view_sql_before = fresh_db.execute(
+        "select sql from sqlite_master where name = 'dogs_view'"
+    ).fetchone()[0]
+    dogs.transform(rename={"name": "title"})
+    view_sql_after = fresh_db.execute(
+        "select sql from sqlite_master where name = 'dogs_view'"
+    ).fetchone()[0]
+    assert view_sql_before == view_sql_after
+
+
+@pytest.mark.parametrize(
+    "transform_params",
+    [
+        {"types": {"name": int}},
+        {"pk": "name"},
+        {"add_foreign_keys": [("other_id", "other", "id")]},
+        {"drop_foreign_keys": ["other_id"]},
+    ],
+)
+def test_transform_variants_preserve_view(fresh_db, transform_params):
+    # Covers retyping, changing primary key and foreign key modifications,
+    # with a view whose columns are untouched by the transform
+    fresh_db["other"].insert({"id": 1}, pk="id")
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo", "other_id": 1}, pk="id")
+    if "drop_foreign_keys" in transform_params:
+        dogs.transform(add_foreign_keys=[("other_id", "other", "id")])
+    fresh_db.execute("create view dogs_view as select id, name from dogs")
+    view_sql_before = fresh_db.execute(
+        "select sql from sqlite_master where name = 'dogs_view'"
+    ).fetchone()[0]
+    dogs.transform(**transform_params)
+    view_sql_after = fresh_db.execute(
+        "select sql from sqlite_master where name = 'dogs_view'"
+    ).fetchone()[0]
+    assert view_sql_before == view_sql_after
+    assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}]
+
+
+def test_transform_view_referencing_renamed_column(fresh_db):
+    # The view survives but querying it raises "no such column" - inherent
+    # to SQLite views, whose SQL is stored as text
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo"}, pk="id")
+    fresh_db.execute("create view dogs_view as select id, name from dogs")
+    dogs.transform(rename={"name": "title"})
+    with pytest.raises(OperationalError, match="no such column"):
+        fresh_db.execute("select * from dogs_view")
+
+
+def test_transform_view_on_view(fresh_db):
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo"}, pk="id")
+    fresh_db.execute("create view v1 as select id, name from dogs")
+    fresh_db.execute("create view v2 as select name from v1")
+    sqls_before = fresh_db.execute(
+        "select sql from sqlite_master where type = 'view' order by name"
+    ).fetchall()
+    dogs.transform(types={"id": str})
+    sqls_after = fresh_db.execute(
+        "select sql from sqlite_master where type = 'view' order by name"
+    ).fetchall()
+    assert sqls_before == sqls_after
+    assert list(fresh_db["v2"].rows) == [{"name": "Cleo"}]
+
+
+def test_transform_keep_table_does_not_repoint_view(fresh_db):
+    # Without legacy_alter_table the ALTER TABLE dogs RENAME TO dogs_backup
+    # step would rewrite the view to select from "dogs_backup"
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo"}, pk="id")
+    fresh_db.execute("create view dogs_view as select id, name from dogs")
+    dogs.transform(types={"name": str}, keep_table="dogs_backup")
+    view_sql = fresh_db.execute(
+        "select sql from sqlite_master where name = 'dogs_view'"
+    ).fetchone()[0]
+    assert "dogs_backup" not in view_sql
+    # View reads from the live table, not the frozen backup
+    dogs.insert({"id": 2, "name": "Pancakes"})
+    assert list(fresh_db["dogs_view"].rows) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 2, "name": "Pancakes"},
+    ]
+
+
+def test_transform_sql_standalone_statements_work_with_view(fresh_db):
+    # The documented "run these statements yourself" workflow should be
+    # standalone-correct, so the pragmas must come from transform_sql()
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo"}, pk="id")
+    fresh_db.execute("create view dogs_view as select id, name from dogs")
+    sqls = dogs.transform_sql(types={"name": str}, tmp_suffix="suffix")
+    assert sqls[-3] == "PRAGMA legacy_alter_table=ON;"
+    assert sqls[-2] == 'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";'
+    assert sqls[-1] == "PRAGMA legacy_alter_table=OFF;"
+    for sql in sqls:
+        fresh_db.execute(sql)
+    assert list(fresh_db["dogs_view"].rows) == [{"id": 1, "name": "Cleo"}]
+
+
+def test_transform_with_view_in_open_transaction(fresh_db):
+    fresh_db.conn.execute("PRAGMA foreign_keys=ON")
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo"}, pk="id")
+    fresh_db.execute("create view dogs_view as select id, name from dogs")
+    with fresh_db.conn:
+        fresh_db.execute("insert into dogs (id, name) values (2, 'Pancakes')")
+        dogs.transform(rename={"name": "title"})
+    assert dogs.columns_dict == {"id": int, "title": str}
+    view_sql = fresh_db.execute(
+        "select sql from sqlite_master where name = 'dogs_view'"
+    ).fetchone()[0]
+    assert view_sql == "CREATE VIEW dogs_view as select id, name from dogs"
+
+
+def test_transform_restores_legacy_alter_table_setting(fresh_db):
+    if sqlite3.sqlite_version_info < (3, 25, 0):
+        pytest.skip("legacy_alter_table pragma requires SQLite 3.25 or higher")
+    dogs = fresh_db["dogs"]
+    dogs.insert({"id": 1, "name": "Cleo"}, pk="id")
+    # Default is OFF, reset to OFF afterwards
+    dogs.transform(types={"name": str})
+    assert fresh_db.execute("PRAGMA legacy_alter_table").fetchone()[0] == 0
+    # If the connection has it ON, it should be restored to ON
+    fresh_db.execute("PRAGMA legacy_alter_table=ON")
+    sqls = dogs.transform_sql(types={"name": str}, tmp_suffix="suffix")
+    assert sqls[-1] == "PRAGMA legacy_alter_table=ON;"
+    dogs.transform(types={"name": str})
+    assert fresh_db.execute("PRAGMA legacy_alter_table").fetchone()[0] == 1