From 9beecc6eb0e1c62a7bb430ac75b835df8eb5209a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Jun 2019 08:39:50 -0700 Subject: [PATCH 1/6] db.add_foreign_keys() method, refs #31 Still needs unit tests and documentation. --- sqlite_utils/db.py | 83 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 19 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 414ec97..0846965 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -252,6 +252,69 @@ class Database: ) ) + def add_foreign_keys(self, foreign_keys): + # foreign_keys is a list of explicit 4-tuples + assert all( + len(fk) == 4 and isinstance(fk, (list, tuple)) for fk in foreign_keys + ), "foreign_keys must be a list of 4-tuples, (table, column, other_table, other_column)" + + foreign_keys_to_create = [] + + # Verify that all tables and columns exist + for table, column, other_table, other_column in foreign_keys: + if not self[table].exists: + raise AlterError("No such table: {}".format(table)) + if column not in self[table].columns_dict: + raise AlterError("No such column: {} in {}".format(column, table)) + if not self[other_table].exists: + raise AlterError("No such other_table: {}".format(other_table)) + if ( + other_column != "rowid" + and other_column not in self[other_table].columns_dict + ): + raise AlterError( + "No such other_column: {} in {}".format(other_column, other_table) + ) + # We will silently skip foreign keys that exist already + if not any( + fk + for fk in self[table].foreign_keys + if fk.column == column + and fk.other_table == other_table + and fk.other_column == other_column + ): + foreign_keys_to_create.append( + (table, column, other_table, other_column) + ) + + # Construct list of args to be used with "UPDATE sqlite_master SET sql = ? WHERE name = ?" + sql_args = [] + for table, column, other_table, other_column in foreign_keys: + old_sql = self[table].schema + extra_sql = ",\n FOREIGN KEY({column}) REFERENCES {other_table}({other_column})\n".format( + column=column, other_table=other_table, other_column=other_column + ) + # Stick that bit in at the very end just before the closing ')' + last_paren = old_sql.rindex(")") + new_sql = old_sql[:last_paren].strip() + extra_sql + old_sql[last_paren:] + sql_args.append((new_sql, table)) + + # And execute it all within a single transaction + with self.conn: + cursor = self.conn.cursor() + schema_version = cursor.execute("PRAGMA schema_version").fetchone()[0] + cursor.execute("PRAGMA writable_schema = 1") + for new_sql, table_name in sql_args: + cursor.execute( + "UPDATE sqlite_master SET sql = ? WHERE name = ?", + (new_sql, table_name), + ) + cursor.execute("PRAGMA schema_version = %d" % (schema_version + 1)) + cursor.execute("PRAGMA writable_schema = 0") + # Have to VACUUM outside the transaction to ensure .foreign_keys property + # can see the newly created foreign key. + self.vacuum() + def vacuum(self): self.conn.execute("VACUUM;") @@ -495,25 +558,7 @@ class Table: column, other_table, other_column ) ) - with self.db.conn: - cursor = self.db.conn.cursor() - schema_version = cursor.execute("PRAGMA schema_version").fetchone()[0] - cursor.execute("PRAGMA writable_schema = 1") - old_sql = self.schema - extra_sql = ",\n FOREIGN KEY({column}) REFERENCES {other_table}({other_column})\n".format( - column=column, other_table=other_table, other_column=other_column - ) - # Stick that bit in at the very end just before the closing ')' - last_paren = old_sql.rindex(")") - new_sql = old_sql[:last_paren].strip() + extra_sql + old_sql[last_paren:] - cursor.execute( - "UPDATE sqlite_master SET sql = ? WHERE name = ?", (new_sql, self.name) - ) - cursor.execute("PRAGMA schema_version = %d" % (schema_version + 1)) - cursor.execute("PRAGMA writable_schema = 0") - # Have to VACUUM outside the transaction to ensure .foreign_keys property - # can see the newly created foreign key. - cursor.execute("VACUUM") + self.db.add_foreign_keys([(self.name, column, other_table, other_column)]) def enable_fts(self, columns, fts_version="FTS5"): "Enables FTS on the specified columns" From 4230560e94e797956cb72b20cad2d5bc61761918 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Jun 2019 17:47:57 -0700 Subject: [PATCH 2/6] Docs for db.add_foreign_keys() --- docs/python-api.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index 1243a17..377a4c5 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -399,6 +399,24 @@ The ``table.add_foreign_key(column, other_table, other_column)`` method takes th - If the column is of format ``author_id``, look for tables called ``author`` or ``authors`` - If the column does not end in ``_id``, try looking for a table with the exact name of the column or that name with an added ``s`` +Adding multiple foreign key constraints at once +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The final step in adding a new foreign key to a SQLite database is to run ``VACUUM``, to ensure the new foreign key is available in future introspection queries. + +``VACUUM`` against a large (multi-GB) database can take several minutes or longer. If you are adding multiple foreign keys using ``table.add_foreign_key(...)`` these can quickly add up. + +Instead, you can use ``db.add_foreign_keys(...)`` to add multiple foreign keys within a single transaction. This method takes a list of four-tuples, each one specifying a ``table``, ``column``, ``other_table`` and ``other_column``. + +Here's an example adding two foreign keys at once: + +.. code-block:: python + + db.add_foreign_keys([ + ("dogs", "breed_id", "breeds", "id"), + ("dogs", "home_town_id", "towns", "id") + ]) + .. _python_api_hash: Setting an ID based on the hash of the row contents From 3db2c26ce2832fee3a2b333241b9f022fd867818 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Jun 2019 17:49:28 -0700 Subject: [PATCH 3/6] .rst fixes --- docs/python-api.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index 377a4c5..8928c03 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -399,8 +399,10 @@ The ``table.add_foreign_key(column, other_table, other_column)`` method takes th - If the column is of format ``author_id``, look for tables called ``author`` or ``authors`` - If the column does not end in ``_id``, try looking for a table with the exact name of the column or that name with an added ``s`` +.. _python_api_add_foreign_keys: + Adding multiple foreign key constraints at once -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------------------------- The final step in adding a new foreign key to a SQLite database is to run ``VACUUM``, to ensure the new foreign key is available in future introspection queries. From 889e2afccf8a252915b9273cd0421bc880ac483b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Jun 2019 23:21:23 -0700 Subject: [PATCH 4/6] Unit test + bug fix for db.add_foreign_keys() --- sqlite_utils/db.py | 10 +++++----- tests/test_create.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 0846965..9be09f6 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -287,24 +287,24 @@ class Database: (table, column, other_table, other_column) ) - # Construct list of args to be used with "UPDATE sqlite_master SET sql = ? WHERE name = ?" - sql_args = [] + # Construct SQL for use with "UPDATE sqlite_master SET sql = ? WHERE name = ?" + table_sql = {} for table, column, other_table, other_column in foreign_keys: - old_sql = self[table].schema + old_sql = table_sql.get(table, self[table].schema) extra_sql = ",\n FOREIGN KEY({column}) REFERENCES {other_table}({other_column})\n".format( column=column, other_table=other_table, other_column=other_column ) # Stick that bit in at the very end just before the closing ')' last_paren = old_sql.rindex(")") new_sql = old_sql[:last_paren].strip() + extra_sql + old_sql[last_paren:] - sql_args.append((new_sql, table)) + table_sql[table] = new_sql # And execute it all within a single transaction with self.conn: cursor = self.conn.cursor() schema_version = cursor.execute("PRAGMA schema_version").fetchone()[0] cursor.execute("PRAGMA writable_schema = 1") - for new_sql, table_name in sql_args: + for table_name, new_sql in table_sql.items(): cursor.execute( "UPDATE sqlite_master SET sql = ? WHERE name = ?", (new_sql, table_name), diff --git a/tests/test_create.py b/tests/test_create.py index 04b1fbf..88d7676 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -303,6 +303,34 @@ def test_add_foreign_key_error_if_already_exists(fresh_db): assert "Foreign key already exists for author_id => authors.id" == ex.value.args[0] +def test_add_foreign_keys(fresh_db): + fresh_db["authors"].insert_all( + [{"id": 1, "name": "Sally"}, {"id": 2, "name": "Asheesh"}], pk="id" + ) + fresh_db["categories"].insert_all([{"id": 1, "name": "Wildlife"}], pk="id") + fresh_db["books"].insert_all( + [{"title": "Hedgehogs of the world", "author_id": 1, "category_id": 1}] + ) + assert [] == fresh_db["books"].foreign_keys + fresh_db.add_foreign_keys( + [ + ("books", "author_id", "authors", "id"), + ("books", "category_id", "categories", "id"), + ] + ) + assert [ + ForeignKey( + table="books", column="author_id", other_table="authors", other_column="id" + ), + ForeignKey( + table="books", + column="category_id", + other_table="categories", + other_column="id", + ), + ] == sorted(fresh_db["books"].foreign_keys) + + def test_add_column_foreign_key(fresh_db): fresh_db.create_table("dogs", {"name": str}) fresh_db.create_table("breeds", {"name": str}) From fc81588cc31df58374c996884c67cedd98a06f4a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Jun 2019 23:23:16 -0700 Subject: [PATCH 5/6] Loop through correct list --- sqlite_utils/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9be09f6..582017f 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -289,7 +289,7 @@ class Database: # Construct SQL for use with "UPDATE sqlite_master SET sql = ? WHERE name = ?" table_sql = {} - for table, column, other_table, other_column in foreign_keys: + for table, column, other_table, other_column in foreign_keys_to_create: old_sql = table_sql.get(table, self[table].schema) extra_sql = ",\n FOREIGN KEY({column}) REFERENCES {other_table}({other_column})\n".format( column=column, other_table=other_table, other_column=other_column From b3b94fb023c5cdf6c00d87cfacdf74269d4f74c9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 28 Jun 2019 23:31:18 -0700 Subject: [PATCH 6/6] Release 1.3 --- docs/changelog.rst | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f75364d..5db9f1b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,12 @@ Changelog =========== +.. _v1_3: + +1.3 (2019-06-28) +---------------- + +- New mechanism for adding multiple foreign key constraints at once: :ref:`db.add_foreign_keys() documentation ` (`#31 `__) .. _v1_2_2: diff --git a/setup.py b/setup.py index 9230369..a38d4a8 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "1.2.2" +VERSION = "1.3" def get_long_description():