changes to allow for compound foreign keys

This commit is contained in:
David Kane 2020-11-16 00:18:11 +00:00
commit 1c9e21f9a2
6 changed files with 213 additions and 95 deletions

View file

@ -51,13 +51,66 @@ except ImportError:
Column = namedtuple(
"Column", ("cid", "name", "type", "notnull", "default_value", "is_pk")
)
ForeignKey = namedtuple(
"ForeignKey", ("table", "column", "other_table", "other_column")
)
Index = namedtuple("Index", ("seq", "name", "unique", "origin", "partial", "columns"))
Trigger = namedtuple("Trigger", ("name", "table", "sql"))
class ForeignKey:
def __init__(self, table, column, other_table, other_column):
self.table = table
if isinstance(column, str):
self.column = (column,)
elif isinstance(column, (tuple, list)):
self.column = tuple(column)
self.other_table = other_table
if isinstance(other_column, str):
self.other_column = (other_column,)
elif isinstance(other_column, (tuple, list)):
self.other_column = tuple(other_column)
@property
def column_str(self):
return ",".join(["[{}]".format(c) for c in self.column])
@property
def other_column_str(self):
return ",".join(["[{}]".format(c) for c in self.other_column])
def __eq__(self, other):
if isinstance(other, ForeignKey):
return all((
self.table == other.table,
self.column == other.column,
self.other_table == other.other_table,
self.other_column == other.other_column,
))
return False
def __lt__(self, other):
if isinstance(other, ForeignKey):
return ((self.table, self.column, self.other_table, self.other_column) <
(other.table, other.column, other.other_table, other.other_column))
return False
def __repr__(self):
return "ForeignKey({table}({column}), {other_table}({other_column}))".format(
table=self.table,
column=self.column_str,
other_table=self.other_table,
other_column=self.other_column_str,
)
@property
def sql(self):
return "FOREIGN KEY({column}) REFERENCES [{other_table}]({other_column})".format(
table=self.table,
column=self.column_str,
other_table=self.other_table,
other_column=self.other_column_str,
)
DEFAULT = object()
COLUMN_TYPE_MAPPING = {
@ -362,12 +415,13 @@ class Database:
pk = hash_id
# Soundness check foreign_keys point to existing tables
for fk in foreign_keys:
if not any(
c for c in self[fk.other_table].columns if c.name == fk.other_column
):
raise AlterError(
"No such column: {}.{}".format(fk.other_table, fk.other_column)
)
for oc in fk.other_column:
if not any(
c for c in self[fk.other_table].columns if c.name == oc
):
raise AlterError(
"No such column: {}.{}".format(fk.other_table, oc)
)
column_defs = []
# ensure pk is a tuple
@ -388,13 +442,6 @@ class Database:
column_extras.append(
"DEFAULT {}".format(self.escape(defaults[column_name]))
)
if column_name in foreign_keys_by_column:
column_extras.append(
"REFERENCES [{other_table}]([{other_column}])".format(
other_table=foreign_keys_by_column[column_name].other_table,
other_column=foreign_keys_by_column[column_name].other_column,
)
)
column_defs.append(
" [{column_name}] {column_type}{column_extras}".format(
column_name=column_name,
@ -409,6 +456,10 @@ class Database:
extra_pk = ",\n PRIMARY KEY ({pks})".format(
pks=", ".join(["[{}]".format(p) for p in pk])
)
for column_name in foreign_keys_by_column:
extra_pk += ",\n {}".format(
foreign_keys_by_column[column_name].sql,
)
columns_sql = ",\n".join(column_defs)
sql = """CREATE TABLE [{table}] (
{columns_sql}{extra_pk}
@ -481,7 +532,7 @@ class Database:
candidates.append(table.name)
return candidates
def add_foreign_keys(self, foreign_keys):
def add_foreign_keys(self, foreign_keys, ignore=True):
# foreign_keys is a list of explicit 4-tuples
assert all(
len(fk) == 4 and isinstance(fk, (list, tuple)) for fk in foreign_keys
@ -490,43 +541,58 @@ class Database:
foreign_keys_to_create = []
# Verify that all tables and columns exist
for table, column, other_table, other_column in foreign_keys:
if not self[table].exists():
raise AlterError("No such table: {}".format(table))
if column not in self[table].columns_dict:
raise AlterError("No such column: {} in {}".format(column, table))
if not self[other_table].exists():
raise AlterError("No such other_table: {}".format(other_table))
if (
other_column != "rowid"
and other_column not in self[other_table].columns_dict
):
raise AlterError(
"No such other_column: {} in {}".format(other_column, other_table)
for fk in foreign_keys:
if not isinstance(fk, ForeignKey):
fk = ForeignKey(
table=fk[0],
column=fk[1],
other_table=fk[2],
other_column=fk[3],
)
if not self[fk.table].exists():
raise AlterError("No such table: {}".format(fk.table))
for c in fk.column:
if c not in self[fk.table].columns_dict:
raise AlterError("No such column: {} in {}".format(c, fk.table))
if not self[fk.other_table].exists():
raise AlterError("No such other_table: {}".format(fk.other_table))
for c in fk.other_column:
if (
c != "rowid"
and c not in self[fk.other_table].columns_dict
):
raise AlterError(
"No such other_column: {} in {}".format(c, fk.other_table)
)
# We will silently skip foreign keys that exist already
if not any(
fk
for fk in self[table].foreign_keys
if fk.column == column
and fk.other_table == other_table
and fk.other_column == other_column
if any(
existing_fk
for existing_fk in self[fk.table].foreign_keys
if existing_fk.column == fk.column
and existing_fk.other_table == fk.other_table
and existing_fk.other_column == fk.other_column
):
foreign_keys_to_create.append(
(table, column, other_table, other_column)
)
if ignore:
continue
else:
raise AlterError(
"Foreign key already exists for {} => {}.{}".format(
fk.column_str, other_table, fk.other_column_str
)
)
else:
foreign_keys_to_create.append(fk)
# Construct SQL for use with "UPDATE sqlite_master SET sql = ? WHERE name = ?"
table_sql = {}
for table, column, other_table, other_column in foreign_keys_to_create:
old_sql = table_sql.get(table, self[table].schema)
extra_sql = ",\n FOREIGN KEY({column}) REFERENCES {other_table}({other_column})\n".format(
column=column, other_table=other_table, other_column=other_column
)
for fk in foreign_keys_to_create:
old_sql = table_sql.get(fk.table, self[fk.table].schema)
extra_sql = ",\n {}\n".format(fk.sql)
# Stick that bit in at the very end just before the closing ')'
last_paren = old_sql.rindex(")")
new_sql = old_sql[:last_paren].strip() + extra_sql + old_sql[last_paren:]
table_sql[table] = new_sql
table_sql[fk.table] = new_sql
# And execute it all within a single transaction
with self.conn:
@ -548,11 +614,11 @@ class Database:
for table_name in self.table_names():
table = self[table_name]
existing_indexes = {
i.columns[0] for i in table.indexes if len(i.columns) == 1
tuple(i.columns) for i in table.indexes
}
for fk in table.foreign_keys:
if fk.column not in existing_indexes:
table.create_index([fk.column])
table.create_index(fk.column)
def vacuum(self):
self.execute("VACUUM;")
@ -688,21 +754,27 @@ class Table(Queryable):
@property
def foreign_keys(self):
fks = []
fks = {}
for row in self.db.execute(
"PRAGMA foreign_key_list([{}])".format(self.name)
).fetchall():
if row is not None:
id, seq, table_name, from_, to_, on_update, on_delete, match = row
fks.append(
ForeignKey(
table=self.name,
column=from_,
other_table=table_name,
other_column=to_,
)
)
return fks
if id not in fks:
fks[id] = {
"column": [],
"other_column": [],
}
fks[id]["table"] = self.name
fks[id]["column"].append(from_)
fks[id]["other_table"] = table_name
fks[id]["other_column"].append(to_)
return [ForeignKey(
table=fk['table'],
column=tuple(fk['column']),
other_table=fk['other_table'],
other_column=tuple(fk['other_column']),
) for fk in fks.values()]
@property
def virtual_table_using(self):
@ -886,10 +958,16 @@ class Table(Queryable):
# foreign_keys
create_table_foreign_keys = []
for table, column, other_table, other_column in self.foreign_keys:
if (drop_foreign_keys is None) or (column not in drop_foreign_keys):
if drop_foreign_keys:
drop_foreign_keys = [
(f,) if not isinstance(f, (tuple, list)) else tuple(f)
for f in drop_foreign_keys
]
for fk in self.foreign_keys:
if (drop_foreign_keys is None) or (fk.column not in drop_foreign_keys):
column_names = tuple(rename.get(c) or c for c in fk.column)
create_table_foreign_keys.append(
(rename.get(column) or column, other_table, other_column)
(column_names, fk.other_table, fk.other_column)
)
if column_order is not None:
@ -1089,6 +1167,8 @@ class Table(Queryable):
self.db.execute("DROP TABLE [{}]".format(self.name))
def guess_foreign_table(self, column):
if isinstance(column, (tuple, list)):
column = column[0]
column = column.lower()
possibilities = [column]
if column.endswith("_id"):
@ -1121,22 +1201,28 @@ class Table(Queryable):
def add_foreign_key(
self, column, other_table=None, other_column=None, ignore=False
):
if not isinstance(column, (tuple, list)):
column = (column,)
# Ensure column exists
if column not in self.columns_dict:
raise AlterError("No such column: {}".format(column))
for c in column:
if c not in self.columns_dict:
raise AlterError("No such column: {}".format(c))
# If other_table is not specified, attempt to guess it from the column
if other_table is None:
other_table = self.guess_foreign_table(column)
# If other_column is not specified, detect the primary key on other_table
if other_column is None:
other_column = self.guess_foreign_column(other_table)
if not isinstance(other_column, (tuple, list)):
other_column = (other_column,)
# Soundness check that the other column exists
if (
not [c for c in self.db[other_table].columns if c.name == other_column]
and other_column != "rowid"
):
raise AlterError("No such column: {}.{}".format(other_table, other_column))
for oc in other_column:
if (
not [c for c in self.db[other_table].columns if c.name == oc]
and oc != "rowid"
):
raise AlterError("No such column: {}.{}".format(other_table, oc))
# Check we do not already have an existing foreign key
if any(
fk
@ -1150,7 +1236,7 @@ class Table(Queryable):
else:
raise AlterError(
"Foreign key already exists for {} => {}.{}".format(
column, other_table, other_column
",".join(column), other_table, ",".join(other_column)
)
)
self.db.add_foreign_keys([(self.name, column, other_table, other_column)])

View file

@ -321,7 +321,7 @@ def test_add_column_foreign_key(db_path):
)
assert 0 == result.exit_code, result.output
assert (
"CREATE TABLE [books] ( [title] TEXT , [author_id] INTEGER, FOREIGN KEY(author_id) REFERENCES authors(id) )"
"CREATE TABLE [books] ( [title] TEXT , [author_id] INTEGER, FOREIGN KEY([author_id]) REFERENCES [authors]([id]) )"
== collapse_whitespace(db["books"].schema)
)
# Try it again with a custom --fk-col
@ -341,8 +341,8 @@ def test_add_column_foreign_key(db_path):
assert 0 == result.exit_code, result.output
assert (
"CREATE TABLE [books] ( [title] TEXT , [author_id] INTEGER, [author_name_ref] TEXT, "
"FOREIGN KEY(author_id) REFERENCES authors(id), "
"FOREIGN KEY(author_name_ref) REFERENCES authors(name) )"
"FOREIGN KEY([author_id]) REFERENCES [authors]([id]), "
"FOREIGN KEY([author_name_ref]) REFERENCES [authors]([name]) )"
== collapse_whitespace(db["books"].schema)
)
# Throw an error if the --fk table does not exist
@ -1225,7 +1225,8 @@ def test_create_table_foreign_key():
"CREATE TABLE [books] (\n"
" [id] INTEGER PRIMARY KEY,\n"
" [title] TEXT,\n"
" [author_id] INTEGER REFERENCES [authors]([id])\n"
" [author_id] INTEGER,\n"
" FOREIGN KEY([author_id]) REFERENCES [authors]([id])\n"
")"
) == db["books"].schema
@ -1597,7 +1598,7 @@ def test_transform_drop_foreign_key(db_path):
schema = db["places"].schema
assert (
schema
== 'CREATE TABLE "places" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [country] INTEGER,\n [city] INTEGER REFERENCES [city]([id])\n)'
== 'CREATE TABLE "places" (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT,\n [country] INTEGER,\n [city] INTEGER,\n FOREIGN KEY([city]) REFERENCES [city]([id])\n)'
)
@ -1611,22 +1612,22 @@ _common_other_schema = (
[
(
[],
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY(species_id) REFERENCES species(id)\n)',
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)',
_common_other_schema,
),
(
["--table", "custom_table"],
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_table_id] INTEGER,\n FOREIGN KEY(custom_table_id) REFERENCES custom_table(id)\n)',
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_table_id] INTEGER,\n FOREIGN KEY([custom_table_id]) REFERENCES [custom_table]([id])\n)',
"CREATE TABLE [custom_table] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)",
),
(
["--fk-column", "custom_fk"],
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_fk] INTEGER,\n FOREIGN KEY(custom_fk) REFERENCES species(id)\n)',
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [custom_fk] INTEGER,\n FOREIGN KEY([custom_fk]) REFERENCES [species]([id])\n)',
_common_other_schema,
),
(
["--rename", "name", "name2"],
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY(species_id) REFERENCES species(id)\n)',
'CREATE TABLE "trees" (\n [id] INTEGER PRIMARY KEY,\n [address] TEXT,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [species]([id])\n)',
"CREATE TABLE [species] (\n [id] INTEGER PRIMARY KEY,\n [species] TEXT\n)",
),
],

View file

@ -268,8 +268,8 @@ def test_create_table_works_for_m2m_with_only_foreign_keys(
] == [{"name": col.name, "type": col.type} for col in fresh_db["m2m"].columns]
assert sorted(
[
{"column": "one_id", "other_table": "one", "other_column": "id"},
{"column": "two_id", "other_table": "two", "other_column": "id"},
{"column": ("one_id",), "other_table": "one", "other_column": ("id",)},
{"column": ("two_id",), "other_table": "two", "other_column": ("id",)},
],
key=lambda s: repr(s),
) == sorted(
@ -355,7 +355,37 @@ def test_add_foreign_key(fresh_db):
assert isinstance(t, Table) and t.name == "books"
assert [
ForeignKey(
table="books", column="author_id", other_table="authors", other_column="id"
table="books", column=("author_id",), other_table="authors", other_column=("id",)
)
] == fresh_db["books"].foreign_keys
def test_add_compound_foreign_key(fresh_db):
fresh_db["authors"].insert_all(
[
{"id": 1, "person_id": 1, "name": "Sally"},
{"id": 2, "person_id": 2, "name": "Asheesh"}
],
pk=("id", "person_id")
)
fresh_db["books"].insert_all(
[
{"title": "Hedgehogs of the world", "author_id": 1, "author_person_id": 1},
{"title": "How to train your wolf", "author_id": 2, "author_person_id": 2},
]
)
assert [] == fresh_db["books"].foreign_keys
t = fresh_db["books"].add_foreign_key(
("author_id", "author_person_id"), "authors", ("id", "person_id")
)
# Ensure it returned self:
assert isinstance(t, Table) and t.name == "books"
assert [
ForeignKey(
table="books",
column=("author_id", "author_person_id"),
other_table="authors",
other_column=("id", "person_id"),
)
] == fresh_db["books"].foreign_keys
@ -378,6 +408,7 @@ def test_add_foreign_key_error_if_already_exists(fresh_db):
fresh_db["books"].insert({"title": "Hedgehogs of the world", "author_id": 1})
fresh_db["authors"].insert({"id": 1, "name": "Sally"}, pk="id")
fresh_db["books"].add_foreign_key("author_id", "authors", "id")
print(fresh_db["books"].foreign_keys)
with pytest.raises(AlterError) as ex:
fresh_db["books"].add_foreign_key("author_id", "authors", "id")
assert "Foreign key already exists for author_id => authors.id" == ex.value.args[0]
@ -423,7 +454,7 @@ def test_add_column_foreign_key(fresh_db):
fresh_db.create_table("breeds", {"name": str})
fresh_db["dogs"].add_column("breed_id", fk="breeds")
assert (
"CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, FOREIGN KEY(breed_id) REFERENCES breeds(rowid) )"
"CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, FOREIGN KEY([breed_id]) REFERENCES [breeds]([rowid]) )"
== collapse_whitespace(fresh_db["dogs"].schema)
)
# And again with an explicit primary key column
@ -431,8 +462,8 @@ def test_add_column_foreign_key(fresh_db):
fresh_db["dogs"].add_column("subbreed_id", fk="subbreeds")
assert (
"CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, [subbreed_id] TEXT, "
"FOREIGN KEY(breed_id) REFERENCES breeds(rowid), "
"FOREIGN KEY(subbreed_id) REFERENCES subbreeds(primkey) )"
"FOREIGN KEY([breed_id]) REFERENCES [breeds]([rowid]), "
"FOREIGN KEY([subbreed_id]) REFERENCES [subbreeds]([primkey]) )"
== collapse_whitespace(fresh_db["dogs"].schema)
)
@ -443,7 +474,7 @@ def test_add_foreign_key_guess_table(fresh_db):
fresh_db["dogs"].add_column("breed_id", int)
fresh_db["dogs"].add_foreign_key("breed_id")
assert (
"CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, FOREIGN KEY(breed_id) REFERENCES breeds(id) )"
"CREATE TABLE [dogs] ( [name] TEXT , [breed_id] INTEGER, FOREIGN KEY([breed_id]) REFERENCES [breeds]([id]) )"
== collapse_whitespace(fresh_db["dogs"].schema)
)

View file

@ -28,7 +28,7 @@ def test_extract_single_column(fresh_db, table, fk_column):
" [name] TEXT,\n"
" [{}] INTEGER,\n".format(expected_fk)
+ " [end] INTEGER,\n"
+ " FOREIGN KEY({}) REFERENCES {}(id)\n".format(expected_fk, expected_table)
+ " FOREIGN KEY([{}]) REFERENCES [{}]([id])\n".format(expected_fk, expected_table)
+ ")"
)
assert fresh_db[expected_table].schema == (
@ -75,7 +75,7 @@ def test_extract_multiple_columns_with_rename(fresh_db):
" [id] INTEGER PRIMARY KEY,\n"
" [name] TEXT,\n"
" [common_name_latin_name_id] INTEGER,\n"
" FOREIGN KEY(common_name_latin_name_id) REFERENCES common_name_latin_name(id)\n"
" FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n"
")"
)
assert fresh_db["common_name_latin_name"].schema == (
@ -127,7 +127,7 @@ def test_extract_rowid_table(fresh_db):
" [rowid] INTEGER PRIMARY KEY,\n"
" [name] TEXT,\n"
" [common_name_latin_name_id] INTEGER,\n"
" FOREIGN KEY(common_name_latin_name_id) REFERENCES common_name_latin_name(id)\n"
" FOREIGN KEY([common_name_latin_name_id]) REFERENCES [common_name_latin_name]([id])\n"
")"
)
@ -144,7 +144,7 @@ def test_reuse_lookup_table(fresh_db):
'CREATE TABLE "sightings" (\n'
" [id] INTEGER PRIMARY KEY,\n"
" [species_id] INTEGER,\n"
" FOREIGN KEY(species_id) REFERENCES species(id)\n"
" FOREIGN KEY([species_id]) REFERENCES [species]([id])\n"
")"
)
assert fresh_db["individuals"].schema == (
@ -152,7 +152,7 @@ def test_reuse_lookup_table(fresh_db):
" [id] INTEGER PRIMARY KEY,\n"
" [name] TEXT,\n"
" [species_id] INTEGER,\n"
" FOREIGN KEY(species_id) REFERENCES species(id)\n"
" FOREIGN KEY([species_id]) REFERENCES [species]([id])\n"
")"
)
assert list(fresh_db["species"].rows) == [

View file

@ -36,7 +36,7 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory):
== fresh_db[expected_table].schema
)
assert (
"CREATE TABLE [Trees] (\n [id] INTEGER,\n [species_id] INTEGER REFERENCES [{}]([id])\n)".format(
"CREATE TABLE [Trees] (\n [id] INTEGER,\n [species_id] INTEGER,\n FOREIGN KEY([species_id]) REFERENCES [{}]([id])\n)".format(
expected_table
)
== fresh_db["Trees"].schema
@ -45,7 +45,7 @@ def test_extracts(fresh_db, kwargs, expected_table, use_table_factory):
assert len(fresh_db["Trees"].foreign_keys) == 1
fk = fresh_db["Trees"].foreign_keys[0]
assert fk.table == "Trees"
assert fk.column == "species_id"
assert fk.column == ("species_id",)
# Should have unique index on Species
assert [

View file

@ -30,7 +30,7 @@ def test_insert_m2m_list(fresh_db):
assert [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}] == list(
humans.rows
)
assert [
assert sorted([
ForeignKey(
table="dogs_humans", column="dogs_id", other_table="dogs", other_column="id"
),
@ -40,7 +40,7 @@ def test_insert_m2m_list(fresh_db):
other_table="humans",
other_column="id",
),
] == dogs_humans.foreign_keys
]) == sorted(dogs_humans.foreign_keys)
def test_insert_m2m_iterable(fresh_db):
@ -103,7 +103,7 @@ def test_m2m_lookup(fresh_db):
tags = fresh_db["tags"]
assert people_tags.exists()
assert tags.exists()
assert [
assert sorted([
ForeignKey(
table="people_tags",
column="people_id",
@ -113,7 +113,7 @@ def test_m2m_lookup(fresh_db):
ForeignKey(
table="people_tags", column="tags_id", other_table="tags", other_column="id"
),
] == people_tags.foreign_keys
]) == sorted(people_tags.foreign_keys)
assert [{"people_id": 1, "tags_id": 1}] == list(people_tags.rows)
assert [{"id": 1, "name": "Wahyu"}] == list(people.rows)
assert [{"id": 1, "tag": "Coworker"}] == list(tags.rows)