From 35eeafaaa33648a528cbcd57ceca966fea19c6ae Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jul 2019 08:31:27 +0300 Subject: [PATCH 1/8] table.m2m(...) method, with tests --- sqlite_utils/db.py | 20 ++++++++++++++++++++ tests/test_m2m.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 tests/test_m2m.py diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index bbc6177..9f50a53 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1048,6 +1048,26 @@ class Table: self.create_index(column_values.keys(), unique=True) return pk + def m2m(self, other_table, record_or_list, pk=None): + our_id = self.last_pk + records = ( + [record_or_list] + if not isinstance(record_or_list, (list, tuple)) + else record_or_list + ) + tables = list(sorted([self.name, other_table])) + columns = ["{}_id".format(t) for t in tables] + m2m_table = self.db.table( + "{}_{}".format(*tables), pk=columns, foreign_keys=columns + ) + # Ensure each record exists in other table + for record in records: + id = self.db[other_table].upsert(record, pk=pk).last_pk + m2m_table.upsert( + {"{}_id".format(other_table): id, "{}_id".format(self.name): our_id} + ) + return self + def chunks(sequence, size): iterator = iter(sequence) diff --git a/tests/test_m2m.py b/tests/test_m2m.py new file mode 100644 index 0000000..e7d6e30 --- /dev/null +++ b/tests/test_m2m.py @@ -0,0 +1,43 @@ +from sqlite_utils.db import ForeignKey +import pytest + + +def test_insert_m2m_single(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + "humans", {"id": 1, "name": "Natalie D"}, pk="id" + ) + assert {"dogs_humans", "humans", "dogs"} == set(fresh_db.table_names()) + humans = fresh_db["humans"] + dogs_humans = fresh_db["dogs_humans"] + assert [{"id": 1, "name": "Natalie D"}] == list(humans.rows) + assert [{"humans_id": 1, "dogs_id": 1}] == list(dogs_humans.rows) + + +def test_insert_m2m_list(fresh_db): + dogs = fresh_db["dogs"] + dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + "humans", + [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}], + pk="id", + ) + assert {"dogs", "humans", "dogs_humans"} == set(fresh_db.table_names()) + humans = fresh_db["humans"] + dogs_humans = fresh_db["dogs_humans"] + assert [{"humans_id": 1, "dogs_id": 1}, {"humans_id": 2, "dogs_id": 1}] == list( + dogs_humans.rows + ) + assert [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}] == list( + humans.rows + ) + assert [ + ForeignKey( + table="dogs_humans", column="dogs_id", other_table="dogs", other_column="id" + ), + ForeignKey( + table="dogs_humans", + column="humans_id", + other_table="humans", + other_column="id", + ), + ] == dogs_humans.foreign_keys From ff2348e71af6705dfa3220d823ce0285e95b127f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 31 Jul 2019 09:16:46 +0300 Subject: [PATCH 2/8] Added failing tests --- tests/test_m2m.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_m2m.py b/tests/test_m2m.py index e7d6e30..c7e593d 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -41,3 +41,20 @@ def test_insert_m2m_list(fresh_db): other_column="id", ), ] == dogs_humans.foreign_keys + + +def test_m2m_explicit_argument(fresh_db): + # .m2m("humans", ..., m2m_table="relationships") + assert False + + +def test_uses_existing_m2m_table_if_exists(fresh_db): + # Code should look for an existing toble with fks to both tables + # and use that if it exists. + assert False + + +def test_requires_explicit_m2m_table_if_multiple_options(fresh_db): + # If the code scans for m2m tables and finds more than one candidate + # it should require that the m2m_table=x argument is used + assert False From ba1211d4456911bf0bd13f2e753a56ed988df3b4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 3 Aug 2019 17:28:03 +0300 Subject: [PATCH 3/8] Implemented .m2m(table, lookup=...) --- sqlite_utils/db.py | 28 +++++++++++++++++++--------- tests/test_m2m.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 9f50a53..e759910 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1048,21 +1048,31 @@ class Table: self.create_index(column_values.keys(), unique=True) return pk - def m2m(self, other_table, record_or_list, pk=None): + def m2m(self, other_table, record_or_list=None, pk=None, lookup=None): our_id = self.last_pk - records = ( - [record_or_list] - if not isinstance(record_or_list, (list, tuple)) - else record_or_list - ) + if lookup is not None: + assert record_or_list is None, "Provide lookup= or record, not both" + else: + assert record_or_list is not None, "Provide lookup= or record, not both" tables = list(sorted([self.name, other_table])) columns = ["{}_id".format(t) for t in tables] m2m_table = self.db.table( "{}_{}".format(*tables), pk=columns, foreign_keys=columns ) - # Ensure each record exists in other table - for record in records: - id = self.db[other_table].upsert(record, pk=pk).last_pk + if lookup is None: + records = ( + [record_or_list] + if not isinstance(record_or_list, (list, tuple)) + else record_or_list + ) + # Ensure each record exists in other table + for record in records: + id = self.db[other_table].upsert(record, pk=pk).last_pk + m2m_table.upsert( + {"{}_id".format(other_table): id, "{}_id".format(self.name): our_id} + ) + else: + id = self.db[other_table].lookup(lookup) m2m_table.upsert( {"{}_id".format(other_table): id, "{}_id".format(self.name): our_id} ) diff --git a/tests/test_m2m.py b/tests/test_m2m.py index c7e593d..471166e 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -43,6 +43,37 @@ def test_insert_m2m_list(fresh_db): ] == dogs_humans.foreign_keys +def test_m2m_lookup(fresh_db): + people = fresh_db.table("people", pk="id") + people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) + people_tags = fresh_db["people_tags"] + tags = fresh_db["tags"] + assert people_tags.exists + assert tags.exists + assert [ + ForeignKey( + table="people_tags", + column="people_id", + other_table="people", + other_column="id", + ), + ForeignKey( + table="people_tags", column="tags_id", other_table="tags", other_column="id" + ), + ] == people_tags.foreign_keys + assert [{"people_id": 1, "tags_id": 1}] == list(people_tags.rows) + assert [{"id": 1, "name": "Wahyu"}] == list(people.rows) + assert [{"id": 1, "tag": "Coworker"}] == list(tags.rows) + + +def test_m2m_requires_either_records_or_lookup(fresh_db): + people = fresh_db.table("people", pk="id").insert({"name": "Wahyu"}) + with pytest.raises(AssertionError): + people.m2m("tags") + with pytest.raises(AssertionError): + people.m2m("tags", {"tag": "hello"}, lookup={"foo": "bar"}) + + def test_m2m_explicit_argument(fresh_db): # .m2m("humans", ..., m2m_table="relationships") assert False From b6b92980c00eda14a4d759b724139a0a2d321007 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 3 Aug 2019 20:51:22 +0300 Subject: [PATCH 4/8] table.m2m(..., m2m_table=x) argument --- sqlite_utils/db.py | 9 +++++---- tests/test_m2m.py | 11 ++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index e759910..fdc4a99 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1048,7 +1048,9 @@ class Table: self.create_index(column_values.keys(), unique=True) return pk - def m2m(self, other_table, record_or_list=None, pk=None, lookup=None): + def m2m( + self, other_table, record_or_list=None, pk=None, lookup=None, m2m_table=None + ): our_id = self.last_pk if lookup is not None: assert record_or_list is None, "Provide lookup= or record, not both" @@ -1056,9 +1058,8 @@ class Table: assert record_or_list is not None, "Provide lookup= or record, not both" tables = list(sorted([self.name, other_table])) columns = ["{}_id".format(t) for t in tables] - m2m_table = self.db.table( - "{}_{}".format(*tables), pk=columns, foreign_keys=columns - ) + m2m_table_name = m2m_table or "{}_{}".format(*tables) + m2m_table = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) if lookup is None: records = ( [record_or_list] diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 471166e..10a3d50 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -74,9 +74,14 @@ def test_m2m_requires_either_records_or_lookup(fresh_db): people.m2m("tags", {"tag": "hello"}, lookup={"foo": "bar"}) -def test_m2m_explicit_argument(fresh_db): - # .m2m("humans", ..., m2m_table="relationships") - assert False +def test_m2m_explicit_table_name_argument(fresh_db): + people = fresh_db.table("people", pk="id") + people.insert({"name": "Wahyu"}).m2m( + "tags", lookup={"tag": "Coworker"}, m2m_table="tagged" + ) + assert fresh_db["tags"].exists + assert fresh_db["tagged"].exists + assert not fresh_db["people_tags"].exists def test_uses_existing_m2m_table_if_exists(fresh_db): From b9256413d26875c2bc3841e68b90d3842e88ccb8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 3 Aug 2019 21:07:06 +0300 Subject: [PATCH 5/8] db.m2m_table_candidates(table, other_table) --- sqlite_utils/db.py | 11 +++++++++++ tests/test_m2m.py | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index fdc4a99..f52b2e8 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -297,6 +297,17 @@ class Database: ) ) + def m2m_table_candidates(self, table, other_table): + "Returns potential m2m tables for arguments, based on FKs" + candidates = [] + tables = {table, other_table} + for table in self.tables: + # Does it have foreign keys to both table and other_table? + has_fks_to = {fk.other_table for fk in table.foreign_keys} + if has_fks_to.issuperset(tables): + candidates.append(table.name) + return candidates + def add_foreign_keys(self, foreign_keys): # foreign_keys is a list of explicit 4-tuples assert all( diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 10a3d50..5cfe29f 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -84,6 +84,28 @@ def test_m2m_explicit_table_name_argument(fresh_db): assert not fresh_db["people_tags"].exists +def test_m2m_table_candidates(fresh_db): + fresh_db.create_table("one", {"id": int, "name": str}, pk="id") + fresh_db.create_table("two", {"id": int, "name": str}, pk="id") + fresh_db.create_table("three", {"id": int, "name": str}, pk="id") + # No candidates at first + assert [] == fresh_db.m2m_table_candidates("one", "two") + # Create a candidate + fresh_db.create_table( + "one_m2m_two", {"one_id": int, "two_id": int}, foreign_keys=["one_id", "two_id"] + ) + assert ["one_m2m_two"] == fresh_db.m2m_table_candidates("one", "two") + # Add another table and there should be two candidates + fresh_db.create_table( + "one_m2m_two_and_three", + {"one_id": int, "two_id": int, "three_id": int}, + foreign_keys=["one_id", "two_id", "three_id"], + ) + assert {"one_m2m_two", "one_m2m_two_and_three"} == set( + fresh_db.m2m_table_candidates("one", "two") + ) + + def test_uses_existing_m2m_table_if_exists(fresh_db): # Code should look for an existing toble with fks to both tables # and use that if it exists. From d96a8f149ecb4d3fd8a8e5226774b7060c96ec95 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sat, 3 Aug 2019 21:15:16 +0300 Subject: [PATCH 6/8] Use existing m2m table if one exists --- sqlite_utils/db.py | 17 ++++++++++++++++- tests/test_m2m.py | 33 +++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f52b2e8..6695580 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1069,7 +1069,22 @@ class Table: assert record_or_list is not None, "Provide lookup= or record, not both" tables = list(sorted([self.name, other_table])) columns = ["{}_id".format(t) for t in tables] - m2m_table_name = m2m_table or "{}_{}".format(*tables) + if m2m_table is not None: + m2m_table_name = m2m_table + else: + # Detect if there is a single, unambiguous option + candidates = self.db.m2m_table_candidates(self.name, other_table) + if len(candidates) == 1: + m2m_table_name = candidates[0] + elif len(candidates) > 1: + raise NoObviousTable( + "No single obvious m2m table for {}, {} - use m2m_table= parameter".format( + self.name, other_table + ) + ) + else: + # If not, create a new table + m2m_table_name = m2m_table or "{}_{}".format(*tables) m2m_table = self.db.table(m2m_table_name, pk=columns, foreign_keys=columns) if lookup is None: records = ( diff --git a/tests/test_m2m.py b/tests/test_m2m.py index 5cfe29f..cdd7d8c 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -1,4 +1,4 @@ -from sqlite_utils.db import ForeignKey +from sqlite_utils.db import ForeignKey, NoObviousTable import pytest @@ -107,12 +107,37 @@ def test_m2m_table_candidates(fresh_db): def test_uses_existing_m2m_table_if_exists(fresh_db): - # Code should look for an existing toble with fks to both tables + # Code should look for an existing table with fks to both tables # and use that if it exists. - assert False + people = fresh_db.create_table("people", {"id": int, "name": str}, pk="id") + fresh_db["tags"].lookup({"tag": "Coworker"}) + fresh_db.create_table( + "tagged", + {"people_id": int, "tags_id": int}, + foreign_keys=["people_id", "tags_id"], + ) + people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) + assert fresh_db["tags"].exists + assert fresh_db["tagged"].exists + assert not fresh_db["people_tags"].exists + assert not fresh_db["tags_people"].exists + assert [{"people_id": 1, "tags_id": 1}] == list(fresh_db["tagged"].rows) def test_requires_explicit_m2m_table_if_multiple_options(fresh_db): # If the code scans for m2m tables and finds more than one candidate # it should require that the m2m_table=x argument is used - assert False + people = fresh_db.create_table("people", {"id": int, "name": str}, pk="id") + fresh_db["tags"].lookup({"tag": "Coworker"}) + fresh_db.create_table( + "tagged", + {"people_id": int, "tags_id": int}, + foreign_keys=["people_id", "tags_id"], + ) + fresh_db.create_table( + "tagged2", + {"people_id": int, "tags_id": int}, + foreign_keys=["people_id", "tags_id"], + ) + with pytest.raises(NoObviousTable): + people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) From 5516175ca6b9b2d48b7a929ba074b1ef69e981b0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 4 Aug 2019 05:09:17 +0300 Subject: [PATCH 7/8] Allow table objects to be passed to .m2m() --- sqlite_utils/db.py | 24 ++++++++++++++++-------- tests/test_m2m.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 6695580..3fcb133 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -1060,26 +1060,28 @@ class Table: return pk def m2m( - self, other_table, record_or_list=None, pk=None, lookup=None, m2m_table=None + self, other_table, record_or_list=None, pk=DEFAULT, lookup=None, m2m_table=None ): + if isinstance(other_table, str): + other_table = self.db.table(other_table, pk=pk) our_id = self.last_pk if lookup is not None: assert record_or_list is None, "Provide lookup= or record, not both" else: assert record_or_list is not None, "Provide lookup= or record, not both" - tables = list(sorted([self.name, other_table])) + tables = list(sorted([self.name, other_table.name])) columns = ["{}_id".format(t) for t in tables] if m2m_table is not None: m2m_table_name = m2m_table else: # Detect if there is a single, unambiguous option - candidates = self.db.m2m_table_candidates(self.name, other_table) + candidates = self.db.m2m_table_candidates(self.name, other_table.name) if len(candidates) == 1: m2m_table_name = candidates[0] elif len(candidates) > 1: raise NoObviousTable( "No single obvious m2m table for {}, {} - use m2m_table= parameter".format( - self.name, other_table + self.name, other_table.name ) ) else: @@ -1094,14 +1096,20 @@ class Table: ) # Ensure each record exists in other table for record in records: - id = self.db[other_table].upsert(record, pk=pk).last_pk + id = other_table.upsert(record, pk=pk).last_pk m2m_table.upsert( - {"{}_id".format(other_table): id, "{}_id".format(self.name): our_id} + { + "{}_id".format(other_table.name): id, + "{}_id".format(self.name): our_id, + } ) else: - id = self.db[other_table].lookup(lookup) + id = other_table.lookup(lookup) m2m_table.upsert( - {"{}_id".format(other_table): id, "{}_id".format(self.name): our_id} + { + "{}_id".format(other_table.name): id, + "{}_id".format(self.name): our_id, + } ) return self diff --git a/tests/test_m2m.py b/tests/test_m2m.py index cdd7d8c..b5b897b 100644 --- a/tests/test_m2m.py +++ b/tests/test_m2m.py @@ -43,6 +43,19 @@ def test_insert_m2m_list(fresh_db): ] == dogs_humans.foreign_keys +def test_m2m_with_table_objects(fresh_db): + dogs = fresh_db.table("dogs", pk="id") + humans = fresh_db.table("humans", pk="id") + dogs.insert({"id": 1, "name": "Cleo"}).m2m( + humans, [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}] + ) + expected_tables = {"dogs", "humans", "dogs_humans"} + assert expected_tables == set(fresh_db.table_names()) + assert 1 == dogs.count + assert 2 == humans.count + assert 2 == fresh_db["dogs_humans"].count + + def test_m2m_lookup(fresh_db): people = fresh_db.table("people", pk="id") people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) From 243bcaa1acd32a173c07b24dca553991493005a0 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 4 Aug 2019 05:29:19 +0300 Subject: [PATCH 8/8] Documentation for .m2m() table method --- docs/python-api.rst | 89 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/docs/python-api.rst b/docs/python-api.rst index b1ad43d..de3c076 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -442,6 +442,95 @@ To extract the ``species`` column out to a separate ``Species`` table, you can d "species": "Common Juniper" }, extracts={"species": "Species"}) +.. _python_api_m2m: + +Working with many-to-many relationships +======================================= + +``sqlite-utils`` includes a shortcut for creating records using many-to-many relationships in the form of the ``table.m2m(...)`` method. + +Here's how to create two new records and connect them via a many-to-many table in a single line of code: + +.. code-block:: python + + db["dogs"].insert({"id": 1, "name": "Cleo"}, pk="id").m2m( + "humans", {"id": 1, "name": "Natalie"}, pk="id" + ) + +Running this example actually creates three tables: ``dogs``, ``humans`` and a many-to-many ``dogs_humans`` table. It will insert a record into each of those tables. + +The ``.m2m()`` method executes against the last record that was affected by ``.insert()`` or ``.update()`` - the record identified by the ``table.last_pk`` property. To execute ``.m2m()`` against a specific record you can first select it by passing its primary key to ``.update()``: + +.. code-block:: python + + db["dogs"].update(1).m2m( + "humans", {"id": 2, "name": "Simon"}, pk="id" + ) + +The first argument to ``.m2m()`` can be either the name of a table as a string or it can be the table object itself. + +The second argument can be a single dictionary record or a list of dictionaries. Thesee dictionaries will be passed to ``.upsert()`` against the specified table. + +Here's alternative code that creates the dog record and adds two people to it: + +.. code-block:: python + + db = Database(memory=True) + dogs = db.table("dogs", pk="id") + humans = db.table("humans", pk="id") + dogs.insert({"id": 1, "name": "Cleo"}).m2m( + humans, [ + {"id": 1, "name": "Natalie"}, + {"id": 2, "name": "Simon"} + ] + ) + +The method will attempt to find an existing many-to-many table by looking for a table that has foreign key relationships against both of the tables in the relationship. + +If it cannot find such a table, it will create a new one using the names of the two tables - ``dogs_humans`` in this example. You can customize the name of this table using the ``m2m_table=`` argument to ``.m2m()``. + +It it finds multiple candidate tables with foreign keys to both of the specified tables it will raise a ``sqlite_utils.db.NoObviousTable`` exception. You can avoid this error by specifying the correct table using ``m2m_table=``. + +.. _python_api_m2m_lookup: + +Using m2m and lookup tables together +------------------------------------ + +You can work with (or create) lookup tables as part of a call to ``.m2m()`` using the ``lookup=`` parameter. This accepts the same argument as ``table.lookup()`` does - a dictionary of values that should be used to lookup or create a row in the lookup table. + +This example creates a dogs table, populates it, creates a characteristics table, populates that and sets up a many-to-many relationship between the two. It chains ``.m2m()`` twice to create two associated characteristics: + +.. code-block:: python + + db = Database(memory=True) + dogs = db.table("dogs", pk="id") + dogs.insert({"id": 1, "name": "Cleo"}).m2m( + "characteristics", lookup={ + "name": "Playful" + } + ).m2m( + "characteristics", lookup={ + "name": "Opinionated" + } + ) + +You can inspect the database to see the results like this:: + + >>> db.table_names() + ['dogs', 'characteristics', 'characteristics_dogs'] + >>> list(db["dogs"].rows) + [{'id': 1, 'name': 'Cleo'}] + >>> list(db["characteristics"].rows) + [{'id': 1, 'name': 'Playful'}, {'id': 2, 'name': 'Opinionated'}] + >>> list(db["characteristics_dogs"].rows) + [{'characteristics_id': 1, 'dogs_id': 1}, {'characteristics_id': 2, 'dogs_id': 1}] + >>> print(db["characteristics_dogs"].schema) + CREATE TABLE [characteristics_dogs] ( + [characteristics_id] INTEGER REFERENCES [characteristics]([id]), + [dogs_id] INTEGER REFERENCES [dogs]([id]), + PRIMARY KEY ([characteristics_id], [dogs_id]) + ) + .. _python_api_add_column: Adding columns