From 866a5bc4878953e805fbd1e3081ca018360591d3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 27 Dec 2019 09:15:31 +0000 Subject: [PATCH 1/4] insert --replace and insert(..., replace=True) Refs #66 --- docs/cli.rst | 8 +++---- sqlite_utils/cli.py | 15 ++++++------ sqlite_utils/db.py | 55 +++++++++++++------------------------------- tests/test_cli.py | 18 +++++++-------- tests/test_create.py | 16 ++++++------- 5 files changed, 45 insertions(+), 67 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 10c3345..43592ea 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -265,15 +265,15 @@ For tab-delimited data, use ``--tsv``:: $ sqlite-utils insert dogs.db dogs docs.tsv --tsv -Upserting data -============== +Insert-replacing data +===================== -Upserting works exactly like inserting, with the exception that if your data has a primary key that matches an already exsting record that record will be replaced with the new data. +Insert-replacing works exactly like inserting, with the exception that if your data has a primary key that matches an already exsting record that record will be replaced with the new data. After running the above ``dogs.json`` example, try running this:: $ echo '{"id": 2, "name": "Pancakes", "age": 3}' | \ - sqlite-utils upsert dogs.db dogs - --pk=id + sqlite-utils insert dogs.db dogs - --pk=id --replace This will replace the record for id=2 (Pancakes) with a new record with an updated age. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 0b51517..c5c4f86 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -353,6 +353,7 @@ def insert_upsert_implementation( alter, upsert, ignore=False, + replace=False, not_null=None, default=None, ): @@ -372,17 +373,12 @@ def insert_upsert_implementation( docs = json.load(json_file) if isinstance(docs, dict): docs = [docs] - if upsert: - method = db[table].upsert_all - extra_kwargs = {} - else: - method = db[table].insert_all - extra_kwargs = {"ignore": ignore} + extra_kwargs = {"ignore": ignore, "replace": replace} if not_null: extra_kwargs["not_null"] = set(not_null) if default: extra_kwargs["defaults"] = dict(default) - method(docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs) + db[table].insert_all(docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs) @cli.command() @@ -390,6 +386,9 @@ def insert_upsert_implementation( @click.option( "--ignore", is_flag=True, default=False, help="Ignore records if pk already exists" ) +@click.option( + "--replace", is_flag=True, default=False, help="Replace records if pk already exists" +) def insert( path, table, @@ -401,6 +400,7 @@ def insert( batch_size, alter, ignore, + replace, not_null, default, ): @@ -422,6 +422,7 @@ def insert( alter=alter, upsert=False, ignore=ignore, + replace=replace, not_null=not_null, default=default, ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index d44b238..874aa8b 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -473,11 +473,11 @@ class Table(Queryable): column_order=None, not_null=None, defaults=None, - upsert=False, batch_size=100, hash_id=None, alter=False, ignore=False, + replace=False, extracts=None, ): super().__init__(db, name) @@ -488,11 +488,11 @@ class Table(Queryable): column_order=column_order, not_null=not_null, defaults=defaults, - upsert=upsert, batch_size=batch_size, hash_id=hash_id, alter=alter, ignore=ignore, + replace=replace, extracts=extracts, ) @@ -915,10 +915,10 @@ class Table(Queryable): column_order=DEFAULT, not_null=DEFAULT, defaults=DEFAULT, - upsert=DEFAULT, hash_id=DEFAULT, alter=DEFAULT, ignore=DEFAULT, + replace=DEFAULT, extracts=DEFAULT, ): return self.insert_all( @@ -928,10 +928,10 @@ class Table(Queryable): column_order=column_order, not_null=not_null, defaults=defaults, - upsert=upsert, hash_id=hash_id, alter=alter, ignore=ignore, + replace=replace, extracts=extracts, ) @@ -943,11 +943,11 @@ class Table(Queryable): column_order=DEFAULT, not_null=DEFAULT, defaults=DEFAULT, - upsert=DEFAULT, batch_size=DEFAULT, hash_id=DEFAULT, alter=DEFAULT, ignore=DEFAULT, + replace=DEFAULT, extracts=DEFAULT, ): """ @@ -960,17 +960,17 @@ class Table(Queryable): column_order = self.value_or_default("column_order", column_order) not_null = self.value_or_default("not_null", not_null) defaults = self.value_or_default("defaults", defaults) - upsert = self.value_or_default("upsert", upsert) batch_size = self.value_or_default("batch_size", batch_size) hash_id = self.value_or_default("hash_id", hash_id) alter = self.value_or_default("alter", alter) ignore = self.value_or_default("ignore", ignore) + replace = self.value_or_default("replace", replace) extracts = self.value_or_default("extracts", extracts) assert not (hash_id and pk), "Use either pk= or hash_id=" assert not ( - ignore and upsert - ), "Use either ignore=True or upsert=True, not both" + ignore and replace + ), "Use either ignore=True or replace=True, not both" all_columns = None first = True # We can only handle a max of 999 variables in a SQL insert, so @@ -1009,7 +1009,7 @@ class Table(Queryable): all_columns.insert(0, hash_id) first = False or_what = "" - if upsert: + if replace: or_what = "OR REPLACE " elif ignore: or_what = "OR IGNORE " @@ -1076,18 +1076,7 @@ class Table(Queryable): alter=DEFAULT, extracts=DEFAULT, ): - return self.insert( - record, - pk=pk, - foreign_keys=foreign_keys, - column_order=column_order, - not_null=not_null, - defaults=defaults, - hash_id=hash_id, - alter=alter, - upsert=True, - extracts=extracts, - ) + raise NotImplementedError def upsert_all( self, @@ -1102,19 +1091,7 @@ class Table(Queryable): alter=DEFAULT, extracts=DEFAULT, ): - return self.insert_all( - records, - pk=pk, - foreign_keys=foreign_keys, - column_order=column_order, - not_null=not_null, - defaults=defaults, - batch_size=100, - hash_id=hash_id, - alter=alter, - upsert=True, - extracts=extracts, - ) + raise NotImplementedError def add_missing_columns(self, records): needed_columns = self.detect_column_types(records) @@ -1183,20 +1160,20 @@ class Table(Queryable): ) # Ensure each record exists in other table for record in records: - id = other_table.upsert(record, pk=pk).last_pk - m2m_table.upsert( + id = other_table.insert(record, pk=pk, replace=True).last_pk + m2m_table.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, - } + }, replace=True ) else: id = other_table.lookup(lookup) - m2m_table.upsert( + m2m_table.insert( { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, - } + }, replace=True ) return self diff --git a/tests/test_cli.py b/tests/test_cli.py index d9ebe55..0c3535d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -581,23 +581,23 @@ def test_only_allow_one_of_nl_tsv_csv(options, db_path, tmpdir): assert "Error: Use just one of --nl, --csv or --tsv" == result.output.strip() -def test_upsert(db_path, tmpdir): +def test_insert_replace(db_path, tmpdir): test_insert_multiple_with_primary_key(db_path, tmpdir) - json_path = str(tmpdir / "upsert.json") + json_path = str(tmpdir / "insert-replace.json") db = Database(db_path) assert 20 == db["dogs"].count - upsert_dogs = [ - {"id": 1, "name": "Upserted 1", "age": 4}, - {"id": 2, "name": "Upserted 2", "age": 4}, + insert_replace_dogs = [ + {"id": 1, "name": "Insert replaced 1", "age": 4}, + {"id": 2, "name": "Insert replaced 2", "age": 4}, {"id": 21, "name": "Fresh insert 21", "age": 6}, ] - open(json_path, "w").write(json.dumps(upsert_dogs)) + open(json_path, "w").write(json.dumps(insert_replace_dogs)) result = CliRunner().invoke( - cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"] ) - assert 0 == result.exit_code + assert 0 == result.exit_code, result.output assert 21 == db["dogs"].count - assert upsert_dogs == db.execute_returning_dicts( + assert insert_replace_dogs == db.execute_returning_dicts( "select * from dogs where id in (1, 2, 21) order by id" ) diff --git a/tests/test_create.py b/tests/test_create.py index e2dec84..a7aab07 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -445,7 +445,7 @@ def test_insert_row_alter_table( @pytest.mark.parametrize("use_table_factory", [True, False]) -def test_upsert_rows_alter_table(fresh_db, use_table_factory): +def test_insert_replace_rows_alter_table(fresh_db, use_table_factory): first_row = {"id": 1, "title": "Hedgehogs of the world", "author_id": 1} next_rows = [ {"id": 1, "title": "Hedgehogs of the World", "species": "hedgehogs"}, @@ -459,11 +459,11 @@ def test_upsert_rows_alter_table(fresh_db, use_table_factory): if use_table_factory: table = fresh_db.table("books", pk="id", alter=True) table.insert(first_row) - table.upsert_all(next_rows) + table.insert_all(next_rows, replace=True) else: table = fresh_db["books"] table.insert(first_row, pk="id") - table.upsert_all(next_rows, alter=True) + table.insert_all(next_rows, alter=True, replace=True) assert { "author_id": int, "id": int, @@ -664,11 +664,11 @@ def test_insert_ignore(fresh_db): def test_insert_hash_id(fresh_db): dogs = fresh_db["dogs"] - id = dogs.upsert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk + id = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id assert 1 == dogs.count - # Upserting a second time should not create a new row - id2 = dogs.upsert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk + # Insert replacing a second time should not create a new row + id2 = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id", replace=True).last_pk assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id2 assert 1 == dogs.count @@ -791,10 +791,10 @@ def test_drop_view(fresh_db): assert [] == fresh_db.view_names() -def test_insert_upsert_all_empty_list(fresh_db): +def test_insert_all_empty_list(fresh_db): fresh_db["t"].insert({"foo": 1}) assert 1 == fresh_db["t"].count fresh_db["t"].insert_all([]) assert 1 == fresh_db["t"].count - fresh_db["t"].upsert_all([]) + fresh_db["t"].insert_all([], replace=True) assert 1 == fresh_db["t"].count From 79cc8b854c04c9e4069545d74bd58ef7e9e1c3a4 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 27 Dec 2019 09:30:29 +0000 Subject: [PATCH 2/4] Ran black, plus added comments for next step Refs #66 --- sqlite_utils/cli.py | 9 +++++++-- sqlite_utils/db.py | 40 +++++++++++++++++++++++++++++++++++++--- tests/test_create.py | 4 +++- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c5c4f86..798931d 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -378,7 +378,9 @@ def insert_upsert_implementation( extra_kwargs["not_null"] = set(not_null) if default: extra_kwargs["defaults"] = dict(default) - db[table].insert_all(docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs) + db[table].insert_all( + docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs + ) @cli.command() @@ -387,7 +389,10 @@ def insert_upsert_implementation( "--ignore", is_flag=True, default=False, help="Ignore records if pk already exists" ) @click.option( - "--replace", is_flag=True, default=False, help="Replace records if pk already exists" + "--replace", + is_flag=True, + default=False, + help="Replace records if pk already exists", ) def insert( path, diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 874aa8b..8aa05e3 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -985,6 +985,8 @@ class Table(Queryable): assert ( num_columns <= SQLITE_MAX_VARS ), "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) + # When calculating this for upsert, the primary keys are referenced twice + # so num_columns needs to have num primary keys added to it: batch_size = max(1, min(batch_size, SQLITE_MAX_VARS // num_columns)) for chunk in chunks(itertools.chain([first_record], records), batch_size): chunk = list(chunk) @@ -1008,11 +1010,21 @@ class Table(Queryable): if hash_id: all_columns.insert(0, hash_id) first = False + + # START of bit that differs for upsert v.s. insert or_what = "" if replace: or_what = "OR REPLACE " elif ignore: or_what = "OR IGNORE " + # INSERT OR IGNORE INTO book(id) VALUES(1001); + # UPDATE book SET name = 'Programming' WHERE id = 1001; + # WAIT NO: this won't work because here we create a single + # giant INSERT, but for upsert we need two statements per + # record which means we need executescript()... but + # executescript doesn't take parameters at all. + # So maybe for upsert() we execute two separate SQL queries + # for every single record? Very different implementation. sql = """ INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows}; """.format( @@ -1028,6 +1040,8 @@ class Table(Queryable): for record in chunk ), ) + # END of bit that differs + values = [] extracts = resolve_extracts(extracts) for record in chunk: @@ -1041,6 +1055,9 @@ class Table(Queryable): value = self.db[extract_table].lookup({"value": value}) record_values.append(value) values.extend(record_values) + + # Except... if upsert() needs to execute multiple queries + # then this bit needs to change too with self.db.conn: try: result = self.db.conn.execute(sql, values) @@ -1051,9 +1068,11 @@ class Table(Queryable): result = self.db.conn.execute(sql, values) else: raise + # How do we get lastrowid for upserts? self.last_rowid = result.lastrowid self.last_pk = self.last_rowid # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened + # Do we need this logic for upsert too? if (hash_id or pk) and self.last_rowid: row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] if hash_id: @@ -1076,7 +1095,17 @@ class Table(Queryable): alter=DEFAULT, extracts=DEFAULT, ): - raise NotImplementedError + return self.upsert_all( + [record], + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + hash_id=hash_id, + alter=alter, + extracts=extracts, + ) def upsert_all( self, @@ -1091,6 +1120,9 @@ class Table(Queryable): alter=DEFAULT, extracts=DEFAULT, ): + # Perform the following for each record: + # INSERT OR IGNORE INTO books(id) VALUES(1001); + # UPDATE books SET name = 'Programming' WHERE id = 1001; raise NotImplementedError def add_missing_columns(self, records): @@ -1165,7 +1197,8 @@ class Table(Queryable): { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, - }, replace=True + }, + replace=True, ) else: id = other_table.lookup(lookup) @@ -1173,7 +1206,8 @@ class Table(Queryable): { "{}_id".format(other_table.name): id, "{}_id".format(self.name): our_id, - }, replace=True + }, + replace=True, ) return self diff --git a/tests/test_create.py b/tests/test_create.py index a7aab07..bfa352f 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -668,7 +668,9 @@ def test_insert_hash_id(fresh_db): assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id assert 1 == dogs.count # Insert replacing a second time should not create a new row - id2 = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id", replace=True).last_pk + id2 = dogs.insert( + {"name": "Cleo", "twitter": "cleopaws"}, hash_id="id", replace=True + ).last_pk assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id2 assert 1 == dogs.count From 4a2244b3e3a7e6e0dbb18784831d8b00a19620c3 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Dec 2019 21:03:43 -0800 Subject: [PATCH 3/4] New upsert implementation, refs #66 --- sqlite_utils/cli.py | 2 + sqlite_utils/db.py | 130 +++++++++++++++++++++++++++---------------- tests/test_cli.py | 56 +++++++++++++++++++ tests/test_upsert.py | 16 ++++++ 4 files changed, 156 insertions(+), 48 deletions(-) create mode 100644 tests/test_upsert.py diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 798931d..14e7365 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -378,6 +378,8 @@ def insert_upsert_implementation( extra_kwargs["not_null"] = set(not_null) if default: extra_kwargs["defaults"] = dict(default) + if upsert: + extra_kwargs["upsert"] = upsert db[table].insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 8aa05e3..7516028 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -949,6 +949,7 @@ class Table(Queryable): ignore=DEFAULT, replace=DEFAULT, extracts=DEFAULT, + upsert=False, ): """ Like .insert() but takes a list of records and ensures that the table @@ -985,8 +986,6 @@ class Table(Queryable): assert ( num_columns <= SQLITE_MAX_VARS ), "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS) - # When calculating this for upsert, the primary keys are referenced twice - # so num_columns needs to have num primary keys added to it: batch_size = max(1, min(batch_size, SQLITE_MAX_VARS // num_columns)) for chunk in chunks(itertools.chain([first_record], records), batch_size): chunk = list(chunk) @@ -1011,37 +1010,9 @@ class Table(Queryable): all_columns.insert(0, hash_id) first = False - # START of bit that differs for upsert v.s. insert - or_what = "" - if replace: - or_what = "OR REPLACE " - elif ignore: - or_what = "OR IGNORE " - # INSERT OR IGNORE INTO book(id) VALUES(1001); - # UPDATE book SET name = 'Programming' WHERE id = 1001; - # WAIT NO: this won't work because here we create a single - # giant INSERT, but for upsert we need two statements per - # record which means we need executescript()... but - # executescript doesn't take parameters at all. - # So maybe for upsert() we execute two separate SQL queries - # for every single record? Very different implementation. - sql = """ - INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows}; - """.format( - or_what=or_what, - table=self.name, - columns=", ".join("[{}]".format(c) for c in all_columns), - rows=", ".join( - """ - ({placeholders}) - """.format( - placeholders=", ".join(["?"] * len(all_columns)) - ) - for record in chunk - ), - ) - # END of bit that differs - + # values is the list of insert data that is passed to the + # .execute() method - but some of them may be replaced by + # new primary keys if we are extracting any columns. values = [] extracts = resolve_extracts(extracts) for record in chunk: @@ -1054,25 +1025,76 @@ class Table(Queryable): extract_table = extracts[key] value = self.db[extract_table].lookup({"value": value}) record_values.append(value) - values.extend(record_values) + values.append(record_values) + + queries_and_params = [] + if upsert: + if isinstance(pk, str): + pks = [pk] + else: + pks = pk + for record_values in values: + # TODO: make more efficient: + record = dict(zip(all_columns, record_values)) + params = [] + sql = "INSERT OR IGNORE INTO [{table}]({pks}) VALUES({pk_placeholders});".format( + table=self.name, + pks=", ".join(["[{}]".format(p) for p in pks]), + pk_placeholders=", ".join(["?" for p in pks]), + ) + queries_and_params.append((sql, [record[col] for col in pks])) + # UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001; + set_cols = [col for col in all_columns if col not in pks] + sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format( + table=self.name, + pairs=", ".join("[{}] = ?".format(col) for col in set_cols), + wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks), + ) + queries_and_params.append( + ( + sql2, + [record[col] for col in set_cols] + + [record[pk] for pk in pks], + ) + ) + else: + or_what = "" + if replace: + or_what = "OR REPLACE " + elif ignore: + or_what = "OR IGNORE " + sql = """ + INSERT {or_what}INTO [{table}] ({columns}) VALUES {rows}; + """.format( + or_what=or_what, + table=self.name, + columns=", ".join("[{}]".format(c) for c in all_columns), + rows=", ".join( + """ + ({placeholders}) + """.format( + placeholders=", ".join(["?"] * len(all_columns)) + ) + for record in chunk + ), + ) + flat_values = list(itertools.chain(*values)) + queries_and_params = [(sql, flat_values)] - # Except... if upsert() needs to execute multiple queries - # then this bit needs to change too with self.db.conn: - try: - result = self.db.conn.execute(sql, values) - except OperationalError as e: - if alter and (" column" in e.args[0]): - # Attempt to add any missing columns, then try again - self.add_missing_columns(chunk) - result = self.db.conn.execute(sql, values) - else: - raise - # How do we get lastrowid for upserts? + for query, params in queries_and_params: + try: + result = self.db.conn.execute(query, params) + except OperationalError as e: + if alter and (" column" in e.args[0]): + # Attempt to add any missing columns, then try again + self.add_missing_columns(chunk) + result = self.db.conn.execute(query, params) + else: + raise self.last_rowid = result.lastrowid self.last_pk = self.last_rowid # self.last_rowid will be 0 if a "INSERT OR IGNORE" happened - # Do we need this logic for upsert too? if (hash_id or pk) and self.last_rowid: row = list(self.rows_where("rowid = ?", [self.last_rowid]))[0] if hash_id: @@ -1123,7 +1145,19 @@ class Table(Queryable): # Perform the following for each record: # INSERT OR IGNORE INTO books(id) VALUES(1001); # UPDATE books SET name = 'Programming' WHERE id = 1001; - raise NotImplementedError + return self.insert_all( + records, + pk=pk, + foreign_keys=foreign_keys, + column_order=column_order, + not_null=not_null, + defaults=defaults, + batch_size=batch_size, + hash_id=hash_id, + alter=alter, + extracts=extracts, + upsert=True, + ) def add_missing_columns(self, records): needed_columns = self.detect_column_types(records) diff --git a/tests/test_cli.py b/tests/test_cli.py index 0c3535d..1f24090 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -765,3 +765,59 @@ def test_rows(db_path, args, expected): ) result = CliRunner().invoke(cli.cli, ["rows", db_path, "dogs"] + args) assert expected == result.output.strip() + + +def test_upsert(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + db = Database(db_path) + insert_dogs = [ + {"id": 1, "name": "Cleo", "age": 4}, + {"id": 2, "name": "Nixie", "age": 4}, + ] + open(json_path, "w").write(json.dumps(insert_dogs)) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert 0 == result.exit_code, result.output + assert 2 == db["dogs"].count + # Now run the upsert to update just their ages + upsert_dogs = [ + {"id": 1, "age": 5}, + {"id": 2, "age": 5}, + ] + open(json_path, "w").write(json.dumps(insert_dogs)) + result = CliRunner().invoke( + cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert 0 == result.exit_code, result.output + assert [ + {"id": 1, "name": "Cleo", "age": 4}, + {"id": 2, "name": "Nixie", "age": 4}, + ] == db.execute_returning_dicts("select * from dogs order by id") + + +def test_upsert_alter(db_path, tmpdir): + json_path = str(tmpdir / "dogs.json") + db = Database(db_path) + insert_dogs = [{"id": 1, "name": "Cleo"}] + open(json_path, "w").write(json.dumps(insert_dogs)) + result = CliRunner().invoke( + cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert 0 == result.exit_code, result.output + # Should fail with error code if no --alter + upsert_dogs = [{"id": 1, "age": 5}] + open(json_path, "w").write(json.dumps(upsert_dogs)) + result = CliRunner().invoke( + cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"] + ) + assert 1 == result.exit_code + assert "no such column: age" == str(result.exception) + # Should succeed with --alter + result = CliRunner().invoke( + cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id", "--alter"] + ) + assert 0 == result.exit_code + assert [{"id": 1, "name": "Cleo", "age": 5},] == db.execute_returning_dicts( + "select * from dogs order by id" + ) diff --git a/tests/test_upsert.py b/tests/test_upsert.py new file mode 100644 index 0000000..96f9d0d --- /dev/null +++ b/tests/test_upsert.py @@ -0,0 +1,16 @@ +def test_upsert(fresh_db): + table = fresh_db["table"] + table.insert({"id": 1, "name": "Cleo"}, pk="id") + table.upsert({"id": 1, "age": 5}, pk="id", alter=True) + assert [{"id": 1, "name": "Cleo", "age": 5}] == list(table.rows) + + +def test_upsert_all(fresh_db): + table = fresh_db["table"] + table.upsert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Nixie"}], pk="id") + table.upsert_all([{"id": 1, "age": 5}, {"id": 2, "age": 5}], pk="id", alter=True) + assert [ + {"id": 1, "name": "Cleo", "age": 5}, + {"id": 2, "name": "Nixie", "age": 5}, + ] == list(table.rows) + assert 2 == table.last_pk From ab8a4bda75fc59871ba8445c6a0fb2332483029c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Sun, 29 Dec 2019 21:23:58 -0800 Subject: [PATCH 4/4] Documentation for new upsert v.s insert-replace Refs #66 --- docs/cli.rst | 23 +++++++++++++++++++++++ docs/python-api.rst | 31 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/docs/cli.rst b/docs/cli.rst index 43592ea..557a860 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -265,6 +265,8 @@ For tab-delimited data, use ``--tsv``:: $ sqlite-utils insert dogs.db dogs docs.tsv --tsv +.. _cli_insert_replace: + Insert-replacing data ===================== @@ -277,6 +279,27 @@ After running the above ``dogs.json`` example, try running this:: This will replace the record for id=2 (Pancakes) with a new record with an updated age. +.. _cli_upsert: + +Upserting data +============== + +Upserting is update-or-insert. If a row exists with the specified primary key the provided columns will be updated. If no row exists that row will be created. + +Unlike ``insert --replace``, an upsert will ignore any column values that exist but are not present in the upsert document. + +For example:: + + $ echo '{"id": 2, "age": 4}' | \ + sqlite-utils upsert dogs.db dogs - --pk=id + +This will update the dog with id=2 to have an age of 4, creating a new record (with a null name) if one does not exist. If a row DOES exist the name will be left as-is. + +The command will fail if you reference columns that do not exist on the table. To automatically create missing columns, use the ``--alter`` option. + +.. note:: + ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 `__ for details of this change. + .. _cli_add_column: Adding columns diff --git a/docs/python-api.rst b/docs/python-api.rst index de67c1d..db988fe 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -358,6 +358,30 @@ The function can accept an iterator or generator of rows and will commit them ac You can skip inserting any records that have a primary key that already exists using ``ignore=True``. This works with both ``.insert({...}, ignore=True)`` and ``.insert_all([...], ignore=True)``. +.. _python_api_insert_replace: + +Insert-replacing data +===================== + +If you want to insert a record or replace an existing record with the same primary key, using the ``replace=True`` argument to ``.insert()`` or ``.insert_all()``:: + + dogs.insert_all([{ + "id": 1, + "name": "Cleo", + "twitter": "cleopaws", + "age": 3, + "is_good_dog": True, + }, { + "id": 2, + "name": "Marnie", + "twitter": "MarnieTheDog", + "age": 16, + "is_good_dog": True, + }], pk="id", replace=True) + +.. note:: + Prior to sqlite-utils 2.x the ``.upsert()`` and ``.upsert_all()`` methods did this. See :ref:`python_api_upsert` for the new behaviour of those methods in 2.x. + .. _python_api_update: Updating a specific record @@ -409,6 +433,8 @@ You can delete all records in a table that match a specific WHERE statement usin Calling ``table.delete_where()`` with no other arguments will delete every row in the table. +.. _python_api_upsert: + Upserting data ============== @@ -428,10 +454,15 @@ For example, given the dogs database you could upsert the record for Cleo like s If a record exists with id=1, it will be updated to match those fields. If it does not exist it will be created. +Any existing columns that are not referenced in the dictionary passed to ``.upsert()`` will be unchanged. If you want to replace a record entirely, use ``.insert(doc, replace=True)`` instead. + Note that the ``pk`` and ``column_order`` parameters here are optional if you are certain that the table has already been created. You should pass them if the table may not exist at the time the first upsert is performed. An ``upsert_all()`` method is also available, which behaves like ``insert_all()`` but performs upserts instead. +.. note:: + ``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 `__ for details of this change. + .. _python_api_lookup_tables: Working with lookup tables