Documentation for new upsert v.s insert-replace

Refs #66

docs/cli.rst

@@ -265,18 +265,41 @@ For tab-delimited data, use ``--tsv``::
 
     $ sqlite-utils insert dogs.db dogs docs.tsv --tsv
 
-Upserting data
+.. _cli_insert_replace:
-==============
 
-Upserting works exactly like inserting, with the exception that if your data has a primary key that matches an already exsting record that record will be replaced with the new data.
+Insert-replacing data
+=====================
+
+Insert-replacing works exactly like inserting, with the exception that if your data has a primary key that matches an already exsting record that record will be replaced with the new data.
 
 After running the above ``dogs.json`` example, try running this::
 
     $ echo '{"id": 2, "name": "Pancakes", "age": 3}' | \
-        sqlite-utils upsert dogs.db dogs - --pk=id
+        sqlite-utils insert dogs.db dogs - --pk=id --replace
 
 This will replace the record for id=2 (Pancakes) with a new record with an updated age.
 
+.. _cli_upsert:
+
+Upserting data
+==============
+
+Upserting is update-or-insert. If a row exists with the specified primary key the provided columns will be updated. If no row exists that row will be created.
+
+Unlike ``insert --replace``, an upsert will ignore any column values that exist but are not present in the upsert document.
+
+For example::
+
+    $ echo '{"id": 2, "age": 4}' | \
+        sqlite-utils upsert dogs.db dogs - --pk=id
+
+This will update the dog with id=2 to have an age of 4, creating a new record (with a null name) if one does not exist. If a row DOES exist the name will be left as-is.
+
+The command will fail if you reference columns that do not exist on the table. To automatically create missing columns, use the ``--alter`` option.
+
+.. note::
+    ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 <https://github.com/simonw/sqlite-utils/issues/66>`__ for details of this change.
+
 .. _cli_add_column:
 
 Adding columns

docs/python-api.rst

@@ -358,6 +358,30 @@ The function can accept an iterator or generator of rows and will commit them ac
 
 You can skip inserting any records that have a primary key that already exists using ``ignore=True``. This works with both ``.insert({...}, ignore=True)`` and ``.insert_all([...], ignore=True)``.
 
+.. _python_api_insert_replace:
+
+Insert-replacing data
+=====================
+
+If you want to insert a record or replace an existing record with the same primary key, using the ``replace=True`` argument to ``.insert()`` or ``.insert_all()``::
+
+    dogs.insert_all([{
+        "id": 1,
+        "name": "Cleo",
+        "twitter": "cleopaws",
+        "age": 3,
+        "is_good_dog": True,
+    }, {
+        "id": 2,
+        "name": "Marnie",
+        "twitter": "MarnieTheDog",
+        "age": 16,
+        "is_good_dog": True,
+    }], pk="id", replace=True)
+
+.. note::
+    Prior to sqlite-utils 2.x the ``.upsert()`` and ``.upsert_all()`` methods did this. See :ref:`python_api_upsert` for the new behaviour of those methods in 2.x.
+
 .. _python_api_update:
 
 Updating a specific record
@@ -409,6 +433,8 @@ You can delete all records in a table that match a specific WHERE statement usin
 
 Calling ``table.delete_where()`` with no other arguments will delete every row in the table.
 
+.. _python_api_upsert:
+
 Upserting data
 ==============
 
@@ -428,10 +454,15 @@ For example, given the dogs database you could upsert the record for Cleo like s
 
 If a record exists with id=1, it will be updated to match those fields. If it does not exist it will be created.
 
+Any existing columns that are not referenced in the dictionary passed to ``.upsert()`` will be unchanged. If you want to replace a record entirely, use ``.insert(doc, replace=True)`` instead.
+
 Note that the ``pk`` and ``column_order`` parameters here are optional if you are certain that the table has already been created. You should pass them if the table may not exist at the time the first upsert is performed.
 
 An ``upsert_all()`` method is also available, which behaves like ``insert_all()`` but performs upserts instead.
 
+.. note::
+    ``.upsert()`` and ``.upsert_all()`` in sqlite-utils 1.x worked like ``.insert(..., replace=True)`` and ``.insert_all(..., replace=True)`` do in 2.x. See `issue #66 <https://github.com/simonw/sqlite-utils/issues/66>`__ for details of this change.
+
 .. _python_api_lookup_tables:
 
 Working with lookup tables

sqlite_utils/cli.py

@@ -353,6 +353,7 @@ def insert_upsert_implementation(
     alter,
     upsert,
     ignore=False,
+    replace=False,
     not_null=None,
     default=None,
 ):
@@ -372,17 +373,16 @@ def insert_upsert_implementation(
         docs = json.load(json_file)
         if isinstance(docs, dict):
             docs = [docs]
-    if upsert:
+    extra_kwargs = {"ignore": ignore, "replace": replace}
-        method = db[table].upsert_all
-        extra_kwargs = {}
-    else:
-        method = db[table].insert_all
-        extra_kwargs = {"ignore": ignore}
     if not_null:
         extra_kwargs["not_null"] = set(not_null)
     if default:
         extra_kwargs["defaults"] = dict(default)
-    method(docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs)
+    if upsert:
+        extra_kwargs["upsert"] = upsert
+    db[table].insert_all(
+        docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs
+    )
 
 
 @cli.command()
@@ -390,6 +390,12 @@ def insert_upsert_implementation(
 @click.option(
     "--ignore", is_flag=True, default=False, help="Ignore records if pk already exists"
 )
+@click.option(
+    "--replace",
+    is_flag=True,
+    default=False,
+    help="Replace records if pk already exists",
+)
 def insert(
     path,
     table,
@@ -401,6 +407,7 @@ def insert(
     batch_size,
     alter,
     ignore,
+    replace,
     not_null,
     default,
 ):
@@ -422,6 +429,7 @@ def insert(
         alter=alter,
         upsert=False,
         ignore=ignore,
+        replace=replace,
         not_null=not_null,
         default=default,
     )

sqlite_utils/db.py

@@ -473,11 +473,11 @@ class Table(Queryable):
         column_order=None,
         not_null=None,
         defaults=None,
-        upsert=False,
         batch_size=100,
         hash_id=None,
         alter=False,
         ignore=False,
+        replace=False,
         extracts=None,
     ):
         super().__init__(db, name)
@@ -488,11 +488,11 @@ class Table(Queryable):
             column_order=column_order,
             not_null=not_null,
             defaults=defaults,
-            upsert=upsert,
             batch_size=batch_size,
             hash_id=hash_id,
             alter=alter,
             ignore=ignore,
+            replace=replace,
             extracts=extracts,
         )
 
@@ -915,10 +915,10 @@ class Table(Queryable):
         column_order=DEFAULT,
         not_null=DEFAULT,
         defaults=DEFAULT,
-        upsert=DEFAULT,
         hash_id=DEFAULT,
         alter=DEFAULT,
         ignore=DEFAULT,
+        replace=DEFAULT,
         extracts=DEFAULT,
     ):
         return self.insert_all(
@@ -928,10 +928,10 @@ class Table(Queryable):
             column_order=column_order,
             not_null=not_null,
             defaults=defaults,
-            upsert=upsert,
             hash_id=hash_id,
             alter=alter,
             ignore=ignore,
+            replace=replace,
             extracts=extracts,
         )
 
@@ -943,12 +943,13 @@ class Table(Queryable):
         column_order=DEFAULT,
         not_null=DEFAULT,
         defaults=DEFAULT,
-        upsert=DEFAULT,
         batch_size=DEFAULT,
         hash_id=DEFAULT,
         alter=DEFAULT,
         ignore=DEFAULT,
+        replace=DEFAULT,
         extracts=DEFAULT,
+        upsert=False,
     ):
         """
         Like .insert() but takes a list of records and ensures that the table
@@ -960,17 +961,17 @@ class Table(Queryable):
         column_order = self.value_or_default("column_order", column_order)
         not_null = self.value_or_default("not_null", not_null)
         defaults = self.value_or_default("defaults", defaults)
-        upsert = self.value_or_default("upsert", upsert)
         batch_size = self.value_or_default("batch_size", batch_size)
         hash_id = self.value_or_default("hash_id", hash_id)
         alter = self.value_or_default("alter", alter)
         ignore = self.value_or_default("ignore", ignore)
+        replace = self.value_or_default("replace", replace)
         extracts = self.value_or_default("extracts", extracts)
 
         assert not (hash_id and pk), "Use either pk= or hash_id="
         assert not (
-            ignore and upsert
+            ignore and replace
-        ), "Use either ignore=True or upsert=True, not both"
+        ), "Use either ignore=True or replace=True, not both"
         all_columns = None
         first = True
         # We can only handle a max of 999 variables in a SQL insert, so
@@ -1008,8 +1009,57 @@ class Table(Queryable):
                 if hash_id:
                     all_columns.insert(0, hash_id)
             first = False
-            or_what = ""
+
+            # values is the list of insert data that is passed to the
+            # .execute() method - but some of them may be replaced by
+            # new primary keys if we are extracting any columns.
+            values = []
+            extracts = resolve_extracts(extracts)
+            for record in chunk:
+                record_values = []
+                for key in all_columns:
+                    value = jsonify_if_needed(
+                        record.get(key, None if key != hash_id else _hash(record))
+                    )
+                    if key in extracts:
+                        extract_table = extracts[key]
+                        value = self.db[extract_table].lookup({"value": value})
+                    record_values.append(value)
+                values.append(record_values)
+
+            queries_and_params = []
             if upsert:
+                if isinstance(pk, str):
+                    pks = [pk]
+                else:
+                    pks = pk
+                for record_values in values:
+                    # TODO: make more efficient:
+                    record = dict(zip(all_columns, record_values))
+                    params = []
+                    sql = "INSERT OR IGNORE INTO [{table}]({pks}) VALUES({pk_placeholders});".format(
+                        table=self.name,
+                        pks=", ".join(["[{}]".format(p) for p in pks]),
+                        pk_placeholders=", ".join(["?" for p in pks]),
+                    )
+                    queries_and_params.append((sql, [record[col] for col in pks]))
+                    # UPDATE [book] SET [name] = 'Programming' WHERE [id] = 1001;
+                    set_cols = [col for col in all_columns if col not in pks]
+                    sql2 = "UPDATE [{table}] SET {pairs} WHERE {wheres}".format(
+                        table=self.name,
+                        pairs=", ".join("[{}] = ?".format(col) for col in set_cols),
+                        wheres=" AND ".join("[{}] = ?".format(pk) for pk in pks),
+                    )
+                    queries_and_params.append(
+                        (
+                            sql2,
+                            [record[col] for col in set_cols]
+                            + [record[pk] for pk in pks],
+                        )
+                    )
+            else:
+                or_what = ""
+                if replace:
                     or_what = "OR REPLACE "
                 elif ignore:
                     or_what = "OR IGNORE "
@@ -1028,27 +1078,18 @@ class Table(Queryable):
                         for record in chunk
                     ),
                 )
-            values = []
+                flat_values = list(itertools.chain(*values))
-            extracts = resolve_extracts(extracts)
+                queries_and_params = [(sql, flat_values)]
-            for record in chunk:
+
-                record_values = []
-                for key in all_columns:
-                    value = jsonify_if_needed(
-                        record.get(key, None if key != hash_id else _hash(record))
-                    )
-                    if key in extracts:
-                        extract_table = extracts[key]
-                        value = self.db[extract_table].lookup({"value": value})
-                    record_values.append(value)
-                values.extend(record_values)
             with self.db.conn:
+                for query, params in queries_and_params:
                     try:
-                    result = self.db.conn.execute(sql, values)
+                        result = self.db.conn.execute(query, params)
                     except OperationalError as e:
                         if alter and (" column" in e.args[0]):
                             # Attempt to add any missing columns, then try again
                             self.add_missing_columns(chunk)
-                        result = self.db.conn.execute(sql, values)
+                            result = self.db.conn.execute(query, params)
                         else:
                             raise
                 self.last_rowid = result.lastrowid
@@ -1076,8 +1117,8 @@ class Table(Queryable):
         alter=DEFAULT,
         extracts=DEFAULT,
     ):
-        return self.insert(
+        return self.upsert_all(
-            record,
+            [record],
             pk=pk,
             foreign_keys=foreign_keys,
             column_order=column_order,
@@ -1085,7 +1126,6 @@ class Table(Queryable):
             defaults=defaults,
             hash_id=hash_id,
             alter=alter,
-            upsert=True,
             extracts=extracts,
         )
 
@@ -1102,6 +1142,9 @@ class Table(Queryable):
         alter=DEFAULT,
         extracts=DEFAULT,
     ):
+        # Perform the following for each record:
+        # INSERT OR IGNORE INTO books(id) VALUES(1001);
+        # UPDATE books SET name = 'Programming' WHERE id = 1001;
         return self.insert_all(
             records,
             pk=pk,
@@ -1109,11 +1152,11 @@ class Table(Queryable):
             column_order=column_order,
             not_null=not_null,
             defaults=defaults,
-            batch_size=100,
+            batch_size=batch_size,
             hash_id=hash_id,
             alter=alter,
-            upsert=True,
             extracts=extracts,
+            upsert=True,
         )
 
     def add_missing_columns(self, records):
@@ -1183,20 +1226,22 @@ class Table(Queryable):
             )
             # Ensure each record exists in other table
             for record in records:
-                id = other_table.upsert(record, pk=pk).last_pk
+                id = other_table.insert(record, pk=pk, replace=True).last_pk
-                m2m_table.upsert(
+                m2m_table.insert(
                     {
                         "{}_id".format(other_table.name): id,
                         "{}_id".format(self.name): our_id,
-                    }
+                    },
+                    replace=True,
                 )
         else:
             id = other_table.lookup(lookup)
-            m2m_table.upsert(
+            m2m_table.insert(
                 {
                     "{}_id".format(other_table.name): id,
                     "{}_id".format(self.name): our_id,
-                }
+                },
+                replace=True,
             )
         return self
 

tests/test_cli.py

@@ -581,23 +581,23 @@ def test_only_allow_one_of_nl_tsv_csv(options, db_path, tmpdir):
     assert "Error: Use just one of --nl, --csv or --tsv" == result.output.strip()
 
 
-def test_upsert(db_path, tmpdir):
+def test_insert_replace(db_path, tmpdir):
     test_insert_multiple_with_primary_key(db_path, tmpdir)
-    json_path = str(tmpdir / "upsert.json")
+    json_path = str(tmpdir / "insert-replace.json")
     db = Database(db_path)
     assert 20 == db["dogs"].count
-    upsert_dogs = [
+    insert_replace_dogs = [
-        {"id": 1, "name": "Upserted 1", "age": 4},
+        {"id": 1, "name": "Insert replaced 1", "age": 4},
-        {"id": 2, "name": "Upserted 2", "age": 4},
+        {"id": 2, "name": "Insert replaced 2", "age": 4},
         {"id": 21, "name": "Fresh insert 21", "age": 6},
     ]
-    open(json_path, "w").write(json.dumps(upsert_dogs))
+    open(json_path, "w").write(json.dumps(insert_replace_dogs))
     result = CliRunner().invoke(
-        cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"]
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id", "--replace"]
     )
-    assert 0 == result.exit_code
+    assert 0 == result.exit_code, result.output
     assert 21 == db["dogs"].count
-    assert upsert_dogs == db.execute_returning_dicts(
+    assert insert_replace_dogs == db.execute_returning_dicts(
         "select * from dogs where id in (1, 2, 21) order by id"
     )
 
@@ -765,3 +765,59 @@ def test_rows(db_path, args, expected):
         )
     result = CliRunner().invoke(cli.cli, ["rows", db_path, "dogs"] + args)
     assert expected == result.output.strip()
+
+
+def test_upsert(db_path, tmpdir):
+    json_path = str(tmpdir / "dogs.json")
+    db = Database(db_path)
+    insert_dogs = [
+        {"id": 1, "name": "Cleo", "age": 4},
+        {"id": 2, "name": "Nixie", "age": 4},
+    ]
+    open(json_path, "w").write(json.dumps(insert_dogs))
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"]
+    )
+    assert 0 == result.exit_code, result.output
+    assert 2 == db["dogs"].count
+    # Now run the upsert to update just their ages
+    upsert_dogs = [
+        {"id": 1, "age": 5},
+        {"id": 2, "age": 5},
+    ]
+    open(json_path, "w").write(json.dumps(insert_dogs))
+    result = CliRunner().invoke(
+        cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"]
+    )
+    assert 0 == result.exit_code, result.output
+    assert [
+        {"id": 1, "name": "Cleo", "age": 4},
+        {"id": 2, "name": "Nixie", "age": 4},
+    ] == db.execute_returning_dicts("select * from dogs order by id")
+
+
+def test_upsert_alter(db_path, tmpdir):
+    json_path = str(tmpdir / "dogs.json")
+    db = Database(db_path)
+    insert_dogs = [{"id": 1, "name": "Cleo"}]
+    open(json_path, "w").write(json.dumps(insert_dogs))
+    result = CliRunner().invoke(
+        cli.cli, ["insert", db_path, "dogs", json_path, "--pk", "id"]
+    )
+    assert 0 == result.exit_code, result.output
+    # Should fail with error code if no --alter
+    upsert_dogs = [{"id": 1, "age": 5}]
+    open(json_path, "w").write(json.dumps(upsert_dogs))
+    result = CliRunner().invoke(
+        cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id"]
+    )
+    assert 1 == result.exit_code
+    assert "no such column: age" == str(result.exception)
+    # Should succeed with --alter
+    result = CliRunner().invoke(
+        cli.cli, ["upsert", db_path, "dogs", json_path, "--pk", "id", "--alter"]
+    )
+    assert 0 == result.exit_code
+    assert [{"id": 1, "name": "Cleo", "age": 5},] == db.execute_returning_dicts(
+        "select * from dogs order by id"
+    )

tests/test_create.py

@@ -445,7 +445,7 @@ def test_insert_row_alter_table(
 
 
 @pytest.mark.parametrize("use_table_factory", [True, False])
-def test_upsert_rows_alter_table(fresh_db, use_table_factory):
+def test_insert_replace_rows_alter_table(fresh_db, use_table_factory):
     first_row = {"id": 1, "title": "Hedgehogs of the world", "author_id": 1}
     next_rows = [
         {"id": 1, "title": "Hedgehogs of the World", "species": "hedgehogs"},
@@ -459,11 +459,11 @@ def test_upsert_rows_alter_table(fresh_db, use_table_factory):
     if use_table_factory:
         table = fresh_db.table("books", pk="id", alter=True)
         table.insert(first_row)
-        table.upsert_all(next_rows)
+        table.insert_all(next_rows, replace=True)
     else:
         table = fresh_db["books"]
         table.insert(first_row, pk="id")
-        table.upsert_all(next_rows, alter=True)
+        table.insert_all(next_rows, alter=True, replace=True)
     assert {
         "author_id": int,
         "id": int,
@@ -664,11 +664,13 @@ def test_insert_ignore(fresh_db):
 
 def test_insert_hash_id(fresh_db):
     dogs = fresh_db["dogs"]
-    id = dogs.upsert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk
+    id = dogs.insert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk
     assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id
     assert 1 == dogs.count
-    # Upserting a second time should not create a new row
+    # Insert replacing a second time should not create a new row
-    id2 = dogs.upsert({"name": "Cleo", "twitter": "cleopaws"}, hash_id="id").last_pk
+    id2 = dogs.insert(
+        {"name": "Cleo", "twitter": "cleopaws"}, hash_id="id", replace=True
+    ).last_pk
     assert "f501265970505d9825d8d9f590bfab3519fb20b1" == id2
     assert 1 == dogs.count
 
@@ -791,10 +793,10 @@ def test_drop_view(fresh_db):
     assert [] == fresh_db.view_names()
 
 
-def test_insert_upsert_all_empty_list(fresh_db):
+def test_insert_all_empty_list(fresh_db):
     fresh_db["t"].insert({"foo": 1})
     assert 1 == fresh_db["t"].count
     fresh_db["t"].insert_all([])
     assert 1 == fresh_db["t"].count
-    fresh_db["t"].upsert_all([])
+    fresh_db["t"].insert_all([], replace=True)
     assert 1 == fresh_db["t"].count

tests/test_upsert.py

@@ -0,0 +1,16 @@
+def test_upsert(fresh_db):
+    table = fresh_db["table"]
+    table.insert({"id": 1, "name": "Cleo"}, pk="id")
+    table.upsert({"id": 1, "age": 5}, pk="id", alter=True)
+    assert [{"id": 1, "name": "Cleo", "age": 5}] == list(table.rows)
+
+
+def test_upsert_all(fresh_db):
+    table = fresh_db["table"]
+    table.upsert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Nixie"}], pk="id")
+    table.upsert_all([{"id": 1, "age": 5}, {"id": 2, "age": 5}], pk="id", alter=True)
+    assert [
+        {"id": 1, "name": "Cleo", "age": 5},
+        {"id": 2, "name": "Nixie", "age": 5},
+    ] == list(table.rows)
+    assert 2 == table.last_pk