.extract() and .lookup() no longer extract null values, closes #186

- table.extract() and the sqlite-utils extract command now skip rows
  where every extracted column is null: the new foreign key column is
  left null and no all-null record is added to the lookup table. Rows
  with at least one non-null extracted column are extracted as before.
- The extracts= option to insert() and friends keeps None values as
  null instead of creating a lookup record for them - previously each
  insert batch added a duplicate null row to the lookup table.
- table.lookup() compares values using IS rather than = so lookup
  values containing None match existing rows correctly, instead of
  inserting a duplicate row on every call.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Simon Willison 2026-07-06 20:39:29 -07:00
commit 2616dec795
8 changed files with 176 additions and 7 deletions

View file

@ -157,3 +157,27 @@ def test_lookup_with_extra_insert_parameters(fresh_db):
def test_lookup_new_table_strict(fresh_db, strict):
fresh_db["species"].lookup({"name": "Palm"}, strict=strict)
assert fresh_db["species"].strict == strict or not fresh_db.supports_strict
def test_lookup_null_value_idempotent(fresh_db):
# https://github.com/simonw/sqlite-utils/issues/186
# Repeated lookups of a null value should return the same row,
# not insert a duplicate row each time
species = fresh_db["species"]
first_id = species.lookup({"name": None})
second_id = species.lookup({"name": None})
assert first_id == second_id
assert list(species.rows) == [{"id": first_id, "name": None}]
def test_lookup_compound_key_with_null_idempotent(fresh_db):
species = fresh_db["species"]
palm_id = species.lookup({"name": "Palm", "type": None})
oak_id = species.lookup({"name": "Oak", "type": "Tree"})
assert palm_id == species.lookup({"name": "Palm", "type": None})
assert oak_id == species.lookup({"name": "Oak", "type": "Tree"})
assert palm_id != oak_id
assert list(species.rows) == [
{"id": palm_id, "name": "Palm", "type": None},
{"id": oak_id, "name": "Oak", "type": "Tree"},
]