mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-10-24 18:27:01 +02:00
fix: list-mode insert_all/upsert_all with hash_id misaligns values
When insert_all() or upsert_all() received lists of values plus an auto-generated hash_id, all_columns had the hash column prepended but the raw rows were then paired against that expanded list. Every value shifted one column to the left and the hash_id column ended up holding the first value instead of a key, e.g. rows (1, 'x') became (1, 'x', NULL) under columns (id, a, b). Compute the hash from the user-declared values and prepend it so columns stay aligned.
This commit is contained in:
parent
6bc1d33d58
commit
5e4525a4f2
2 changed files with 56 additions and 3 deletions
|
|
@ -4353,16 +4353,32 @@ class Table(Queryable):
|
|||
if list_mode:
|
||||
# In list mode, records are already lists of values
|
||||
num_columns = len(all_columns)
|
||||
# With hash_id, all_columns has the hash column prepended, but the
|
||||
# records themselves only carry the user-declared columns.
|
||||
user_columns = all_columns[1:] if hash_id else all_columns
|
||||
user_num_columns = len(user_columns)
|
||||
has_extracts = bool(extracts)
|
||||
for record in chunk:
|
||||
# Pad short records with None, truncate long ones
|
||||
record_len = len(record)
|
||||
if record_len < num_columns:
|
||||
if record_len < user_num_columns:
|
||||
record_values = [jsonify_if_needed(v) for v in record] + [None] * (
|
||||
num_columns - record_len
|
||||
user_num_columns - record_len
|
||||
)
|
||||
else:
|
||||
record_values = [jsonify_if_needed(v) for v in record[:num_columns]]
|
||||
record_values = [
|
||||
jsonify_if_needed(v) for v in record[:user_num_columns]
|
||||
]
|
||||
if hash_id:
|
||||
# Compute the hash from the user-declared values so the
|
||||
# hash_id column gets the right key instead of shifting
|
||||
# every value one column over.
|
||||
record_values.insert(
|
||||
0,
|
||||
hash_record(
|
||||
dict(zip(user_columns, record_values)), hash_id_columns
|
||||
),
|
||||
)
|
||||
# Only process extracts if there are any
|
||||
if has_extracts:
|
||||
for i, key in enumerate(all_columns):
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Tests for list-based iteration in insert_all and upsert_all
|
|||
import pytest
|
||||
|
||||
from sqlite_utils import Database
|
||||
from sqlite_utils.utils import hash_record
|
||||
|
||||
|
||||
def test_insert_all_list_mode_basic():
|
||||
|
|
@ -287,3 +288,39 @@ def test_list_mode_single_record_upsert_last_pk():
|
|||
|
||||
# Verify last_pk is populated correctly
|
||||
assert table.last_pk == 1
|
||||
|
||||
|
||||
def test_insert_all_list_mode_with_hash_id_aligns_values():
|
||||
"""hash_id in list mode must hash user values, not shift every column left."""
|
||||
db = Database(memory=True)
|
||||
table = db.table("items")
|
||||
table.insert_all([["a", "b"], [1, "x"], [2, "y"]], hash_id="id")
|
||||
|
||||
rows = list(table.rows)
|
||||
assert '"a" INTEGER' in table.schema
|
||||
assert '"b" TEXT' in table.schema
|
||||
assert rows[0] == {"id": hash_record({"a": 1, "b": "x"}), "a": 1, "b": "x"}
|
||||
assert rows[1] == {"id": hash_record({"a": 2, "b": "y"}), "a": 2, "b": "y"}
|
||||
|
||||
|
||||
def test_insert_all_list_mode_with_hash_id_pads_and_truncates():
|
||||
"""Short rows are padded and long rows truncated before hashing."""
|
||||
db = Database(memory=True)
|
||||
table = db.table("items")
|
||||
table.insert_all([["c1", "c2"], [5], [6, 7, 999]], hash_id="id")
|
||||
|
||||
rows = list(table.rows)
|
||||
assert rows[0] == {"id": hash_record({"c1": 5, "c2": None}), "c1": 5, "c2": None}
|
||||
assert rows[1] == {"id": hash_record({"c1": 6, "c2": 7}), "c1": 6, "c2": 7}
|
||||
|
||||
|
||||
def test_upsert_all_list_mode_with_hash_id():
|
||||
db = Database(memory=True)
|
||||
table = db.table("data")
|
||||
table.upsert_all([["k", "v"], [1, "a"], [1, "b"]], hash_id="id")
|
||||
|
||||
rows = list(table.rows)
|
||||
assert rows[0]["id"] == hash_record({"k": 1, "v": "a"})
|
||||
assert rows[0]["k"] == 1 and rows[0]["v"] == "a"
|
||||
assert rows[1]["id"] == hash_record({"k": 1, "v": "b"})
|
||||
assert rows[1]["k"] == 1 and rows[1]["v"] == "b"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue