mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-07-23 09:24:31 +02:00
Preserve duplicate column names in query results
Queries returning duplicate column names - e.g. joins between tables sharing column names - silently lost values because rows were built with dict(zip(keys, row)), where the last duplicate wins. Later occurrences are now renamed with a numeric suffix: id, id becomes id, id_2 - skipping any suffix that would collide with a real column in the same query. The new utils.dedupe_keys() helper transforms the key list once per query, so the per-row dict construction is unchanged and there is no measurable performance impact. Applied in Database.query() (including the PRAGMA and RETURNING paths), Table.rows_where(), Table.search() and the CLI's JSON output. CSV, TSV and table output keep the original duplicate headers. Closes #624
This commit is contained in:
parent
a00ed60efc
commit
07b603e562
10 changed files with 128 additions and 4 deletions
|
|
@ -613,6 +613,37 @@ def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) ->
|
|||
).hexdigest()
|
||||
|
||||
|
||||
def dedupe_keys(keys: Iterable[str]) -> List[str]:
|
||||
"""
|
||||
Rename duplicates in a list of column names so every name is unique,
|
||||
by appending ``_2``, ``_3``... to later occurrences - skipping any
|
||||
suffix that would collide with another column in the list.
|
||||
|
||||
Used when converting SQL query rows to dictionaries, where duplicate
|
||||
column names would otherwise silently overwrite each other.
|
||||
|
||||
:param keys: List of column names, possibly containing duplicates
|
||||
"""
|
||||
keys = list(keys)
|
||||
taken = set(keys)
|
||||
if len(taken) == len(keys):
|
||||
# No duplicates - the common case
|
||||
return keys
|
||||
seen: set = set()
|
||||
result = []
|
||||
for key in keys:
|
||||
if key in seen:
|
||||
new_key = key
|
||||
suffix = 2
|
||||
while new_key in seen or new_key in taken:
|
||||
new_key = "{}_{}".format(key, suffix)
|
||||
suffix += 1
|
||||
key = new_key
|
||||
seen.add(key)
|
||||
result.append(key)
|
||||
return result
|
||||
|
||||
|
||||
def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]:
|
||||
for key, value in d.items():
|
||||
if isinstance(value, dict):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue