Preserve duplicate column names in query results

Queries returning duplicate column names - e.g. joins between tables
sharing column names - silently lost values because rows were built
with dict(zip(keys, row)), where the last duplicate wins.

Later occurrences are now renamed with a numeric suffix: id, id
becomes id, id_2 - skipping any suffix that would collide with a
real column in the same query.

The new utils.dedupe_keys() helper transforms the key list once per
query, so the per-row dict construction is unchanged and there is no
measurable performance impact.

Applied in Database.query() (including the PRAGMA and RETURNING
paths), Table.rows_where(), Table.search() and the CLI's JSON output.
CSV, TSV and table output keep the original duplicate headers.

Closes #624

sqlite_utils/cli.py

@@ -34,6 +34,7 @@ from .utils import (
     OperationalError,
     _compile_code,
     chunks,
+    dedupe_keys,
     file_progress,
     find_spatialite,
     flatten as _flatten,
@@ -3493,6 +3494,10 @@ FILE_COLUMNS = {
 
 
 def output_rows(iterator, headers, nl, arrays, json_cols):
+    # Duplicate column names would collide as dictionary keys, so rename
+    # later occurrences id, id -> id, id_2 - CSV and table output keep
+    # the original duplicate headers since they never build dictionaries
+    headers = dedupe_keys(headers)
     # We have to iterate two-at-a-time so we can know if we
     # should output a trailing comma or if we have reached
     # the last row.

sqlite_utils/db.py

@@ -1,5 +1,6 @@
 from .utils import (
     chunks,
+    dedupe_keys,
     hash_record,
     sqlite3,
     OperationalError,
@@ -786,7 +787,7 @@ class Database:
                     cursor = self.conn.execute(sql, *args)
             if cursor.description is None:
                 raise ValueError(message)
-            keys = [d[0] for d in cursor.description]
+            keys = dedupe_keys(d[0] for d in cursor.description)
             return (dict(zip(keys, row)) for row in cursor)
         # Execute inside a savepoint, so a statement that turns out not to
         # return rows can be rolled back before the ValueError is raised
@@ -796,7 +797,7 @@ class Database:
             cursor = self.conn.execute(sql, *args)
             if cursor.description is None:
                 raise ValueError(message)
-            keys = [d[0] for d in cursor.description]
+            keys = dedupe_keys(d[0] for d in cursor.description)
             try:
                 self.conn.execute('RELEASE "sqlite_utils_query"')
                 released = True
@@ -1865,7 +1866,7 @@ class Queryable:
         if offset is not None:
             sql += " offset {}".format(offset)
         cursor = self.db.execute(sql, where_args or [])
-        columns = [c[0] for c in cursor.description]
+        columns = dedupe_keys(c[0] for c in cursor.description)
         for row in cursor:
             yield dict(zip(columns, row))
 
@@ -3398,7 +3399,7 @@ class Table(Queryable):
             ),
             args,
         )
-        columns = [c[0] for c in cursor.description]
+        columns = dedupe_keys(c[0] for c in cursor.description)
         for row in cursor:
             yield dict(zip(columns, row))
 

sqlite_utils/utils.py

@@ -613,6 +613,37 @@ def hash_record(record: Dict[str, Any], keys: Optional[Iterable[str]] = None) ->
     ).hexdigest()
 
 
+def dedupe_keys(keys: Iterable[str]) -> List[str]:
+    """
+    Rename duplicates in a list of column names so every name is unique,
+    by appending ``_2``, ``_3``... to later occurrences - skipping any
+    suffix that would collide with another column in the list.
+
+    Used when converting SQL query rows to dictionaries, where duplicate
+    column names would otherwise silently overwrite each other.
+
+    :param keys: List of column names, possibly containing duplicates
+    """
+    keys = list(keys)
+    taken = set(keys)
+    if len(taken) == len(keys):
+        # No duplicates - the common case
+        return keys
+    seen: set = set()
+    result = []
+    for key in keys:
+        if key in seen:
+            new_key = key
+            suffix = 2
+            while new_key in seen or new_key in taken:
+                new_key = "{}_{}".format(key, suffix)
+                suffix += 1
+            key = new_key
+        seen.add(key)
+        result.append(key)
+    return result
+
+
 def _flatten(d: Dict[str, Any]) -> Generator[Tuple[str, Any], None, None]:
     for key, value in d.items():
         if isinstance(value, dict):