alter=True/--alter option to automatically add missing columns

Closes #18

sqlite_utils/cli.py

@@ -262,6 +262,11 @@ def insert_upsert_options(fn):
             click.option(
                 "--batch-size", type=int, default=100, help="Commit every X records"
             ),
+            click.option(
+                "--alter",
+                is_flag=True,
+                help="Alter existing table to add any missing columns",
+            ),
         )
     ):
         fn = decorator(fn)
@@ -269,7 +274,7 @@ def insert_upsert_options(fn):
 
 
 def insert_upsert_implementation(
-    path, table, json_file, pk, nl, csv, batch_size, upsert
+    path, table, json_file, pk, nl, csv, batch_size, alter, upsert
 ):
     db = sqlite_utils.Database(path)
     if nl and csv:
@@ -289,12 +294,12 @@ def insert_upsert_implementation(
         method = db[table].upsert_all
     else:
         method = db[table].insert_all
-    method(docs, pk=pk, batch_size=batch_size)
+    method(docs, pk=pk, batch_size=batch_size, alter=alter)
 
 
 @cli.command()
 @insert_upsert_options
-def insert(path, table, json_file, pk, nl, csv, batch_size):
+def insert(path, table, json_file, pk, nl, csv, batch_size, alter):
     """
     Insert records from JSON file into a table, creating the table if it
     does not already exist.
@@ -302,20 +307,20 @@ def insert(path, table, json_file, pk, nl, csv, batch_size):
     Input should be a JSON array of objects, unless --nl or --csv is used.
     """
     insert_upsert_implementation(
-        path, table, json_file, pk, nl, csv, batch_size, upsert=False
+        path, table, json_file, pk, nl, csv, batch_size, alter=alter, upsert=False
     )
 
 
 @cli.command()
 @insert_upsert_options
-def upsert(path, table, json_file, pk, nl, csv, batch_size):
+def upsert(path, table, json_file, pk, nl, csv, batch_size, alter):
     """
     Upsert records based on their primary key. Works like 'insert' but if
     an incoming record has a primary key that matches an existing record
     the existing record will be replaced.
     """
     insert_upsert_implementation(
-        path, table, json_file, pk, nl, csv, batch_size, upsert=True
+        path, table, json_file, pk, nl, csv, batch_size, alter=alter, upsert=True
     )
 
 

sqlite_utils/db.py

@@ -59,6 +59,14 @@ if np:
     )
 
 
+REVERSE_COLUMN_TYPE_MAPPING = {
+    "TEXT": str,
+    "BLOB": bytes,
+    "INTEGER": int,
+    "FLOAT": float,
+}
+
+
 class AlterError(Exception):
     pass
 
@@ -182,6 +190,14 @@ class Table:
         ).fetchall()
         return [Column(*row) for row in rows]
 
+    @property
+    def columns_dict(self):
+        "Returns {column: python-type} dictionary"
+        return {
+            column.name: REVERSE_COLUMN_TYPE_MAPPING[column.type]
+            for column in self.columns
+        }
+
     @property
     def rows(self):
         if not self.exists:
@@ -235,7 +251,7 @@ class Table:
             for seqno, cid, name in self.db.conn.execute(column_sql).fetchall():
                 columns.append(name)
             row["columns"] = columns
-            # These coluns may be missing on older SQLite versions:
+            # These columns may be missing on older SQLite versions:
             for key, default in {"origin": "c", "partial": 0}.items():
                 if key not in row:
                     row[key] = default
@@ -429,6 +445,7 @@ class Table:
         upsert=False,
         column_order=None,
         hash_id=None,
+        alter=False,
     ):
         return self.insert_all(
             [record],
@@ -437,6 +454,7 @@ class Table:
             upsert=upsert,
             column_order=column_order,
             hash_id=hash_id,
+            alter=alter,
         )
 
     def insert_all(
@@ -448,6 +466,7 @@ class Table:
         batch_size=100,
         column_order=None,
         hash_id=None,
+        alter=False,
     ):
         """
         Like .insert() but takes a list of records and ensures that the table
@@ -500,7 +519,15 @@ class Table:
                     for key in all_columns
                 )
             with self.db.conn:
-                result = self.db.conn.execute(sql, values)
+                try:
+                    result = self.db.conn.execute(sql, values)
+                except sqlite3.OperationalError as e:
+                    if alter and (" has no column " in e.args[0]):
+                        # Attempt to add any missing columns, then try again
+                        self.add_missing_columns(chunk)
+                        result = self.db.conn.execute(sql, values)
+                    else:
+                        raise
                 self.last_rowid = result.lastrowid
                 self.last_pk = None
                 if hash_id or pk:
@@ -513,7 +540,13 @@ class Table:
         return self
 
     def upsert(
-        self, record, pk=None, foreign_keys=None, column_order=None, hash_id=None
+        self,
+        record,
+        pk=None,
+        foreign_keys=None,
+        column_order=None,
+        hash_id=None,
+        alter=False,
     ):
         return self.insert(
             record,
@@ -522,6 +555,7 @@ class Table:
             upsert=True,
             column_order=column_order,
             hash_id=hash_id,
+            alter=alter,
         )
 
     def upsert_all(
@@ -532,6 +566,7 @@ class Table:
         column_order=None,
         batch_size=100,
         hash_id=None,
+        alter=False,
     ):
         return self.insert_all(
             records,
@@ -541,8 +576,16 @@ class Table:
             batch_size=100,
             upsert=True,
             hash_id=hash_id,
+            alter=alter,
         )
 
+    def add_missing_columns(self, records):
+        needed_columns = self.detect_column_types(records)
+        current_columns = self.columns_dict
+        for col_name, col_type in needed_columns.items():
+            if col_name not in current_columns:
+                self.add_column(col_name, col_type)
+
 
 def chunks(sequence, size):
     iterator = iter(sequence)