From 947bb7626fd1763608a470adf9cf5f156ef003e9 Mon Sep 17 00:00:00 2001
From: Simon Wiles <simonjwiles@gmail.com>
Date: Fri, 28 Aug 2020 15:30:13 -0700
Subject: [PATCH] insert_all(..., alter=True) works for columns introduced
 after first 100 records

* Insert all columns for every chunk
* Update unit test to reflect new behaviour
* Test that exception is raised
* Update documentation

Closes #139. Thanks, Simon Wiles!
---
 docs/python-api.rst  |  2 +-
 sqlite_utils/db.py   |  8 ++++++++
 tests/test_create.py | 17 ++++++++++++++---
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/docs/python-api.rst b/docs/python-api.rst
index 9697fd3..5860397 100644
--- a/docs/python-api.rst
+++ b/docs/python-api.rst
@@ -410,7 +410,7 @@ Use it like this:
         "is_good_dog": True,
     }], pk="id", column_order=("id", "twitter", "name"))
 
-The column types used in the ``CREATE TABLE`` statement are automatically derived from the types of data in that first batch of rows. Any additional or missing columns in subsequent batches will be ignored.
+The column types used in the ``CREATE TABLE`` statement are automatically derived from the types of data in that first batch of rows. Any additional columns in subsequent batches will cause a ``sqlite3.OperationalError`` exception to be raised unless the ``alter=True`` argument is supplied, in which case the new columns will be created.
 
 The function can accept an iterator or generator of rows and will commit them according to the batch size. The default batch size is 100, but you can specify a different size using the ``batch_size`` parameter:
 
diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py
index a8791c3..75599f6 100644
--- a/sqlite_utils/db.py
+++ b/sqlite_utils/db.py
@@ -1074,6 +1074,14 @@ class Table(Queryable):
                 all_columns = list(sorted(all_columns))
                 if hash_id:
                     all_columns.insert(0, hash_id)
+            else:
+                all_columns += [
+                    column
+                    for record in chunk
+                    for column in record
+                    if column not in all_columns
+                ]
+
             validate_column_names(all_columns)
             first = False
             # values is the list of insert data that is passed to the
diff --git a/tests/test_create.py b/tests/test_create.py
index a84eb8d..fc8edc0 100644
--- a/tests/test_create.py
+++ b/tests/test_create.py
@@ -707,13 +707,24 @@ def test_insert_thousands_using_generator(fresh_db):
     assert 10000 == fresh_db["test"].count
 
 
-def test_insert_thousands_ignores_extra_columns_after_first_100(fresh_db):
+def test_insert_thousands_raises_exception_wtih_extra_columns_after_first_100(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/139
+    with pytest.raises(Exception, match="table test has no column named extra"):
+        fresh_db["test"].insert_all(
+            [{"i": i, "word": "word_{}".format(i)} for i in range(100)]
+            + [{"i": 101, "extra": "This extra column should cause an exception"}],
+        )
+
+
+def test_insert_thousands_adds_extra_columns_after_first_100_with_alter(fresh_db):
+    # https://github.com/simonw/sqlite-utils/issues/139
     fresh_db["test"].insert_all(
         [{"i": i, "word": "word_{}".format(i)} for i in range(100)]
-        + [{"i": 101, "extra": "This extra column should cause an exception"}]
+        + [{"i": 101, "extra": "Should trigger ALTER"}],
+        alter=True,
     )
     rows = fresh_db.execute_returning_dicts("select * from test where i = 101")
-    assert [{"i": 101, "word": None}] == rows
+    assert [{"i": 101, "word": None, "extra": "Should trigger ALTER"}] == rows
 
 
 def test_insert_ignore(fresh_db):