From 62d292252804aa0a0c1d6fdc9ea1722b5ffb20a8 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 28 Jul 2019 14:10:56 +0300
Subject: [PATCH 1/2] Fix for too many SQL variables, closes #50

---
 sqlite_utils/db.py   |  9 ++++++++-
 tests/test_create.py | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py
index ef55976..29b2286 100644
--- a/sqlite_utils/db.py
+++ b/sqlite_utils/db.py
@@ -848,7 +848,14 @@ class Table:
         ), "Use either ignore=True or upsert=True, not both"
         all_columns = None
         first = True
-        for chunk in chunks(records, batch_size):
+        # We can only handle a max of 999 variables in a SQL insert, so
+        # we need to adjust the batch_size down if we have too many cols
+        records = iter(records)
+        # Peek at first record to count its columns:
+        first_record = next(records)
+        num_columns = len(first_record.keys())
+        batch_size = max(1, min(batch_size, 999 // num_columns))
+        for chunk in chunks(itertools.chain([first_record], records), batch_size):
             chunk = list(chunk)
             if first:
                 if not self.exists:
diff --git a/tests/test_create.py b/tests/test_create.py
index 222a967..8f30347 100644
--- a/tests/test_create.py
+++ b/tests/test_create.py
@@ -500,6 +500,42 @@ def test_upsert_rows_alter_table(fresh_db, use_table_factory):
     ] == list(table.rows)
 
 
+def test_bulk_insert_more_than_999_values(fresh_db):
+    "Inserting 100 items with 11 columns should work"
+    fresh_db["big"].insert_all(
+        (
+            {
+                "id": i + 1,
+                "c2": 2,
+                "c3": 3,
+                "c4": 4,
+                "c5": 5,
+                "c6": 6,
+                "c7": 7,
+                "c8": 8,
+                "c8": 9,
+                "c10": 10,
+                "c11": 11,
+            }
+            for i in range(100)
+        ),
+        pk="id",
+    )
+    assert 100 == fresh_db["big"].count
+
+
+@pytest.mark.parametrize(
+    "num_columns,should_error", ((900, False), (999, False), (1000, True))
+)
+def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error):
+    record = dict([("c{}".format(i), i) for i in range(num_columns)])
+    if should_error:
+        with pytest.raises(Exception):
+            fresh_db["big"].insert(record)
+    else:
+        fresh_db["big"].insert(record)
+
+
 @pytest.mark.parametrize(
     "columns,index_name,expected_index",
     (

From 0c1b8b7f96be874bb63801f69323960f277aa49a Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 28 Jul 2019 14:41:57 +0300
Subject: [PATCH 2/2] Use assertion to enforce <=999 columns

---
 sqlite_utils/db.py   | 7 ++++++-
 tests/test_create.py | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py
index 29b2286..586014b 100644
--- a/sqlite_utils/db.py
+++ b/sqlite_utils/db.py
@@ -6,6 +6,8 @@ import itertools
 import json
 import pathlib
 
+SQLITE_MAX_VARS = 999
+
 try:
     import numpy as np
 except ImportError:
@@ -854,7 +856,10 @@ class Table:
         # Peek at first record to count its columns:
         first_record = next(records)
         num_columns = len(first_record.keys())
-        batch_size = max(1, min(batch_size, 999 // num_columns))
+        assert (
+            num_columns <= SQLITE_MAX_VARS
+        ), "Rows can have a maximum of {} columns".format(SQLITE_MAX_VARS)
+        batch_size = max(1, min(batch_size, SQLITE_MAX_VARS // num_columns))
         for chunk in chunks(itertools.chain([first_record], records), batch_size):
             chunk = list(chunk)
             if first:
diff --git a/tests/test_create.py b/tests/test_create.py
index 8f30347..71b1583 100644
--- a/tests/test_create.py
+++ b/tests/test_create.py
@@ -530,7 +530,7 @@ def test_bulk_insert_more_than_999_values(fresh_db):
 def test_error_if_more_than_999_columns(fresh_db, num_columns, should_error):
     record = dict([("c{}".format(i), i) for i in range(num_columns)])
     if should_error:
-        with pytest.raises(Exception):
+        with pytest.raises(AssertionError):
             fresh_db["big"].insert(record)
     else:
         fresh_db["big"].insert(record)