From 1a93b72ba710ea2271eaabc204685a27d2469374 Mon Sep 17 00:00:00 2001
From: Simon Willison <swillison@gmail.com>
Date: Sun, 14 Feb 2021 13:33:21 -0800
Subject: [PATCH] Increase csv field_size_limit to maximum, closes #229

Refs #227
---
 sqlite_utils/cli.py | 12 ++++++++++++
 tests/test_cli.py   | 20 ++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py
index c871232..5fc181a 100644
--- a/sqlite_utils/cli.py
+++ b/sqlite_utils/cli.py
@@ -31,6 +31,18 @@ It's often worth trying: --encoding=latin-1
 """.strip()
 
 
+# Increase CSV field size limit to maximim possible
+# https://stackoverflow.com/a/15063941
+field_size_limit = sys.maxsize
+
+while True:
+    try:
+        csv_std.field_size_limit(field_size_limit)
+        break
+    except OverflowError:
+        field_size_limit = int(field_size_limit / 10)
+
+
 def output_options(fn):
     for decorator in reversed(
         (
diff --git a/tests/test_cli.py b/tests/test_cli.py
index e324bdd..a3669ab 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -1799,3 +1799,23 @@ def test_triggers(tmpdir, extra_args, expected):
     )
     assert result.exit_code == 0
     assert result.output == expected
+
+
+def test_long_csv_column_value(tmpdir):
+    db_path = str(tmpdir / "test.db")
+    csv_path = str(tmpdir / "test.csv")
+    csv_file = open(csv_path, "w")
+    long_string = "a" * 131073
+    csv_file.write("id,text\n")
+    csv_file.write("1,{}\n".format(long_string))
+    csv_file.close()
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "bigtable", csv_path, "--csv"],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    db = Database(db_path)
+    rows = list(db["bigtable"].rows)
+    assert len(rows) == 1
+    assert rows[0]["text"] == long_string