Increase csv field_size_limit to maximum, closes #229

Refs #227

sqlite_utils/cli.py

@@ -31,6 +31,18 @@ It's often worth trying: --encoding=latin-1
 """.strip()
 
 
+# Increase CSV field size limit to maximim possible
+# https://stackoverflow.com/a/15063941
+field_size_limit = sys.maxsize
+
+while True:
+    try:
+        csv_std.field_size_limit(field_size_limit)
+        break
+    except OverflowError:
+        field_size_limit = int(field_size_limit / 10)
+
+
 def output_options(fn):
     for decorator in reversed(
         (

tests/test_cli.py

@@ -1799,3 +1799,23 @@ def test_triggers(tmpdir, extra_args, expected):
     )
     assert result.exit_code == 0
     assert result.output == expected
+
+
+def test_long_csv_column_value(tmpdir):
+    db_path = str(tmpdir / "test.db")
+    csv_path = str(tmpdir / "test.csv")
+    csv_file = open(csv_path, "w")
+    long_string = "a" * 131073
+    csv_file.write("id,text\n")
+    csv_file.write("1,{}\n".format(long_string))
+    csv_file.close()
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "bigtable", csv_path, "--csv"],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    db = Database(db_path)
+    rows = list(db["bigtable"].rows)
+    assert len(rows) == 1
+    assert rows[0]["text"] == long_string