diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c871232..5fc181a 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -31,6 +31,18 @@ It's often worth trying: --encoding=latin-1 """.strip() +# Increase CSV field size limit to maximim possible +# https://stackoverflow.com/a/15063941 +field_size_limit = sys.maxsize + +while True: + try: + csv_std.field_size_limit(field_size_limit) + break + except OverflowError: + field_size_limit = int(field_size_limit / 10) + + def output_options(fn): for decorator in reversed( ( diff --git a/tests/test_cli.py b/tests/test_cli.py index e324bdd..a3669ab 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1799,3 +1799,23 @@ def test_triggers(tmpdir, extra_args, expected): ) assert result.exit_code == 0 assert result.output == expected + + +def test_long_csv_column_value(tmpdir): + db_path = str(tmpdir / "test.db") + csv_path = str(tmpdir / "test.csv") + csv_file = open(csv_path, "w") + long_string = "a" * 131073 + csv_file.write("id,text\n") + csv_file.write("1,{}\n".format(long_string)) + csv_file.close() + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "bigtable", csv_path, "--csv"], + catch_exceptions=False, + ) + assert result.exit_code == 0 + db = Database(db_path) + rows = list(db["bigtable"].rows) + assert len(rows) == 1 + assert rows[0]["text"] == long_string