diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 05ff9d3..86eddfb 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -6,6 +6,7 @@ import hashlib import pathlib import sqlite_utils from sqlite_utils.db import AlterError, BadMultiValues, DescIndex +from sqlite_utils.utils import maximize_csv_field_size_limit from sqlite_utils import recipes import textwrap import inspect @@ -46,17 +47,7 @@ If you do not know the encoding, running 'file filename.csv' may tell you. It's often worth trying: --encoding=latin-1 """.strip() - -# Increase CSV field size limit to maximum possible -# https://stackoverflow.com/a/15063941 -field_size_limit = sys.maxsize - -while True: - try: - csv_std.field_size_limit(field_size_limit) - break - except OverflowError: - field_size_limit = int(field_size_limit / 10) +maximize_csv_field_size_limit() def output_options(fn): diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 141356a..d2ccc5f 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -7,6 +7,7 @@ import io import itertools import json import os +import sys from . import recipes from typing import Dict, cast, BinaryIO, Iterable, Optional, Tuple, Type @@ -29,6 +30,24 @@ SPATIALITE_PATHS = ( "/usr/local/lib/mod_spatialite.dylib", ) +# Mainly so we can restore it if needed in the tests: +ORIGINAL_CSV_FIELD_SIZE_LIMIT = csv.field_size_limit() + + +def maximize_csv_field_size_limit(): + """ + Increase the CSV field size limit to the maximum possible. + """ + # https://stackoverflow.com/a/15063941 + field_size_limit = sys.maxsize + + while True: + try: + csv.field_size_limit(field_size_limit) + break + except OverflowError: + field_size_limit = int(field_size_limit / 10) + def find_spatialite() -> Optional[str]: """ diff --git a/tests/test_rows_from_file.py b/tests/test_rows_from_file.py index 5575b34..fdc7a03 100644 --- a/tests/test_rows_from_file.py +++ b/tests/test_rows_from_file.py @@ -19,7 +19,7 @@ def test_rows_from_file_detect_format(input, expected_format): @pytest.mark.parametrize( - "ignore_extras,restkey,expected", + "ignore_extras,extras_key,expected", ( (True, None, [{"id": "1", "name": "Cleo"}]), (False, "_rest", [{"id": "1", "name": "Cleo", "_rest": ["oops"]}]), @@ -27,13 +27,13 @@ def test_rows_from_file_detect_format(input, expected_format): (False, False, None), ), ) -def test_rows_from_file_extra_fields_strategies(ignore_extras, restkey, expected): +def test_rows_from_file_extra_fields_strategies(ignore_extras, extras_key, expected): try: rows, format = rows_from_file( BytesIO(b"id,name\r\n1,Cleo,oops"), format=Format.CSV, ignore_extras=ignore_extras, - restkey=restkey, + extras_key=extras_key, ) list_rows = list(rows) except RowError: