diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 980a66b..c558bf0 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -266,6 +266,7 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr --nl Expect newline-delimited JSON -c, --csv Expect CSV input --tsv Expect TSV input + --empty-null Treat empty strings as NULL --lines Treat each line as a single value called 'line' --text Treat input as a single value called 'text' --convert TEXT Python code to convert each item @@ -321,6 +322,7 @@ See :ref:`cli_upsert`. --nl Expect newline-delimited JSON -c, --csv Expect CSV input --tsv Expect TSV input + --empty-null Treat empty strings as NULL --lines Treat each line as a single value called 'line' --text Treat input as a single value called 'text' --convert TEXT Python code to convert each item @@ -372,6 +374,7 @@ See :ref:`cli_bulk`. --nl Expect newline-delimited JSON -c, --csv Expect CSV input --tsv Expect TSV input + --empty-null Treat empty strings as NULL --lines Treat each line as a single value called 'line' --text Treat input as a single value called 'text' --convert TEXT Python code to convert each item diff --git a/docs/cli.rst b/docs/cli.rst index 77ef0ee..7ed53b6 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1281,6 +1281,22 @@ You can set the ``SQLITE_UTILS_DETECT_TYPES`` environment variable if you want ` export SQLITE_UTILS_DETECT_TYPES=1 +If a CSV or TSV file includes empty cells, like this one: + +.. code-block:: csv + + name,age,weight + Cleo,6, + Dori,,3.5 + +They will be imported into SQLite as empty string values, ``""``. + +To import them as ``NULL`` values instead, use the ``--empty-null`` option: + +.. code-block:: bash + + sqlite-utils insert creatures.db creatures creatures.csv --csv --empty-null + .. _cli_insert_csv_tsv_delimiter: Alternative delimiters and quote characters diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 38247e1..30fc20c 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -810,6 +810,7 @@ _import_options = ( click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), click.option("-c", "--csv", is_flag=True, help="Expect CSV input"), click.option("--tsv", is_flag=True, help="Expect TSV input"), + click.option("--empty-null", is_flag=True, help="Treat empty strings as NULL"), click.option( "--lines", is_flag=True, @@ -916,6 +917,7 @@ def insert_upsert_implementation( nl, csv, tsv, + empty_null, lines, text, convert, @@ -951,6 +953,8 @@ def insert_upsert_implementation( raise click.ClickException("Use just one of --nl, --csv or --tsv") if (csv or tsv) and flatten: raise click.ClickException("--flatten cannot be used with --csv or --tsv") + if empty_null and not (csv or tsv): + raise click.ClickException("--empty-null can only be used with --csv or --tsv") if encoding and not (csv or tsv): raise click.ClickException("--encoding must be used with --csv or --tsv") if pk and len(pk) == 1: @@ -989,7 +993,13 @@ def insert_upsert_implementation( reader = itertools.chain([first_row], reader) else: headers = first_row - docs = (dict(zip(headers, row)) for row in reader) + if empty_null: + docs = ( + dict(zip(headers, [None if cell == "" else cell for cell in row])) + for row in reader + ) + else: + docs = (dict(zip(headers, row)) for row in reader) if detect_types: tracker = TypeTracker() docs = tracker.wrap(docs) @@ -1141,6 +1151,7 @@ def insert( nl, csv, tsv, + empty_null, lines, text, convert, @@ -1217,6 +1228,7 @@ def insert( nl, csv, tsv, + empty_null, lines, text, convert, @@ -1255,6 +1267,7 @@ def upsert( nl, csv, tsv, + empty_null, lines, text, convert, @@ -1297,6 +1310,7 @@ def upsert( nl, csv, tsv, + empty_null, lines, text, convert, @@ -1345,6 +1359,7 @@ def bulk( nl, csv, tsv, + empty_null, lines, text, convert, @@ -1379,6 +1394,7 @@ def bulk( nl=nl, csv=csv, tsv=tsv, + empty_null=empty_null, lines=lines, text=text, convert=convert, diff --git a/tests/test_cli.py b/tests/test_cli.py index eb569f0..57e852a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -55,7 +55,7 @@ def test_help(options): def test_tables(db_path): - result = CliRunner().invoke(cli.cli, ["tables", db_path]) + result = CliRunner().invoke(cli.cli, ["tables", db_path], catch_exceptions=False) assert '[{"table": "Gosh"},\n {"table": "Gosh2"}]' == result.output.strip() diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index ca261aa..274f414 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -229,6 +229,24 @@ def test_insert_csv_tsv(content, options, db_path, tmpdir): assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows) +@pytest.mark.parametrize("empty_null", (True, False)) +def test_insert_csv_empty_null(db_path, empty_null): + options = ["--csv"] + if empty_null: + options.append("--empty-null") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "data", "-"] + options, + catch_exceptions=False, + input="foo,bar,baz\n1,,cat,dog", + ) + assert result.exit_code == 0 + db = Database(db_path) + assert [r for r in db["data"].rows] == [ + {"foo": "1", "bar": None if empty_null else "", "baz": "cat"} + ] + + @pytest.mark.parametrize( "input,args", (