--empty-null option for CSV and TSV imports, closes #563

docs/cli-reference.rst

@@ -266,6 +266,7 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr
       --nl                      Expect newline-delimited JSON
       -c, --csv                 Expect CSV input
       --tsv                     Expect TSV input
+      --empty-null              Treat empty strings as NULL
       --lines                   Treat each line as a single value called 'line'
       --text                    Treat input as a single value called 'text'
       --convert TEXT            Python code to convert each item
@@ -321,6 +322,7 @@ See :ref:`cli_upsert`.
       --nl                      Expect newline-delimited JSON
       -c, --csv                 Expect CSV input
       --tsv                     Expect TSV input
+      --empty-null              Treat empty strings as NULL
       --lines                   Treat each line as a single value called 'line'
       --text                    Treat input as a single value called 'text'
       --convert TEXT            Python code to convert each item
@@ -372,6 +374,7 @@ See :ref:`cli_bulk`.
       --nl                   Expect newline-delimited JSON
       -c, --csv              Expect CSV input
       --tsv                  Expect TSV input
+      --empty-null           Treat empty strings as NULL
       --lines                Treat each line as a single value called 'line'
       --text                 Treat input as a single value called 'text'
       --convert TEXT         Python code to convert each item

docs/cli.rst

@@ -1281,6 +1281,22 @@ You can set the ``SQLITE_UTILS_DETECT_TYPES`` environment variable if you want `
 
     export SQLITE_UTILS_DETECT_TYPES=1
 
+If a CSV or TSV file includes empty cells, like this one:
+
+.. code-block:: csv
+
+    name,age,weight
+    Cleo,6,
+    Dori,,3.5
+
+They will be imported into SQLite as empty string values, ``""``.
+
+To import them as ``NULL`` values instead, use the ``--empty-null`` option:
+
+.. code-block:: bash
+
+    sqlite-utils insert creatures.db creatures creatures.csv --csv --empty-null
+
 .. _cli_insert_csv_tsv_delimiter:
 
 Alternative delimiters and quote characters

sqlite_utils/cli.py

@@ -810,6 +810,7 @@ _import_options = (
     click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"),
     click.option("-c", "--csv", is_flag=True, help="Expect CSV input"),
     click.option("--tsv", is_flag=True, help="Expect TSV input"),
+    click.option("--empty-null", is_flag=True, help="Treat empty strings as NULL"),
     click.option(
         "--lines",
         is_flag=True,
@@ -916,6 +917,7 @@ def insert_upsert_implementation(
     nl,
     csv,
     tsv,
+    empty_null,
     lines,
     text,
     convert,
@@ -951,6 +953,8 @@ def insert_upsert_implementation(
         raise click.ClickException("Use just one of --nl, --csv or --tsv")
     if (csv or tsv) and flatten:
         raise click.ClickException("--flatten cannot be used with --csv or --tsv")
+    if empty_null and not (csv or tsv):
+        raise click.ClickException("--empty-null can only be used with --csv or --tsv")
     if encoding and not (csv or tsv):
         raise click.ClickException("--encoding must be used with --csv or --tsv")
     if pk and len(pk) == 1:
@@ -989,7 +993,13 @@ def insert_upsert_implementation(
                 reader = itertools.chain([first_row], reader)
             else:
                 headers = first_row
-            docs = (dict(zip(headers, row)) for row in reader)
+            if empty_null:
+                docs = (
+                    dict(zip(headers, [None if cell == "" else cell for cell in row]))
+                    for row in reader
+                )
+            else:
+                docs = (dict(zip(headers, row)) for row in reader)
             if detect_types:
                 tracker = TypeTracker()
                 docs = tracker.wrap(docs)
@@ -1141,6 +1151,7 @@ def insert(
     nl,
     csv,
     tsv,
+    empty_null,
     lines,
     text,
     convert,
@@ -1217,6 +1228,7 @@ def insert(
             nl,
             csv,
             tsv,
+            empty_null,
             lines,
             text,
             convert,
@@ -1255,6 +1267,7 @@ def upsert(
     nl,
     csv,
     tsv,
+    empty_null,
     lines,
     text,
     convert,
@@ -1297,6 +1310,7 @@ def upsert(
             nl,
             csv,
             tsv,
+            empty_null,
             lines,
             text,
             convert,
@@ -1345,6 +1359,7 @@ def bulk(
     nl,
     csv,
     tsv,
+    empty_null,
     lines,
     text,
     convert,
@@ -1379,6 +1394,7 @@ def bulk(
             nl=nl,
             csv=csv,
             tsv=tsv,
+            empty_null=empty_null,
             lines=lines,
             text=text,
             convert=convert,

tests/test_cli.py

@@ -55,7 +55,7 @@ def test_help(options):
 
 
 def test_tables(db_path):
-    result = CliRunner().invoke(cli.cli, ["tables", db_path])
+    result = CliRunner().invoke(cli.cli, ["tables", db_path], catch_exceptions=False)
     assert '[{"table": "Gosh"},\n {"table": "Gosh2"}]' == result.output.strip()
 
 

tests/test_cli_insert.py

@@ -229,6 +229,24 @@ def test_insert_csv_tsv(content, options, db_path, tmpdir):
     assert [{"foo": "1", "bar": "2", "baz": "cat,dog"}] == list(db["data"].rows)
 
 
+@pytest.mark.parametrize("empty_null", (True, False))
+def test_insert_csv_empty_null(db_path, empty_null):
+    options = ["--csv"]
+    if empty_null:
+        options.append("--empty-null")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "data", "-"] + options,
+        catch_exceptions=False,
+        input="foo,bar,baz\n1,,cat,dog",
+    )
+    assert result.exit_code == 0
+    db = Database(db_path)
+    assert [r for r in db["data"].rows] == [
+        {"foo": "1", "bar": None if empty_null else "", "baz": "cat"}
+    ]
+
+
 @pytest.mark.parametrize(
     "input,args",
     (