errors=r.SET_NULL/r.IGNORE options for parsedate/parsedatetime, closes #416

docs/cli-reference.rst

@@ -547,15 +547,25 @@ See :ref:`cli_convert`.
 
       r.jsonsplit(value, delimiter=',', type=<class 'str'>)
 
-        Convert a string like a,b,c into a JSON array ["a", "b", "c"]
+          Convert a string like a,b,c into a JSON array ["a", "b", "c"]
 
-      r.parsedate(value, dayfirst=False, yearfirst=False)
+      r.parsedate(value, dayfirst=False, yearfirst=False, errors=None)
 
-        Parse a date and convert it to ISO date format: yyyy-mm-dd
+          Parse a date and convert it to ISO date format: yyyy-mm-dd
+          
+          - dayfirst=True: treat xx as the day in xx/yy/zz
+          - yearfirst=True: treat xx as the year in xx/yy/zz
+          - errors=r.IGNORE to ignore values that cannot be parsed
+          - errors=r.SET_NULL to set values that cannot be parsed to null
 
-      r.parsedatetime(value, dayfirst=False, yearfirst=False)
+      r.parsedatetime(value, dayfirst=False, yearfirst=False, errors=None)
 
-        Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS
+          Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS
+          
+          - dayfirst=True: treat xx as the day in xx/yy/zz
+          - yearfirst=True: treat xx as the year in xx/yy/zz
+          - errors=r.IGNORE to ignore values that cannot be parsed
+          - errors=r.SET_NULL to set values that cannot be parsed to null
 
       You can use these recipes like so:
 

docs/cli.rst

@@ -1303,12 +1303,20 @@ Various built-in recipe functions are available for common operations. These are
 
   Would produce an array like this: ``[1.2, 3.0, 4.5]``
 
-``r.parsedate(value, dayfirst=False, yearfirst=False)``
+``r.parsedate(value, dayfirst=False, yearfirst=False, errors=None)``
   Parse a date and convert it to ISO date format: ``yyyy-mm-dd``
 
   In the case of dates such as ``03/04/05`` U.S. ``MM/DD/YY`` format is assumed - you can use ``dayfirst=True`` or ``yearfirst=True`` to change how these ambiguous dates are interpreted.
 
-``r.parsedatetime(value, dayfirst=False, yearfirst=False)``
+  Use the ``errors=`` parameter to specify what should happen if a value cannot be parsed.
+
+  By default, if any value cannot be parsed an error will be occurred and all values will be left as they were.
+
+  Set ``errors=r.IGNORE`` to ignore any values that cannot be parsed, leaving them unchanged.
+
+  Set ``errors=r.SET_NULL`` to set any values that cannot be parsed to ``null``.
+
+``r.parsedatetime(value, dayfirst=False, yearfirst=False, errors=None)``
   Parse a datetime and convert it to ISO datetime format: ``yyyy-mm-ddTHH:MM:SS``
 
 These recipes can be used in the code passed to ``sqlite-utils convert`` like this::

sqlite_utils/cli.py

@@ -2583,12 +2583,16 @@ def _generate_convert_help():
     """
     ).strip()
     recipe_names = [
-        n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")
+        n
+        for n in dir(recipes)
+        if not n.startswith("_")
+        and n not in ("json", "parser")
+        and callable(getattr(recipes, n))
     ]
     for name in recipe_names:
         fn = getattr(recipes, name)
-        help += "\n\nr.{}{}\n\n  {}".format(
-            name, str(inspect.signature(fn)), fn.__doc__
+        help += "\n\nr.{}{}\n\n\b{}".format(
+            name, str(inspect.signature(fn)), fn.__doc__.rstrip()
         )
     help += "\n\n"
     help += textwrap.dedent(

sqlite_utils/recipes.py

@@ -1,19 +1,56 @@
 from dateutil import parser
 import json
 
-
-def parsedate(value, dayfirst=False, yearfirst=False):
-    "Parse a date and convert it to ISO date format: yyyy-mm-dd"
-    return (
-        parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).date().isoformat()
-    )
+IGNORE = object()
+SET_NULL = object()
 
 
-def parsedatetime(value, dayfirst=False, yearfirst=False):
-    "Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS"
-    return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat()
+def parsedate(value, dayfirst=False, yearfirst=False, errors=None):
+    """
+    Parse a date and convert it to ISO date format: yyyy-mm-dd
+    \b
+    - dayfirst=True: treat xx as the day in xx/yy/zz
+    - yearfirst=True: treat xx as the year in xx/yy/zz
+    - errors=r.IGNORE to ignore values that cannot be parsed
+    - errors=r.SET_NULL to set values that cannot be parsed to null
+    """
+    try:
+        return (
+            parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst)
+            .date()
+            .isoformat()
+        )
+    except parser.ParserError:
+        if errors is IGNORE:
+            return value
+        elif errors is SET_NULL:
+            return None
+        else:
+            raise
+
+
+def parsedatetime(value, dayfirst=False, yearfirst=False, errors=None):
+    """
+    Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS
+    \b
+    - dayfirst=True: treat xx as the day in xx/yy/zz
+    - yearfirst=True: treat xx as the year in xx/yy/zz
+    - errors=r.IGNORE to ignore values that cannot be parsed
+    - errors=r.SET_NULL to set values that cannot be parsed to null
+    """
+    try:
+        return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat()
+    except parser.ParserError:
+        if errors is IGNORE:
+            return value
+        elif errors is SET_NULL:
+            return None
+        else:
+            raise
 
 
 def jsonsplit(value, delimiter=",", type=str):
-    'Convert a string like a,b,c into a JSON array ["a", "b", "c"]'
+    """
+    Convert a string like a,b,c into a JSON array ["a", "b", "c"]
+    """
     return json.dumps([type(s.strip()) for s in value.split(delimiter)])

tests/test_docs.py

@@ -48,7 +48,13 @@ def test_convert_help():
 
 @pytest.mark.parametrize(
     "recipe",
-    [n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")],
+    [
+        n
+        for n in dir(recipes)
+        if not n.startswith("_")
+        and n not in ("json", "parser")
+        and callable(getattr(recipes, n))
+    ],
 )
 def test_recipes_are_documented(documented_recipes, recipe):
     assert recipe in documented_recipes

tests/test_recipes.py

@@ -1,4 +1,5 @@
 from sqlite_utils import recipes
+from sqlite_utils.utils import sqlite3
 import json
 import pytest
 
@@ -61,6 +62,28 @@ def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected):
     ]
 
 
+@pytest.mark.parametrize("fn", ("parsedate", "parsedatetime"))
+@pytest.mark.parametrize("errors", (None, recipes.SET_NULL, recipes.IGNORE))
+def test_dateparse_errors(fresh_db, fn, errors):
+    fresh_db["example"].insert_all(
+        [
+            {"id": 1, "dt": "invalid"},
+        ],
+        pk="id",
+    )
+    if errors is None:
+        # Should raise an error
+        with pytest.raises(sqlite3.OperationalError):
+            fresh_db["example"].convert("dt", lambda value: getattr(recipes, fn)(value))
+    else:
+        fresh_db["example"].convert(
+            "dt", lambda value: getattr(recipes, fn)(value, errors=errors)
+        )
+        rows = list(fresh_db["example"].rows)
+        expected = [{"id": 1, "dt": None if errors is recipes.SET_NULL else "invalid"}]
+        assert rows == expected
+
+
 @pytest.mark.parametrize("delimiter", [None, ";", "-"])
 def test_jsonsplit(fresh_db, delimiter):
     fresh_db["example"].insert_all(