diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 19b714b..c3f7a24 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -547,15 +547,25 @@ See :ref:`cli_convert`. r.jsonsplit(value, delimiter=',', type=) - Convert a string like a,b,c into a JSON array ["a", "b", "c"] + Convert a string like a,b,c into a JSON array ["a", "b", "c"] - r.parsedate(value, dayfirst=False, yearfirst=False) + r.parsedate(value, dayfirst=False, yearfirst=False, errors=None) - Parse a date and convert it to ISO date format: yyyy-mm-dd + Parse a date and convert it to ISO date format: yyyy-mm-dd +  + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null - r.parsedatetime(value, dayfirst=False, yearfirst=False) + r.parsedatetime(value, dayfirst=False, yearfirst=False, errors=None) - Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS + Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS +  + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null You can use these recipes like so: diff --git a/docs/cli.rst b/docs/cli.rst index 03689c1..8bd3650 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1303,12 +1303,20 @@ Various built-in recipe functions are available for common operations. These are Would produce an array like this: ``[1.2, 3.0, 4.5]`` -``r.parsedate(value, dayfirst=False, yearfirst=False)`` +``r.parsedate(value, dayfirst=False, yearfirst=False, errors=None)`` Parse a date and convert it to ISO date format: ``yyyy-mm-dd`` In the case of dates such as ``03/04/05`` U.S. ``MM/DD/YY`` format is assumed - you can use ``dayfirst=True`` or ``yearfirst=True`` to change how these ambiguous dates are interpreted. -``r.parsedatetime(value, dayfirst=False, yearfirst=False)`` + Use the ``errors=`` parameter to specify what should happen if a value cannot be parsed. + + By default, if any value cannot be parsed an error will be occurred and all values will be left as they were. + + Set ``errors=r.IGNORE`` to ignore any values that cannot be parsed, leaving them unchanged. + + Set ``errors=r.SET_NULL`` to set any values that cannot be parsed to ``null``. + +``r.parsedatetime(value, dayfirst=False, yearfirst=False, errors=None)`` Parse a datetime and convert it to ISO datetime format: ``yyyy-mm-ddTHH:MM:SS`` These recipes can be used in the code passed to ``sqlite-utils convert`` like this:: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8255b56..0cf0468 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2583,12 +2583,16 @@ def _generate_convert_help(): """ ).strip() recipe_names = [ - n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser") + n + for n in dir(recipes) + if not n.startswith("_") + and n not in ("json", "parser") + and callable(getattr(recipes, n)) ] for name in recipe_names: fn = getattr(recipes, name) - help += "\n\nr.{}{}\n\n {}".format( - name, str(inspect.signature(fn)), fn.__doc__ + help += "\n\nr.{}{}\n\n\b{}".format( + name, str(inspect.signature(fn)), fn.__doc__.rstrip() ) help += "\n\n" help += textwrap.dedent( diff --git a/sqlite_utils/recipes.py b/sqlite_utils/recipes.py index 6918661..ac41954 100644 --- a/sqlite_utils/recipes.py +++ b/sqlite_utils/recipes.py @@ -1,19 +1,56 @@ from dateutil import parser import json - -def parsedate(value, dayfirst=False, yearfirst=False): - "Parse a date and convert it to ISO date format: yyyy-mm-dd" - return ( - parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).date().isoformat() - ) +IGNORE = object() +SET_NULL = object() -def parsedatetime(value, dayfirst=False, yearfirst=False): - "Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS" - return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat() +def parsedate(value, dayfirst=False, yearfirst=False, errors=None): + """ + Parse a date and convert it to ISO date format: yyyy-mm-dd + \b + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null + """ + try: + return ( + parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst) + .date() + .isoformat() + ) + except parser.ParserError: + if errors is IGNORE: + return value + elif errors is SET_NULL: + return None + else: + raise + + +def parsedatetime(value, dayfirst=False, yearfirst=False, errors=None): + """ + Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS + \b + - dayfirst=True: treat xx as the day in xx/yy/zz + - yearfirst=True: treat xx as the year in xx/yy/zz + - errors=r.IGNORE to ignore values that cannot be parsed + - errors=r.SET_NULL to set values that cannot be parsed to null + """ + try: + return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat() + except parser.ParserError: + if errors is IGNORE: + return value + elif errors is SET_NULL: + return None + else: + raise def jsonsplit(value, delimiter=",", type=str): - 'Convert a string like a,b,c into a JSON array ["a", "b", "c"]' + """ + Convert a string like a,b,c into a JSON array ["a", "b", "c"] + """ return json.dumps([type(s.strip()) for s in value.split(delimiter)]) diff --git a/tests/test_docs.py b/tests/test_docs.py index d760629..7f2e3ed 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -48,7 +48,13 @@ def test_convert_help(): @pytest.mark.parametrize( "recipe", - [n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")], + [ + n + for n in dir(recipes) + if not n.startswith("_") + and n not in ("json", "parser") + and callable(getattr(recipes, n)) + ], ) def test_recipes_are_documented(documented_recipes, recipe): assert recipe in documented_recipes diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 89240a2..afb3e1a 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -1,4 +1,5 @@ from sqlite_utils import recipes +from sqlite_utils.utils import sqlite3 import json import pytest @@ -61,6 +62,28 @@ def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected): ] +@pytest.mark.parametrize("fn", ("parsedate", "parsedatetime")) +@pytest.mark.parametrize("errors", (None, recipes.SET_NULL, recipes.IGNORE)) +def test_dateparse_errors(fresh_db, fn, errors): + fresh_db["example"].insert_all( + [ + {"id": 1, "dt": "invalid"}, + ], + pk="id", + ) + if errors is None: + # Should raise an error + with pytest.raises(sqlite3.OperationalError): + fresh_db["example"].convert("dt", lambda value: getattr(recipes, fn)(value)) + else: + fresh_db["example"].convert( + "dt", lambda value: getattr(recipes, fn)(value, errors=errors) + ) + rows = list(fresh_db["example"].rows) + expected = [{"id": 1, "dt": None if errors is recipes.SET_NULL else "invalid"}] + assert rows == expected + + @pytest.mark.parametrize("delimiter", [None, ";", "-"]) def test_jsonsplit(fresh_db, delimiter): fresh_db["example"].insert_all(