Implemented recipes for sqlite-utils convert, refs #251

docs/cli.rst

@@ -947,6 +947,37 @@ You can specify Python modules that should be imported and made available to you
 
 The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database.
 
+.. _cli_convert_recipes:
+
+sqlite-utils convert recipes
+----------------------------
+
+Various built-in recipe functions are available for common operations. These are:
+
+``r.jsonsplit(value, delimiter=',', type=<class 'str'>)``
+  Convert a string like ``a,b,c`` into a JSON array ``["a", "b", "c"]``
+
+  The ``delimiter`` parameter can be used to specify a different delimiter.
+
+  The ``type`` parameter can be set to ``float`` or ``int`` to produce a JSON array of different types, for example if the column's string value was ``1.2,3,4`` the following::
+
+      r.jsonsplit(value, type=float)
+
+  Would produce an array like this: ``[1.2, 3.0, 4.5]``
+
+``r.parsedate(value, dayfirst=False, yearfirst=False)``
+  Parse a date and convert it to ISO date format: ``yyyy-mm-dd``
+
+  In the case of dates such as ``03/04/05`` U.S. ``MM/DD/YY`` format is assumed - you can use ``dayfirst=True`` or ``yearfirst=True`` to change how these ambiguous dates are interpreted.
+
+``r.parsedatetime(value, dayfirst=False, yearfirst=False)``
+  Parse a datetime and convert it to ISO datetime format: ``yyyy-mm-ddTHH:MM:SS``
+
+These recipes can be used in the code passed to ``sqlite-utils convert`` like this::
+
+    $ sqlite-utils convert my.db mytable mycolumn \\
+      'r.jsonsplit(value, delimiter=":")'
+
 .. _cli_convert_output:
 
 Saving the result to a different column

setup.py

@@ -22,7 +22,13 @@ setup(
     version=VERSION,
     license="Apache License, Version 2.0",
     packages=find_packages(exclude=["tests", "tests.*"]),
-    install_requires=["sqlite-fts4", "click", "click-default-group", "tabulate"],
+    install_requires=[
+        "sqlite-fts4",
+        "click",
+        "click-default-group",
+        "tabulate",
+        "dateutils",
+    ],
     setup_requires=["pytest-runner"],
     extras_require={
         "test": ["pytest", "black", "hypothesis"],

sqlite_utils/cli.py

@@ -6,7 +6,9 @@ import hashlib
 import pathlib
 import sqlite_utils
 from sqlite_utils.db import AlterError, DescIndex
+from sqlite_utils import recipes
 import textwrap
+import inspect
 import io
 import itertools
 import json
@@ -1904,7 +1906,43 @@ def analyze_tables(
         click.echo(details)
 
 
-@cli.command(name="convert")
+def _generate_convert_help():
+    help = textwrap.dedent(
+        """
+    Convert columns using Python code you supply. For example:
+
+    \b
+    $ sqlite-utils convert my.db mytable mycolumn \\
+        '"\\n".join(textwrap.wrap(value, 10))' \\
+        --import=textwrap
+
+    "value" is a variable with the column value to be converted.
+
+    The following common operations are available as recipe functions:
+    """
+    ).strip()
+    recipe_names = [
+        n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")
+    ]
+    for name in recipe_names:
+        fn = getattr(recipes, name)
+        help += "\n\nr.{}{}\n\n  {}".format(
+            name, str(inspect.signature(fn)), fn.__doc__
+        )
+    help += "\n\n"
+    help += textwrap.dedent(
+        """
+    You can use these recipes like so:
+
+    \b
+    $ sqlite-utils convert my.db mytable mycolumn \\
+        'r.jsonsplit(value, delimiter=":")'
+    """
+    ).strip()
+    return help
+
+
+@cli.command(help=_generate_convert_help())
 @click.argument(
     "db_path",
     type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
@@ -1944,16 +1982,7 @@ def convert(
     drop,
     silent,
 ):
-    """
+    sqlite3.enable_callback_tracebacks(True)
-    Convert columns using Python code you supply. For example:
-
-    \b
-    $ sqlite-utils convert my.db mytable mycolumn \\
-        '"\\n".join(textwrap.wrap(value, 10))' \\
-        --import=textwrap
-
-    "value" is a variable with the column value to be converted.
-    """
     if output is not None and len(columns) > 1:
         raise click.ClickException("Cannot use --output with more than one column")
     if multi and len(columns) > 1:
@@ -1967,7 +1996,7 @@ def convert(
         new_code.append("    {}".format(line))
     code_o = compile("\n".join(new_code), "<string>", "exec")
     locals = {}
-    globals = {}
+    globals = {"r": recipes, "recipes": recipes}
     for import_ in imports:
         globals[import_] = __import__(import_)
     exec(code_o, globals, locals)

sqlite_utils/recipes.py

@@ -0,0 +1,19 @@
+from dateutil import parser
+import json
+
+
+def parsedate(value, dayfirst=False, yearfirst=False):
+    "Parse a date and convert it to ISO date format: yyyy-mm-dd"
+    return (
+        parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).date().isoformat()
+    )
+
+
+def parsedatetime(value, dayfirst=False, yearfirst=False):
+    "Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS"
+    return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat()
+
+
+def jsonsplit(value, delimiter=",", type=str):
+    'Convert a string like a,b,c into a JSON array ["a", "b", "c"]'
+    return json.dumps([type(s.strip()) for s in value.split(delimiter)])

tests/test_cli_convert.py

@@ -1,6 +1,7 @@
 from click.testing import CliRunner
 from sqlite_utils import cli
 import sqlite_utils
+import json
 import textwrap
 import pathlib
 import pytest
@@ -329,3 +330,76 @@ def test_convert_multi_complex_column_types(fresh_db_and_path):
         "   [id] INTEGER PRIMARY KEY\n"
         ", [is_str] TEXT, [is_float] FLOAT, [is_int] INTEGER, [is_bytes] BLOB)"
     )
+
+
+@pytest.mark.parametrize("delimiter", [None, ";", "-"])
+def test_recipe_jsonsplit(tmpdir, delimiter):
+    db_path = str(pathlib.Path(tmpdir) / "data.db")
+    db = sqlite_utils.Database(db_path)
+    db["example"].insert_all(
+        [
+            {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])},
+            {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])},
+        ],
+        pk="id",
+    )
+    code = "r.jsonsplit(value)"
+    if delimiter:
+        code = 'recipes.jsonsplit(value, delimiter="{}")'.format(delimiter)
+    args = ["convert", db_path, "example", "tags", code]
+    result = CliRunner().invoke(cli.cli, args)
+    assert 0 == result.exit_code, result.output
+    assert list(db["example"].rows) == [
+        {"id": 1, "tags": '["foo", "bar"]'},
+        {"id": 2, "tags": '["bar", "baz"]'},
+    ]
+
+
+@pytest.mark.parametrize(
+    "type,expected_array",
+    (
+        (None, ["1", "2", "3"]),
+        ("float", [1.0, 2.0, 3.0]),
+        ("int", [1, 2, 3]),
+    ),
+)
+def test_recipe_jsonsplit_type(fresh_db_and_path, type, expected_array):
+    db, db_path = fresh_db_and_path
+    db["example"].insert_all(
+        [
+            {"id": 1, "records": "1,2,3"},
+        ],
+        pk="id",
+    )
+    code = "r.jsonsplit(value)"
+    if type:
+        code = "recipes.jsonsplit(value, type={})".format(type)
+    args = ["convert", db_path, "example", "records", code]
+    result = CliRunner().invoke(cli.cli, args)
+    assert 0 == result.exit_code, result.output
+    assert json.loads(db["example"].get(1)["records"]) == expected_array
+
+
+@pytest.mark.parametrize("drop", (True, False))
+def test_recipe_jsonsplit_output(fresh_db_and_path, drop):
+    db, db_path = fresh_db_and_path
+    db["example"].insert_all(
+        [
+            {"id": 1, "records": "1,2,3"},
+        ],
+        pk="id",
+    )
+    code = "r.jsonsplit(value)"
+    args = ["convert", db_path, "example", "records", code, "--output", "tags"]
+    if drop:
+        args += ["--drop"]
+    result = CliRunner().invoke(cli.cli, args)
+    assert 0 == result.exit_code, result.output
+    expected = {
+        "id": 1,
+        "records": "1,2,3",
+        "tags": '["1", "2", "3"]',
+    }
+    if drop:
+        del expected["records"]
+    assert db["example"].get(1) == expected

tests/test_docs.py

@@ -1,10 +1,12 @@
-from sqlite_utils import cli
+from click.testing import CliRunner
+from sqlite_utils import cli, recipes
 from pathlib import Path
 import pytest
 import re
 
 docs_path = Path(__file__).parent.parent / "docs"
 commands_re = re.compile(r"(?:\$ |    )sqlite-utils (\S+) ")
+recipes_re = re.compile(r"r\.(\w+)\(")
 
 
 @pytest.fixture(scope="session")
@@ -17,11 +19,36 @@ def documented_commands():
     }
 
 
+@pytest.fixture(scope="session")
+def documented_recipes():
+    rst = (docs_path / "cli.rst").read_text()
+    return set(recipes_re.findall(rst))
+
+
 @pytest.mark.parametrize("command", cli.cli.commands.keys())
 def test_commands_are_documented(documented_commands, command):
     assert command in documented_commands
 
 
 @pytest.mark.parametrize("command", cli.cli.commands.values())
-def test_commands_have_docstrings(command):
+def test_commands_have_help(command):
-    assert command.__doc__, "{} is missing a docstring".format(command)
+    assert command.help, "{} is missing its help".format(command)
+
+
+def test_convert_help():
+    result = CliRunner().invoke(cli.cli, ["convert", "--help"])
+    assert result.exit_code == 0
+    for expected in (
+        "r.jsonsplit(value, ",
+        "r.parsedate(value, ",
+        "r.parsedatetime(value, ",
+    ):
+        assert expected in result.output
+
+
+@pytest.mark.parametrize(
+    "recipe",
+    [n for n in dir(recipes) if not n.startswith("_") and n not in ("json", "parser")],
+)
+def test_recipes_are_documented(documented_recipes, recipe):
+    assert recipe in documented_recipes