sqlite-utils convert can now use callable references, closes #686

This now works:

    echo '{"date": "13th January 2025"}' | sqlite-utils insert dates.db dates -
    sqlite-utils convert dates.db dates date r.parsedate
    sqlite-utils rows dates.db dates

Previously the command would have been:

    sqlite-utils convert dates.db dates date 'r.parsedate(value)'
This commit is contained in:
Simon Willison 2025-11-29 14:43:02 -08:00 committed by GitHub
commit 5a7f522980
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 74 additions and 3 deletions

View file

@ -1849,7 +1849,19 @@ These recipes can be used in the code passed to ``sqlite-utils convert`` like th
sqlite-utils convert my.db mytable mycolumn \
'r.jsonsplit(value)'
To use any of the documented parameters, do this:
You can also pass the recipe function directly without the ``(value)`` part - sqlite-utils will detect that it is a callable and use it automatically:
.. code-block:: bash
sqlite-utils convert my.db mytable mycolumn r.parsedate
This shorter syntax works for any callable, including functions from imported modules:
.. code-block:: bash
sqlite-utils convert my.db mytable mycolumn json.loads --import json
To use any of the documented parameters, use the full function call syntax:
.. code-block:: bash

View file

@ -450,6 +450,10 @@ def progressbar(*args, **kwargs):
def _compile_code(code, imports, variable="value"):
globals = {"r": recipes, "recipes": recipes}
# Handle imports first so they're available for all approaches
for import_ in imports:
globals[import_.split(".")[0]] = __import__(import_)
# If user defined a convert() function, return that
try:
exec(code, globals)
@ -457,6 +461,15 @@ def _compile_code(code, imports, variable="value"):
except (AttributeError, SyntaxError, NameError, KeyError, TypeError):
pass
# Check if code is a direct callable reference
# e.g. "r.parsedate" instead of "r.parsedate(value)"
try:
fn = eval(code, globals)
if callable(fn):
return fn
except Exception:
pass
# Try compiling their code as a function instead
body_variants = [code]
# If single line and no 'return', try adding the return
@ -478,8 +491,6 @@ def _compile_code(code, imports, variable="value"):
if code_o is None:
raise SyntaxError("Could not compile code")
for import_ in imports:
globals[import_.split(".")[0]] = __import__(import_)
exec(code_o, globals)
return globals["fn"]

View file

@ -645,3 +645,51 @@ def test_convert_handles_falsey_values(fresh_db_and_path):
assert result.exit_code == 0, result.output
assert db["t"].get(1)["x"] == 1
assert db["t"].get(2)["x"] == 2
@pytest.mark.parametrize(
"code",
[
# Direct callable reference (issue #686)
"r.parsedate",
"recipes.parsedate",
# Traditional call syntax still works
"r.parsedate(value)",
"recipes.parsedate(value)",
],
)
def test_convert_callable_reference(test_db_and_path, code):
"""Test that callable references like r.parsedate work without (value)"""
db, db_path = test_db_and_path
result = CliRunner().invoke(
cli.cli, ["convert", db_path, "example", "dt", code], catch_exceptions=False
)
assert result.exit_code == 0, result.output
rows = list(db["example"].rows)
assert rows[0]["dt"] == "2019-10-05"
assert rows[1]["dt"] == "2019-10-06"
assert rows[2]["dt"] == ""
assert rows[3]["dt"] is None
def test_convert_callable_reference_with_import(fresh_db_and_path):
"""Test callable reference from an imported module"""
db, db_path = fresh_db_and_path
db["example"].insert({"id": 1, "data": '{"name": "test"}'})
result = CliRunner().invoke(
cli.cli,
[
"convert",
db_path,
"example",
"data",
"json.loads",
"--import",
"json",
],
catch_exceptions=False,
)
assert result.exit_code == 0, result.output
# json.loads returns a dict, which sqlite stores as JSON string
row = db["example"].get(1)
assert row["data"] == '{"name": "test"}'