diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 18207c8..9e14c1a 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -653,7 +653,6 @@ See :ref:`cli_convert`. --output-type [integer|float|blob|text] Column type to use for the output column --drop Drop original column afterwards - --no-skip-false Don't skip falsey values -s, --silent Don't show a progress bar --pdb Open pdb debugger on first error -h, --help Show this message and exit. diff --git a/docs/cli.rst b/docs/cli.rst index 53a9a5f..3fe2d8e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1711,8 +1711,6 @@ You can include named parameters in your where clause and populate them using on The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database. -By default any rows with a falsey value for the column - such as ``0`` or ``null`` - will be skipped. Use the ``--no-skip-false`` option to disable this behaviour. - .. _cli_convert_import: Importing additional modules diff --git a/docs/python-api.rst b/docs/python-api.rst index bb2b3ee..2e066f3 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1009,8 +1009,6 @@ This will add the new column, if it does not already exist. You can pass ``outpu If you want to drop the original column after saving the results in a separate output column, pass ``drop=True``. -By default any rows with a falsey value for the column - such as ``0`` or ``None`` - will be skipped. Pass ``skip_false=False`` to disable this behaviour. - You can create multiple new columns from a single input column by passing ``multi=True`` and a conversion function that returns a Python dictionary. This example creates new ``upper`` and ``lower`` columns populated from the single ``title`` column: .. code-block:: python diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 8cb3b4f..8025133 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2952,7 +2952,6 @@ def _generate_convert_help(): type=click.Choice(["integer", "float", "blob", "text"]), ) @click.option("--drop", is_flag=True, help="Drop original column afterwards") -@click.option("--no-skip-false", is_flag=True, help="Don't skip falsey values") @click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") @click.option("pdb_", "--pdb", is_flag=True, help="Open pdb debugger on first error") def convert( @@ -2968,7 +2967,6 @@ def convert( output, output_type, drop, - no_skip_false, silent, pdb_, ): @@ -3045,7 +3043,6 @@ def convert( output=output, output_type=output_type, drop=drop, - skip_false=not no_skip_false, multi=multi, show_progress=not silent, ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 04a4c01..5326ae5 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -2909,7 +2909,6 @@ class Table(Queryable): where: Optional[str] = None, where_args: Optional[Union[Iterable, dict]] = None, show_progress: bool = False, - skip_false: bool = True, ): """ Apply conversion function ``fn`` to every value in the specified columns. @@ -2952,8 +2951,6 @@ class Table(Queryable): def convert_value(v): bar.update(1) - if skip_false and not v: - return v return jsonify_if_needed(fn(v)) fn_name = fn.__name__ diff --git a/sqlite_utils/recipes.py b/sqlite_utils/recipes.py index ac41954..e9c0a6e 100644 --- a/sqlite_utils/recipes.py +++ b/sqlite_utils/recipes.py @@ -14,6 +14,8 @@ def parsedate(value, dayfirst=False, yearfirst=False, errors=None): - errors=r.IGNORE to ignore values that cannot be parsed - errors=r.SET_NULL to set values that cannot be parsed to null """ + if not value: + return value try: return ( parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst) @@ -38,6 +40,8 @@ def parsedatetime(value, dayfirst=False, yearfirst=False, errors=None): - errors=r.IGNORE to ignore values that cannot be parsed - errors=r.SET_NULL to set values that cannot be parsed to null """ + if not value: + return value try: return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat() except parser.ParserError: diff --git a/tests/test_cli_convert.py b/tests/test_cli_convert.py index 53c33be..b287280 100644 --- a/tests/test_cli_convert.py +++ b/tests/test_cli_convert.py @@ -80,7 +80,7 @@ def test_convert_import(test_db_and_path): db_path, "example", "dt", - "return re.sub('O..', 'OXX', value)", + "return re.sub('O..', 'OXX', value) if value else value", "--import", "re", ], @@ -223,7 +223,7 @@ def test_convert_output_column(test_db_and_path, drop): db_path, "example", "dt", - "value.replace('October', 'Spooktober')", + "value.replace('October', 'Spooktober') if value else value", "--output", "newcol", ] @@ -628,14 +628,8 @@ def test_convert_initialization_pattern(fresh_db_and_path): ] -@pytest.mark.parametrize( - "no_skip_false,expected", - ( - (True, 1), - (False, 0), - ), -) -def test_convert_no_skip_false(fresh_db_and_path, no_skip_false, expected): +def test_convert_handles_falsey_values(fresh_db_and_path): + # Falsey values like 0 should be converted (issue #527) db, db_path = fresh_db_and_path args = [ "convert", @@ -644,12 +638,10 @@ def test_convert_no_skip_false(fresh_db_and_path, no_skip_false, expected): "x", "-", ] - if no_skip_false: - args.append("--no-skip-false") db["t"].insert_all([{"x": 0}, {"x": 1}]) assert db["t"].get(1)["x"] == 0 assert db["t"].get(2)["x"] == 1 result = CliRunner().invoke(cli.cli, args, input="value + 1") assert result.exit_code == 0, result.output - assert db["t"].get(1)["x"] == expected + assert db["t"].get(1)["x"] == 1 assert db["t"].get(2)["x"] == 2 diff --git a/tests/test_convert.py b/tests/test_convert.py index 5279bc5..848b39a 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -50,12 +50,13 @@ def test_convert_where(fresh_db, where, where_args): assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}] -def test_convert_skip_false(fresh_db): +def test_convert_handles_falsey_values(fresh_db): + # Falsey values like 0 should be converted (issue #527) table = fresh_db["table"] table.insert_all([{"x": 0}, {"x": 1}]) assert table.get(1)["x"] == 0 assert table.get(2)["x"] == 1 - table.convert("x", lambda x: x + 1, skip_false=False) + table.convert("x", lambda x: x + 1) assert table.get(1)["x"] == 1 assert table.get(2)["x"] == 2