remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-07-23 09:24:31 +02:00
Code Issues Projects Releases Packages Wiki Activity

transform: coerce empty strings to NULL when converting TEXT columns to numeric types

Browse source 
When a TEXT column is transformed to INTEGER, FLOAT, or REAL and a row
contains an empty string, the empty string is now converted to NULL during
the INSERT...SELECT copy, matching the expected behavior described in #488.

Fixes #488
This commit is contained in:
Claude 2026-07-18 14:50:50 +00:00
commit 58f5aeb37f
No known key found for this signature in database
2 changed files with 40 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

21
sqlite_utils/db.py
View file

@ -2856,6 +2856,19 @@ class Table(Queryable):
).strip()
)
# Columns being changed from TEXT to a numeric type: coerce empty strings to NULL
_numeric_sql_types = {"INTEGER", "REAL", "FLOAT", "NUMERIC"}
text_to_numeric_cols = {
col_name
for col_name, new_type in types.items()
if existing_columns.get(col_name) == str
and COLUMN_TYPE_MAPPING.get(
new_type,
new_type.upper() if isinstance(new_type, str) else "",
)
in _numeric_sql_types
}
# Copy across data, respecting any renamed columns
new_cols = []
old_cols = []
@ -2866,10 +2879,16 @@ class Table(Queryable):
if "rowid" not in new_cols:
new_cols.insert(0, "rowid")
old_cols.insert(0, "rowid")
def _copy_expr(col):
if col in text_to_numeric_cols:
return "NULLIF({}, '')".format(quote_identifier(col))
return quote_identifier(col)
copy_sql = "INSERT INTO {} ({new_cols})\n SELECT {old_cols} FROM {};".format(
quote_identifier(new_table_name),
quote_identifier(self.name),
old_cols=", ".join(quote_identifier(col) for col in old_cols),
old_cols=", ".join(_copy_expr(col) for col in old_cols),
new_cols=", ".join(quote_identifier(col) for col in new_cols),
)
sqls.append(copy_sql)

23
tests/test_transform.py
View file

@ -23,7 +23,7 @@ import pytest
{"types": {"age": int}},
[
'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "age" INTEGER\n);',
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";',
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";',
'DROP TABLE "dogs";',
'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
],
@ -53,7 +53,7 @@ import pytest
{"types": {"age": int}, "rename": {"age": "dog_age"}},
[
'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER PRIMARY KEY,\n "name" TEXT,\n "dog_age" INTEGER\n);',
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";',
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "dog_age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";',
'DROP TABLE "dogs";',
'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
],
@ -146,7 +146,7 @@ def test_transform_sql_table_with_primary_key(
{"types": {"age": int}},
[
'CREATE TABLE "dogs_new_suffix" (\n "id" INTEGER,\n "name" TEXT,\n "age" INTEGER\n);',
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", "age" FROM "dogs";',
'INSERT INTO "dogs_new_suffix" ("rowid", "id", "name", "age")\n SELECT "rowid", "id", "name", NULLIF("age", \'\') FROM "dogs";',
'DROP TABLE "dogs";',
'ALTER TABLE "dogs_new_suffix" RENAME TO "dogs";',
],
@ -904,3 +904,20 @@ def test_transform_with_unique_constraint_implicit_index(fresh_db):
"You must manually drop this index prior to running this transformation and manually recreate the new index after running this transformation."
in str(excinfo.value)
)
@pytest.mark.parametrize("new_type", [int, float, "integer", "float", "REAL"])
def test_transform_empty_string_to_null_for_numeric_types(fresh_db, new_type):
# Empty strings in TEXT columns should become NULL when transforming to numeric types
fresh_db["test"].insert_all(
[
{"id": 1, "value": "42"},
{"id": 2, "value": ""},
{"id": 3, "value": None},
]
)
fresh_db["test"].transform(types={"value": new_type})
rows = {r["id"]: r["value"] for r in fresh_db["test"].rows}
assert rows[1] in (42, 42.0)
assert rows[2] is None
assert rows[3] is None