Replace SQL prefix matching with a first-token scanner

query() and execute() classified statements using
sql.lstrip().upper().startswith(...), which a leading SQL comment
defeated: db.query('/* c */ COMMIT') inside db.atomic() committed
the caller's transaction and masked the ValueError with a 'no such
savepoint' error, comment-prefixed PRAGMAs ran inside the savepoint
guard where journal mode changes are refused, and a comment-prefixed
BEGIN passed to db.execute() was instantly auto-committed.

The new _first_keyword() helper skips leading whitespace and -- or
/* */ comments - the only things SQLite's tokenizer allows before
the first token - and returns that token for exact comparison, so
keyword detection now matches what SQLite itself will see.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Simon Willison 2026-07-05 11:49:59 -07:00
commit 87cf1c5a00
3 changed files with 116 additions and 9 deletions

View file

@ -247,6 +247,17 @@ def test_execute_write_respects_explicit_transaction(fresh_db):
assert [r["id"] for r in fresh_db["t"].rows] == [1]
def test_execute_comment_prefixed_begin_leaves_transaction_open(fresh_db):
# A BEGIN hidden behind a leading comment must not be auto-committed
# out from under the caller
fresh_db["t"].insert({"id": 1}, pk="id")
fresh_db.execute("-- start a transaction\nbegin")
assert fresh_db.conn.in_transaction
fresh_db.execute("insert into t (id) values (2)")
fresh_db.rollback()
assert [r["id"] for r in fresh_db["t"].rows] == [1]
def test_query_returning_commits_after_iteration(tmpdir):
if sqlite3.sqlite_version_info < (3, 35, 0):
import pytest as _pytest

View file

@ -48,7 +48,18 @@ def test_query_rejected_write_inside_transaction_is_rolled_back(fresh_db):
@pytest.mark.parametrize(
"sql", ["begin", "commit", "rollback", "vacuum", "detach database foo"]
"sql",
[
"begin",
"commit",
"rollback",
"vacuum",
"detach database foo",
"/* comment */ commit",
"-- comment\nbegin",
"/* multi\nline */ -- and another\n vacuum",
"\t /* a */ /* b */ savepoint s1",
],
)
def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql):
with pytest.raises(ValueError) as ex:
@ -57,6 +68,21 @@ def test_query_rejects_transaction_control_and_vacuum(fresh_db, sql):
assert not fresh_db.conn.in_transaction
def test_query_comment_prefixed_commit_does_not_commit_transaction(fresh_db):
# A COMMIT hidden behind a leading comment must not slip past the
# keyword check - previously it committed the caller's open
# transaction before the ValueError was raised
fresh_db["dogs"].insert({"name": "Cleo"})
fresh_db.begin()
fresh_db.execute("insert into dogs (name) values ('Pancakes')")
with pytest.raises(ValueError):
fresh_db.query("/* comment */ COMMIT")
# The explicit transaction is still open and can still be rolled back
assert fresh_db.conn.in_transaction
fresh_db.rollback()
assert [row["name"] for row in fresh_db["dogs"].rows] == ["Cleo"]
def test_query_error_leaves_no_transaction_open(fresh_db):
with pytest.raises(sqlite3.OperationalError):
fresh_db.query("select * from missing_table")
@ -75,6 +101,41 @@ def test_query_pragma(tmpdir):
db.close()
def test_query_comment_prefixed_pragma(tmpdir):
from sqlite_utils import Database
db = Database(str(tmpdir / "test.db"))
# A leading comment must not stop a PRAGMA being recognized as one -
# previously it was executed inside the savepoint guard, where
# journal mode changes are refused
assert list(db.query("-- set WAL mode\npragma journal_mode = wal")) == [
{"journal_mode": "wal"}
]
db.close()
@pytest.mark.parametrize(
"sql,expected",
[
("select 1", "SELECT"),
(" \t\n select 1", "SELECT"),
("-- comment\nbegin", "BEGIN"),
("/* one */ /* two */ pragma user_version", "PRAGMA"),
("/* multi\nline */vacuum", "VACUUM"),
("insert into t values (1)", "INSERT"),
("-- only a comment", ""),
("/* unterminated", ""),
("", ""),
(" ", ""),
("123", ""),
],
)
def test_first_keyword(sql, expected):
from sqlite_utils.db import _first_keyword
assert _first_keyword(sql) == expected
@pytest.mark.skipif(
sqlite3.sqlite_version_info < (3, 35, 0),
reason="RETURNING requires SQLite 3.35.0 or higher",