sqlite-utils populate-fts --tokenize= option, closes #130

This commit is contained in:
Simon Willison 2020-08-01 13:51:05 -07:00
commit 57e4eb8e55
4 changed files with 35 additions and 11 deletions

View file

@ -676,6 +676,10 @@ A better solution here is to use database triggers. You can set up database trig
$ sqlite-utils enable-fts mydb.db documents title summary --create-triggers $ sqlite-utils enable-fts mydb.db documents title summary --create-triggers
To set a custom FTS tokenizer, e.g. to enable Porter stemming, use ``--tokenize=``::
$ sqlite-utils populate-fts mydb.db documents title summary --tokenize=porter
To remove the FTS tables and triggers you created, use ``disable-fts``:: To remove the FTS tables and triggers you created, use ``disable-fts``::
$ sqlite-utils disable-fts mydb.db documents $ sqlite-utils disable-fts mydb.db documents

View file

@ -325,13 +325,14 @@ def create_index(path, table, column, name, unique, if_not_exists):
@click.argument("column", nargs=-1, required=True) @click.argument("column", nargs=-1, required=True)
@click.option("--fts4", help="Use FTS4", default=False, is_flag=True) @click.option("--fts4", help="Use FTS4", default=False, is_flag=True)
@click.option("--fts5", help="Use FTS5", default=False, is_flag=True) @click.option("--fts5", help="Use FTS5", default=False, is_flag=True)
@click.option("--tokenize", help="Tokenizer to use, e.g. porter")
@click.option( @click.option(
"--create-triggers", "--create-triggers",
help="Create triggers to update the FTS tables when the parent table changes.", help="Create triggers to update the FTS tables when the parent table changes.",
default=False, default=False,
is_flag=True, is_flag=True,
) )
def enable_fts(path, table, column, fts4, fts5, create_triggers): def enable_fts(path, table, column, fts4, fts5, tokenize, create_triggers):
"Enable FTS for specific table and columns" "Enable FTS for specific table and columns"
fts_version = "FTS5" fts_version = "FTS5"
if fts4 and fts5: if fts4 and fts5:
@ -342,7 +343,10 @@ def enable_fts(path, table, column, fts4, fts5, create_triggers):
db = sqlite_utils.Database(path) db = sqlite_utils.Database(path)
db[table].enable_fts( db[table].enable_fts(
column, fts_version=fts_version, create_triggers=create_triggers column,
fts_version=fts_version,
tokenize=tokenize,
create_triggers=create_triggers,
) )

View file

@ -776,7 +776,7 @@ class Table(Queryable):
table=self.name, table=self.name,
columns=", ".join("[{}]".format(c) for c in columns), columns=", ".join("[{}]".format(c) for c in columns),
fts_version=fts_version, fts_version=fts_version,
tokenize="\n tokenize='{}',\n".format(tokenize) tokenize="\n tokenize='{}',".format(tokenize)
if tokenize if tokenize
else "", else "",
) )

View file

@ -344,25 +344,41 @@ def test_index_foreign_keys(db_path):
def test_enable_fts(db_path): def test_enable_fts(db_path):
assert None == Database(db_path)["Gosh"].detect_fts() db = Database(db_path)
assert None == db["Gosh"].detect_fts()
result = CliRunner().invoke( result = CliRunner().invoke(
cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"]
) )
assert 0 == result.exit_code assert 0 == result.exit_code
assert "Gosh_fts" == Database(db_path)["Gosh"].detect_fts() assert "Gosh_fts" == db["Gosh"].detect_fts()
# Table names with restricted chars are handled correctly. # Table names with restricted chars are handled correctly.
# colons and dots are restricted characters for table names. # colons and dots are restricted characters for table names.
Database(db_path)["http://example.com"].create({"c1": str, "c2": str, "c3": str}) db["http://example.com"].create({"c1": str, "c2": str, "c3": str})
assert None == Database(db_path)["http://example.com"].detect_fts() assert None == db["http://example.com"].detect_fts()
result = CliRunner().invoke( result = CliRunner().invoke(
cli.cli, ["enable-fts", db_path, "http://example.com", "c1", "--fts4"] cli.cli,
[
"enable-fts",
db_path,
"http://example.com",
"c1",
"--fts4",
"--tokenize",
"porter",
],
) )
assert 0 == result.exit_code assert 0 == result.exit_code
assert "http://example.com_fts" == db["http://example.com"].detect_fts()
# Check tokenize was set to porter
assert ( assert (
"http://example.com_fts" == Database(db_path)["http://example.com"].detect_fts() "CREATE VIRTUAL TABLE [http://example.com_fts] USING FTS4 (\n"
) " [c1],\n"
Database(db_path)["http://example.com"].drop() " tokenize='porter',\n"
" content=[http://example.com]"
"\n )"
) == db["http://example.com_fts"].schema
db["http://example.com"].drop()
def test_enable_fts_with_triggers(db_path): def test_enable_fts_with_triggers(db_path):