diff --git a/docs/cli.rst b/docs/cli.rst index a9adf63..1136a9e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -676,6 +676,10 @@ A better solution here is to use database triggers. You can set up database trig $ sqlite-utils enable-fts mydb.db documents title summary --create-triggers +To set a custom FTS tokenizer, e.g. to enable Porter stemming, use ``--tokenize=``:: + + $ sqlite-utils populate-fts mydb.db documents title summary --tokenize=porter + To remove the FTS tables and triggers you created, use ``disable-fts``:: $ sqlite-utils disable-fts mydb.db documents diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index c60fd01..ff3f50f 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -325,13 +325,14 @@ def create_index(path, table, column, name, unique, if_not_exists): @click.argument("column", nargs=-1, required=True) @click.option("--fts4", help="Use FTS4", default=False, is_flag=True) @click.option("--fts5", help="Use FTS5", default=False, is_flag=True) +@click.option("--tokenize", help="Tokenizer to use, e.g. porter") @click.option( "--create-triggers", help="Create triggers to update the FTS tables when the parent table changes.", default=False, is_flag=True, ) -def enable_fts(path, table, column, fts4, fts5, create_triggers): +def enable_fts(path, table, column, fts4, fts5, tokenize, create_triggers): "Enable FTS for specific table and columns" fts_version = "FTS5" if fts4 and fts5: @@ -342,7 +343,10 @@ def enable_fts(path, table, column, fts4, fts5, create_triggers): db = sqlite_utils.Database(path) db[table].enable_fts( - column, fts_version=fts_version, create_triggers=create_triggers + column, + fts_version=fts_version, + tokenize=tokenize, + create_triggers=create_triggers, ) diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 7d3a8fe..88f5440 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -776,7 +776,7 @@ class Table(Queryable): table=self.name, columns=", ".join("[{}]".format(c) for c in columns), fts_version=fts_version, - tokenize="\n tokenize='{}',\n".format(tokenize) + tokenize="\n tokenize='{}',".format(tokenize) if tokenize else "", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2156def..22d42e5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -344,25 +344,41 @@ def test_index_foreign_keys(db_path): def test_enable_fts(db_path): - assert None == Database(db_path)["Gosh"].detect_fts() + db = Database(db_path) + assert None == db["Gosh"].detect_fts() result = CliRunner().invoke( cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"] ) assert 0 == result.exit_code - assert "Gosh_fts" == Database(db_path)["Gosh"].detect_fts() + assert "Gosh_fts" == db["Gosh"].detect_fts() # Table names with restricted chars are handled correctly. # colons and dots are restricted characters for table names. - Database(db_path)["http://example.com"].create({"c1": str, "c2": str, "c3": str}) - assert None == Database(db_path)["http://example.com"].detect_fts() + db["http://example.com"].create({"c1": str, "c2": str, "c3": str}) + assert None == db["http://example.com"].detect_fts() result = CliRunner().invoke( - cli.cli, ["enable-fts", db_path, "http://example.com", "c1", "--fts4"] + cli.cli, + [ + "enable-fts", + db_path, + "http://example.com", + "c1", + "--fts4", + "--tokenize", + "porter", + ], ) assert 0 == result.exit_code + assert "http://example.com_fts" == db["http://example.com"].detect_fts() + # Check tokenize was set to porter assert ( - "http://example.com_fts" == Database(db_path)["http://example.com"].detect_fts() - ) - Database(db_path)["http://example.com"].drop() + "CREATE VIRTUAL TABLE [http://example.com_fts] USING FTS4 (\n" + " [c1],\n" + " tokenize='porter',\n" + " content=[http://example.com]" + "\n )" + ) == db["http://example.com_fts"].schema + db["http://example.com"].drop() def test_enable_fts_with_triggers(db_path):