sqlite-utils populate-fts --tokenize= option, closes #130

docs/cli.rst

@@ -676,6 +676,10 @@ A better solution here is to use database triggers. You can set up database trig
 
     $ sqlite-utils enable-fts mydb.db documents title summary --create-triggers
 
+To set a custom FTS tokenizer, e.g. to enable Porter stemming, use ``--tokenize=``::
+
+    $ sqlite-utils populate-fts mydb.db documents title summary --tokenize=porter
+
 To remove the FTS tables and triggers you created, use ``disable-fts``::
 
     $ sqlite-utils disable-fts mydb.db documents

sqlite_utils/cli.py

@@ -325,13 +325,14 @@ def create_index(path, table, column, name, unique, if_not_exists):
 @click.argument("column", nargs=-1, required=True)
 @click.option("--fts4", help="Use FTS4", default=False, is_flag=True)
 @click.option("--fts5", help="Use FTS5", default=False, is_flag=True)
+@click.option("--tokenize", help="Tokenizer to use, e.g. porter")
 @click.option(
     "--create-triggers",
     help="Create triggers to update the FTS tables when the parent table changes.",
     default=False,
     is_flag=True,
 )
-def enable_fts(path, table, column, fts4, fts5, create_triggers):
+def enable_fts(path, table, column, fts4, fts5, tokenize, create_triggers):
     "Enable FTS for specific table and columns"
     fts_version = "FTS5"
     if fts4 and fts5:
@@ -342,7 +343,10 @@ def enable_fts(path, table, column, fts4, fts5, create_triggers):
 
     db = sqlite_utils.Database(path)
     db[table].enable_fts(
-        column, fts_version=fts_version, create_triggers=create_triggers
+        column,
+        fts_version=fts_version,
+        tokenize=tokenize,
+        create_triggers=create_triggers,
     )
 
 

sqlite_utils/db.py

@@ -776,7 +776,7 @@ class Table(Queryable):
             table=self.name,
             columns=", ".join("[{}]".format(c) for c in columns),
             fts_version=fts_version,
-            tokenize="\n                tokenize='{}',\n".format(tokenize)
+            tokenize="\n                tokenize='{}',".format(tokenize)
             if tokenize
             else "",
         )

tests/test_cli.py

@@ -344,25 +344,41 @@ def test_index_foreign_keys(db_path):
 
 
 def test_enable_fts(db_path):
-    assert None == Database(db_path)["Gosh"].detect_fts()
+    db = Database(db_path)
+    assert None == db["Gosh"].detect_fts()
     result = CliRunner().invoke(
         cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"]
     )
     assert 0 == result.exit_code
-    assert "Gosh_fts" == Database(db_path)["Gosh"].detect_fts()
+    assert "Gosh_fts" == db["Gosh"].detect_fts()
 
     # Table names with restricted chars are handled correctly.
     # colons and dots are restricted characters for table names.
-    Database(db_path)["http://example.com"].create({"c1": str, "c2": str, "c3": str})
-    assert None == Database(db_path)["http://example.com"].detect_fts()
+    db["http://example.com"].create({"c1": str, "c2": str, "c3": str})
+    assert None == db["http://example.com"].detect_fts()
     result = CliRunner().invoke(
-        cli.cli, ["enable-fts", db_path, "http://example.com", "c1", "--fts4"]
+        cli.cli,
+        [
+            "enable-fts",
+            db_path,
+            "http://example.com",
+            "c1",
+            "--fts4",
+            "--tokenize",
+            "porter",
+        ],
     )
     assert 0 == result.exit_code
+    assert "http://example.com_fts" == db["http://example.com"].detect_fts()
+    # Check tokenize was set to porter
     assert (
-        "http://example.com_fts" == Database(db_path)["http://example.com"].detect_fts()
-    )
-    Database(db_path)["http://example.com"].drop()
+        "CREATE VIRTUAL TABLE [http://example.com_fts] USING FTS4 (\n"
+        "                [c1],\n"
+        "                tokenize='porter',\n"
+        "                content=[http://example.com]"
+        "\n            )"
+    ) == db["http://example.com_fts"].schema
+    db["http://example.com"].drop()
 
 
 def test_enable_fts_with_triggers(db_path):