sqlite-utils rebuild-fts command, closes #155

docs/cli.rst

@@ -689,6 +689,14 @@ To remove the FTS tables and triggers you created, use ``disable-fts``::
 
     $ sqlite-utils disable-fts mydb.db documents
 
+To rebuild one or more FTS tables (see :ref:`python_api_fts_rebuild`), use ``rebuild-fts``::
+
+    $ sqlite-utils rebuild-fts mydb.db documents
+
+You can rebuild every FTS table by running ``rebuild-fts`` without passing any table names::
+
+    $ sqlite-utils rebuild-fts mydb.db
+
 .. _cli_vacuum:
 
 Vacuum

docs/python-api.rst

@@ -1213,6 +1213,8 @@ To remove the FTS tables and triggers you created, use the ``disable_fts()`` tab
 
     dogs.disable_fts()
 
+.. _python_api_fts_rebuild:
+
 Rebuilding a full-text search table
 ===================================
 
@@ -1232,6 +1234,8 @@ This runs the following SQL::
 
     INSERT INTO dogs_fts (dogs_fts) VALUES ("rebuild");
 
+.. _python_api_fts_optimize:
+
 Optimizing a full-text search table
 ===================================
 

sqlite_utils/cli.py

@@ -232,6 +232,34 @@ def optimize(path, tables, no_vacuum):
         db.vacuum()
 
 
+@cli.command(name="rebuild-fts")
+@click.argument(
+    "path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("tables", nargs=-1)
+def rebuild_fts(path, tables):
+    """Rebuild specific FTS tables, or all FTS tables if none are specified"""
+    db = sqlite_utils.Database(path)
+    if not tables:
+        tables = db.table_names(fts4=True) + db.table_names(fts5=True)
+    with db.conn:
+        for table in tables:
+            db[table].rebuild_fts()
+
+
+@cli.command()
+@click.argument(
+    "path",
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+def vacuum(path):
+    """Run VACUUM against the database"""
+    sqlite_utils.Database(path).vacuum()
+
+
 @cli.command(name="add-column")
 @click.argument(
     "path",

sqlite_utils/db.py

@@ -173,9 +173,9 @@ class Database:
     def table_names(self, fts4=False, fts5=False):
         where = ["type = 'table'"]
         if fts4:
-            where.append("sql like '%FTS4%'")
+            where.append("sql like '%USING FTS4%'")
         if fts5:
-            where.append("sql like '%FTS5%'")
+            where.append("sql like '%USING FTS5%'")
         sql = "select name from sqlite_master where {}".format(" AND ".join(where))
         return [r[0] for r in self.execute(sql).fetchall()]
 

tests/test_cli.py

@@ -481,6 +481,57 @@ def test_optimize(db_path, tables):
     assert 0 == result.exit_code
 
 
+@pytest.mark.parametrize("tables", ([], ["fts4_table"], ["fts5_table"]))
+def test_rebuild_fts(db_path, tables):
+    db = Database(db_path, recursive_triggers=False)
+    records = [
+        {
+            "c1": "verb{}".format(i),
+            "c2": "noun{}".format(i),
+            "c3": "adjective{}".format(i),
+        }
+        for i in range(10000)
+    ]
+    with db.conn:
+        db["fts4_table"].insert_all(records, pk="c1")
+        db["fts5_table"].insert_all(records, pk="c1")
+        db["fts4_table"].enable_fts(
+            ["c1", "c2", "c3"], fts_version="FTS4", create_triggers=True
+        )
+        db["fts5_table"].enable_fts(
+            ["c1", "c2", "c3"], fts_version="FTS5", create_triggers=True
+        )
+    # Search should work
+    assert db["fts4_table"].search("verb1")
+    assert db["fts5_table"].search("verb1")
+    # Deleting _fts_segments to break FTS4
+    with db.conn:
+        db["fts4_table_fts_segments"].delete_where()
+    # Now this should error:
+    with pytest.raises(sqlite3.DatabaseError):
+        db["fts4_table"].search("verb1")
+    # Replicate docsize error from this issue for FTS5
+    # https://github.com/simonw/sqlite-utils/issues/149
+    assert db["fts5_table_fts_docsize"].count == 10000
+    db["fts5_table"].insert_all(records, replace=True)
+    assert db["fts5_table"].count == 10000
+    assert db["fts5_table_fts_docsize"].count == 20000
+    # Running rebuild-fts should fix both issues
+    print(["rebuild-fts", db_path] + tables)
+    result = CliRunner().invoke(cli.cli, ["rebuild-fts", db_path] + tables)
+    assert 0 == result.exit_code
+    fixed_tables = tables or ["fts4_table", "fts5_table"]
+    if "fts4_table" in fixed_tables:
+        assert db["fts4_table"].search("verb1")
+    else:
+        with pytest.raises(sqlite3.DatabaseError):
+            db["fts4_table"].search("verb1")
+    if "fts5_table" in fixed_tables:
+        assert db["fts5_table_fts_docsize"].count == 10000
+    else:
+        assert db["fts5_table_fts_docsize"].count == 20000
+
+
 def test_insert_simple(tmpdir):
     json_path = str(tmpdir / "dog.json")
     db_path = str(tmpdir / "dogs.db")