diff --git a/docs/cli.rst b/docs/cli.rst index 100f3e0..5eeb507 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -689,6 +689,14 @@ To remove the FTS tables and triggers you created, use ``disable-fts``:: $ sqlite-utils disable-fts mydb.db documents +To rebuild one or more FTS tables (see :ref:`python_api_fts_rebuild`), use ``rebuild-fts``:: + + $ sqlite-utils rebuild-fts mydb.db documents + +You can rebuild every FTS table by running ``rebuild-fts`` without passing any table names:: + + $ sqlite-utils rebuild-fts mydb.db + .. _cli_vacuum: Vacuum diff --git a/docs/python-api.rst b/docs/python-api.rst index 2564376..9404c78 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1213,6 +1213,8 @@ To remove the FTS tables and triggers you created, use the ``disable_fts()`` tab dogs.disable_fts() +.. _python_api_fts_rebuild: + Rebuilding a full-text search table =================================== @@ -1232,6 +1234,8 @@ This runs the following SQL:: INSERT INTO dogs_fts (dogs_fts) VALUES ("rebuild"); +.. _python_api_fts_optimize: + Optimizing a full-text search table =================================== diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index beb9943..4a01cc8 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -232,6 +232,34 @@ def optimize(path, tables, no_vacuum): db.vacuum() +@cli.command(name="rebuild-fts") +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("tables", nargs=-1) +def rebuild_fts(path, tables): + """Rebuild specific FTS tables, or all FTS tables if none are specified""" + db = sqlite_utils.Database(path) + if not tables: + tables = db.table_names(fts4=True) + db.table_names(fts5=True) + with db.conn: + for table in tables: + db[table].rebuild_fts() + + +@cli.command() +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +def vacuum(path): + """Run VACUUM against the database""" + sqlite_utils.Database(path).vacuum() + + @cli.command(name="add-column") @click.argument( "path", diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index f535f6b..f938ef4 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -173,9 +173,9 @@ class Database: def table_names(self, fts4=False, fts5=False): where = ["type = 'table'"] if fts4: - where.append("sql like '%FTS4%'") + where.append("sql like '%USING FTS4%'") if fts5: - where.append("sql like '%FTS5%'") + where.append("sql like '%USING FTS5%'") sql = "select name from sqlite_master where {}".format(" AND ".join(where)) return [r[0] for r in self.execute(sql).fetchall()] diff --git a/tests/test_cli.py b/tests/test_cli.py index 5c4404a..0a8c0ba 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -481,6 +481,57 @@ def test_optimize(db_path, tables): assert 0 == result.exit_code +@pytest.mark.parametrize("tables", ([], ["fts4_table"], ["fts5_table"])) +def test_rebuild_fts(db_path, tables): + db = Database(db_path, recursive_triggers=False) + records = [ + { + "c1": "verb{}".format(i), + "c2": "noun{}".format(i), + "c3": "adjective{}".format(i), + } + for i in range(10000) + ] + with db.conn: + db["fts4_table"].insert_all(records, pk="c1") + db["fts5_table"].insert_all(records, pk="c1") + db["fts4_table"].enable_fts( + ["c1", "c2", "c3"], fts_version="FTS4", create_triggers=True + ) + db["fts5_table"].enable_fts( + ["c1", "c2", "c3"], fts_version="FTS5", create_triggers=True + ) + # Search should work + assert db["fts4_table"].search("verb1") + assert db["fts5_table"].search("verb1") + # Deleting _fts_segments to break FTS4 + with db.conn: + db["fts4_table_fts_segments"].delete_where() + # Now this should error: + with pytest.raises(sqlite3.DatabaseError): + db["fts4_table"].search("verb1") + # Replicate docsize error from this issue for FTS5 + # https://github.com/simonw/sqlite-utils/issues/149 + assert db["fts5_table_fts_docsize"].count == 10000 + db["fts5_table"].insert_all(records, replace=True) + assert db["fts5_table"].count == 10000 + assert db["fts5_table_fts_docsize"].count == 20000 + # Running rebuild-fts should fix both issues + print(["rebuild-fts", db_path] + tables) + result = CliRunner().invoke(cli.cli, ["rebuild-fts", db_path] + tables) + assert 0 == result.exit_code + fixed_tables = tables or ["fts4_table", "fts5_table"] + if "fts4_table" in fixed_tables: + assert db["fts4_table"].search("verb1") + else: + with pytest.raises(sqlite3.DatabaseError): + db["fts4_table"].search("verb1") + if "fts5_table" in fixed_tables: + assert db["fts5_table_fts_docsize"].count == 10000 + else: + assert db["fts5_table_fts_docsize"].count == 20000 + + def test_insert_simple(tmpdir): json_path = str(tmpdir / "dog.json") db_path = str(tmpdir / "dogs.db")