table.optimize() deletes junk docsize rows

Closes #153. Closes #149.

docs/python-api.rst

@@ -1178,6 +1178,10 @@ Once you have populated a FTS table you can optimize it to dramatically reduce i
 This runs the following SQL::
 
     INSERT INTO dogs_fts (dogs_fts) VALUES ("optimize");
+    DELETE FROM [dogs_fts_docsize] WHERE id NOT IN (
+        SELECT rowid FROM [dogs_fts]);
+
+That ``DELETE`` statement cleans up rows that may have been created by `an obscure bug <https://github.com/simonw/sqlite-utils/issues/153>`__ in previous versions of ``sqlite-utils``.
 
 Creating indexes
 ================

sqlite_utils/db.py

@@ -903,6 +903,16 @@ class Table(Queryable):
                     table=fts_table
                 )
             )
+            self.db.conn.execute(
+                """
+                DELETE FROM [{table}_docsize] WHERE {column} NOT IN (
+                    SELECT rowid FROM [{table}]);
+            """.format(
+                    # FTS5 uses 'id' but FTS4 uses 'docid'
+                    column=self.db["{}_docsize".format(fts_table)].columns[0].name,
+                    table=fts_table,
+                )
+            )
         return self
 
     def search(self, q):

tests/test_optimize.py

@@ -0,0 +1,20 @@
+import pytest
+from sqlite_utils import Database
+import sqlite3
+
+
+@pytest.mark.parametrize("fts_version", ["FTS4", "FTS5"])
+def test_optimizes_removes_junk_docsize_rows(tmpdir, fts_version):
+    # Recreating https://github.com/simonw/sqlite-utils/issues/149
+    path = tmpdir / "test.db"
+    db = Database(str(path), recursive_triggers=False)
+    licenses = [{"key": "apache2", "name": "Apache 2"}, {"key": "bsd", "name": "BSD"}]
+    db["licenses"].insert_all(licenses, pk="key", replace=True)
+    db["licenses"].enable_fts(["name"], create_triggers=True, fts_version=fts_version)
+    assert db["licenses_fts_docsize"].count == 2
+    # Bug: insert with replace increases the number of rows in _docsize:
+    db["licenses"].insert_all(licenses, pk="key", replace=True)
+    assert db["licenses_fts_docsize"].count == 4
+    # Optimize should fix this:
+    db["licenses_fts"].optimize()
+    assert db["licenses_fts_docsize"].count == 2