Better test for rebuild, refs #354

Refs #346 and #344.

.github/workflows/test.yml

@@ -13,6 +13,11 @@ jobs:
         python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
         numpy: [0, 1]
         os: [ubuntu-latest, macos-latest, windows-latest]
+        include:
+        - os: ubuntu-latest
+          python-version: "3.10"
+          numpy: 0
+          pysqlite3_sqlite_3_37: 1
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -32,6 +37,20 @@ jobs:
     - name: Optionally install numpy
       if: matrix.numpy == 1
       run: pip install numpy
+    - name: Optionally install pysqlite3 with SQLite 3.37
+      if: matrix.pysqlite3_sqlite_3_37 == 1
+      run: |-
+        cd /tmp
+        mkdir sqlite-3.37
+        cd sqlite-3.37
+        wget 'https://www.sqlite.org/2021/sqlite-amalgamation-3370000.zip'
+        unzip sqlite-amalgamation-3370000.zip
+        git clone https://github.com/coleifer/pysqlite3
+        cp sqlite-amalgamation-3370000/sqlite3.[ch] pysqlite3
+        cd pysqlite3
+        pip install wheel
+        python setup.py build_static build bdist_wheel
+        pip install /tmp/sqlite-3.37/pysqlite3/dist/*.whl
     - name: Run tests
       run: |
         pytest -v

sqlite_utils/cli.py

@@ -19,6 +19,7 @@ import tabulate
 from .utils import (
     file_progress,
     find_spatialite,
+    iterdump,
     sqlite3,
     decode_base64_values,
     progressbar,
@@ -326,7 +327,7 @@ def dump(path, load_extension):
     """Output a SQL dump of the schema and full contents of the database"""
     db = sqlite_utils.Database(path)
     _load_extensions(db, load_extension)
-    for line in db.conn.iterdump():
+    for line in iterdump(db.conn):
         click.echo(line)
 
 
@@ -1320,7 +1321,7 @@ def memory(
         return
 
     if dump:
-        for line in db.conn.iterdump():
+        for line in iterdump(db.conn):
             click.echo(line)
         return
 
@@ -1330,7 +1331,7 @@ def memory(
 
     if save:
         db2 = sqlite_utils.Database(save)
-        for line in db.conn.iterdump():
+        for line in iterdump(db.conn):
             db2.execute(line)
         return
 

sqlite_utils/utils.py

@@ -5,6 +5,7 @@ import enum
 import io
 import json
 import os
+from sqlite3.dump import _iterdump as iterdump  # type: ignore # noqa: F401
 from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type
 
 import click

tests/test_analyze_tables.py

@@ -1,5 +1,6 @@
 from sqlite_utils.db import Database, ColumnDetails
 from sqlite_utils import cli
+from sqlite_utils.utils import iterdump
 from click.testing import CliRunner
 import pytest
 import sqlite3
@@ -79,7 +80,7 @@ def test_analyze_column(db_to_analyze, column, expected):
 def db_to_analyze_path(db_to_analyze, tmpdir):
     path = str(tmpdir / "test.db")
     db = sqlite3.connect(path)
-    db.executescript("\n".join(db_to_analyze.conn.iterdump()))
+    db.executescript("\n".join(iterdump(db_to_analyze.conn)))
     return path
 
 

tests/test_fts.py

@@ -254,13 +254,12 @@ def test_disable_fts(fresh_db, create_triggers):
     assert ["searchable"] == fresh_db.table_names()
 
 
-@pytest.mark.parametrize("table_to_fix", ["searchable", "searchable_fts"])
-def test_rebuild_fts(fresh_db, table_to_fix):
+def test_rebuild_fts(fresh_db):
     table = fresh_db["searchable"]
     table.insert(search_records[0])
     table.enable_fts(["text", "country"])
     # Run a search
-    rows = list(table.search("tanuki"))
+    rows = list(table.search("are"))
     assert len(rows) == 1
     assert {
         "rowid": 1,
@@ -268,21 +267,14 @@ def test_rebuild_fts(fresh_db, table_to_fix):
         "country": "Japan",
         "not_searchable": "foo",
     }.items() <= rows[0].items()
-    # Delete from searchable_fts_data
-    fresh_db["searchable_fts_data"].delete_where()
-    # This should have broken the index
-    with pytest.raises(sqlite3.DatabaseError):
-        list(table.search("tanuki"))
+    # Insert another record
+    table.insert(search_records[1])
+    # This should NOT show up in a searchs
+    assert len(list(table.search("are"))) == 1
     # Running rebuild_fts() should fix it
-    fresh_db[table_to_fix].rebuild_fts()
-    rows2 = list(table.search("tanuki"))
-    assert len(rows2) == 1
-    assert {
-        "rowid": 1,
-        "text": "tanuki are running tricksters",
-        "country": "Japan",
-        "not_searchable": "foo",
-    }.items() <= rows2[0].items()
+    table.rebuild_fts()
+    rows2 = list(table.search("are"))
+    assert len(rows2) == 2
 
 
 @pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"])