mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-11-20 17:09:46 +01:00
sqlite-utils populate-fts --tokenize= option, closes #130
This commit is contained in:
parent
617e6f070c
commit
57e4eb8e55
4 changed files with 35 additions and 11 deletions
|
|
@ -676,6 +676,10 @@ A better solution here is to use database triggers. You can set up database trig
|
||||||
|
|
||||||
$ sqlite-utils enable-fts mydb.db documents title summary --create-triggers
|
$ sqlite-utils enable-fts mydb.db documents title summary --create-triggers
|
||||||
|
|
||||||
|
To set a custom FTS tokenizer, e.g. to enable Porter stemming, use ``--tokenize=``::
|
||||||
|
|
||||||
|
$ sqlite-utils populate-fts mydb.db documents title summary --tokenize=porter
|
||||||
|
|
||||||
To remove the FTS tables and triggers you created, use ``disable-fts``::
|
To remove the FTS tables and triggers you created, use ``disable-fts``::
|
||||||
|
|
||||||
$ sqlite-utils disable-fts mydb.db documents
|
$ sqlite-utils disable-fts mydb.db documents
|
||||||
|
|
|
||||||
|
|
@ -325,13 +325,14 @@ def create_index(path, table, column, name, unique, if_not_exists):
|
||||||
@click.argument("column", nargs=-1, required=True)
|
@click.argument("column", nargs=-1, required=True)
|
||||||
@click.option("--fts4", help="Use FTS4", default=False, is_flag=True)
|
@click.option("--fts4", help="Use FTS4", default=False, is_flag=True)
|
||||||
@click.option("--fts5", help="Use FTS5", default=False, is_flag=True)
|
@click.option("--fts5", help="Use FTS5", default=False, is_flag=True)
|
||||||
|
@click.option("--tokenize", help="Tokenizer to use, e.g. porter")
|
||||||
@click.option(
|
@click.option(
|
||||||
"--create-triggers",
|
"--create-triggers",
|
||||||
help="Create triggers to update the FTS tables when the parent table changes.",
|
help="Create triggers to update the FTS tables when the parent table changes.",
|
||||||
default=False,
|
default=False,
|
||||||
is_flag=True,
|
is_flag=True,
|
||||||
)
|
)
|
||||||
def enable_fts(path, table, column, fts4, fts5, create_triggers):
|
def enable_fts(path, table, column, fts4, fts5, tokenize, create_triggers):
|
||||||
"Enable FTS for specific table and columns"
|
"Enable FTS for specific table and columns"
|
||||||
fts_version = "FTS5"
|
fts_version = "FTS5"
|
||||||
if fts4 and fts5:
|
if fts4 and fts5:
|
||||||
|
|
@ -342,7 +343,10 @@ def enable_fts(path, table, column, fts4, fts5, create_triggers):
|
||||||
|
|
||||||
db = sqlite_utils.Database(path)
|
db = sqlite_utils.Database(path)
|
||||||
db[table].enable_fts(
|
db[table].enable_fts(
|
||||||
column, fts_version=fts_version, create_triggers=create_triggers
|
column,
|
||||||
|
fts_version=fts_version,
|
||||||
|
tokenize=tokenize,
|
||||||
|
create_triggers=create_triggers,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -776,7 +776,7 @@ class Table(Queryable):
|
||||||
table=self.name,
|
table=self.name,
|
||||||
columns=", ".join("[{}]".format(c) for c in columns),
|
columns=", ".join("[{}]".format(c) for c in columns),
|
||||||
fts_version=fts_version,
|
fts_version=fts_version,
|
||||||
tokenize="\n tokenize='{}',\n".format(tokenize)
|
tokenize="\n tokenize='{}',".format(tokenize)
|
||||||
if tokenize
|
if tokenize
|
||||||
else "",
|
else "",
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -344,25 +344,41 @@ def test_index_foreign_keys(db_path):
|
||||||
|
|
||||||
|
|
||||||
def test_enable_fts(db_path):
|
def test_enable_fts(db_path):
|
||||||
assert None == Database(db_path)["Gosh"].detect_fts()
|
db = Database(db_path)
|
||||||
|
assert None == db["Gosh"].detect_fts()
|
||||||
result = CliRunner().invoke(
|
result = CliRunner().invoke(
|
||||||
cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"]
|
cli.cli, ["enable-fts", db_path, "Gosh", "c1", "--fts4"]
|
||||||
)
|
)
|
||||||
assert 0 == result.exit_code
|
assert 0 == result.exit_code
|
||||||
assert "Gosh_fts" == Database(db_path)["Gosh"].detect_fts()
|
assert "Gosh_fts" == db["Gosh"].detect_fts()
|
||||||
|
|
||||||
# Table names with restricted chars are handled correctly.
|
# Table names with restricted chars are handled correctly.
|
||||||
# colons and dots are restricted characters for table names.
|
# colons and dots are restricted characters for table names.
|
||||||
Database(db_path)["http://example.com"].create({"c1": str, "c2": str, "c3": str})
|
db["http://example.com"].create({"c1": str, "c2": str, "c3": str})
|
||||||
assert None == Database(db_path)["http://example.com"].detect_fts()
|
assert None == db["http://example.com"].detect_fts()
|
||||||
result = CliRunner().invoke(
|
result = CliRunner().invoke(
|
||||||
cli.cli, ["enable-fts", db_path, "http://example.com", "c1", "--fts4"]
|
cli.cli,
|
||||||
|
[
|
||||||
|
"enable-fts",
|
||||||
|
db_path,
|
||||||
|
"http://example.com",
|
||||||
|
"c1",
|
||||||
|
"--fts4",
|
||||||
|
"--tokenize",
|
||||||
|
"porter",
|
||||||
|
],
|
||||||
)
|
)
|
||||||
assert 0 == result.exit_code
|
assert 0 == result.exit_code
|
||||||
|
assert "http://example.com_fts" == db["http://example.com"].detect_fts()
|
||||||
|
# Check tokenize was set to porter
|
||||||
assert (
|
assert (
|
||||||
"http://example.com_fts" == Database(db_path)["http://example.com"].detect_fts()
|
"CREATE VIRTUAL TABLE [http://example.com_fts] USING FTS4 (\n"
|
||||||
)
|
" [c1],\n"
|
||||||
Database(db_path)["http://example.com"].drop()
|
" tokenize='porter',\n"
|
||||||
|
" content=[http://example.com]"
|
||||||
|
"\n )"
|
||||||
|
) == db["http://example.com_fts"].schema
|
||||||
|
db["http://example.com"].drop()
|
||||||
|
|
||||||
|
|
||||||
def test_enable_fts_with_triggers(db_path):
|
def test_enable_fts_with_triggers(db_path):
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue