'sqlite-utils enable-counts' command, closes #214

This commit is contained in:
Simon Willison 2021-01-02 20:26:39 -08:00
commit ce042ff1f0
3 changed files with 84 additions and 0 deletions

View file

@ -1057,6 +1057,23 @@ Use the ``--sql`` option to output the SQL that would be executed, rather than r
order by
[documents_fts].rank
.. _cli_enable_counts:
Enabling cached counts
======================
``select count(*)`` queries can take a long time against large tables. ``sqlite-utils`` can speed these up by adding triggers to maintain a ``_counts`` table, see :ref:`python_api_enable_counts`.
The ``sqlite-utils enable-counts`` command can be used to configure these triggers, either for every table in the database or for specific tables.
::
# Configure triggers for every table in the database
$ sqlite-utils enable-counts mydb.db
# Configure triggers just for specific tables
$ sqlite-utils enable-counts mydb.db table1 table2
.. _cli_vacuum:
Vacuum

View file

@ -548,6 +548,29 @@ def disable_wal(path, load_extension):
db.disable_wal()
@cli.command(name="enable-counts")
@click.argument(
"path",
type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
required=True,
)
@click.argument("tables", nargs=-1)
@load_extension_option
def enable_counts(path, tables, load_extension):
"Configure triggers to update a _counts table with row counts"
db = sqlite_utils.Database(path)
_load_extensions(db, load_extension)
if not tables:
db.enable_counts()
else:
# Check all tables exist
bad_tables = [table for table in tables if not db[table].exists()]
if bad_tables:
raise click.ClickException("Invalid tables: {}".format(bad_tables))
for table in tables:
db[table].enable_counts()
def insert_upsert_options(fn):
for decorator in reversed(
(

View file

@ -1,3 +1,9 @@
from sqlite_utils import Database
from sqlite_utils import cli
from click.testing import CliRunner
import pytest
def test_enable_counts_specific_table(fresh_db):
foo = fresh_db["foo"]
assert fresh_db.table_names() == []
@ -52,3 +58,41 @@ def test_enable_counts_all_tables(fresh_db):
{"count": 1, "table": "foo_fts_docsize"},
{"count": 1, "table": "foo_fts_config"},
]
@pytest.fixture
def counts_db_path(tmpdir):
path = str(tmpdir / "test.db")
db = Database(path)
db["foo"].insert({"name": "Cleo"})
db["bar"].insert({"name": "Cleo"})
return path
@pytest.mark.parametrize(
"extra_args,expected_triggers",
[
(
[],
[
"foo_counts_insert",
"foo_counts_delete",
"bar_counts_insert",
"bar_counts_delete",
],
),
(
["bar"],
[
"bar_counts_insert",
"bar_counts_delete",
],
),
],
)
def test_cli_enable_counts(counts_db_path, extra_args, expected_triggers):
db = Database(counts_db_path)
assert list(db.triggers_dict.keys()) == []
result = CliRunner().invoke(cli.cli, ["enable-counts", counts_db_path] + extra_args)
assert result.exit_code == 0
assert list(db.triggers_dict.keys()) == expected_triggers