diff --git a/docs/changelog.rst b/docs/changelog.rst index f14870b..9b6b303 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,27 @@ Changelog =========== +.. _v3_21: + +3.21 (2022-01-10) +----------------- + +CLI and Python library improvements to help run `ANALYZE `__ after creating indexes or inserting rows, to gain better performance from the SQLite query planner when it runs against indexes. + +Three new CLI commands: ``create-database``, ``analyze`` and ``bulk``. + +- New ``sqlite-utils create-database`` command for creating new empty database files. (:issue:`348`) +- New Python methods for running ``ANALYZE`` against a database, table or index: ``db.analyze()`` and ``table.analyze()``, see :ref:`python_api_analyze`. (:issue:`366`) +- New :ref:`sqlite-utils analyze command ` for running ``ANALYZE`` using the CLI. (:issue:`379`) +- The ``create-index``, ``insert`` and ``update`` commands now have a new ``--analyze`` option for running ``ANALYZE`` after the command has completed. (:issue:`379`) +- New :ref:`sqlite-utils bulk command ` which can import records in the same way as ``sqlite-utils insert`` (from JSON, CSV or TSV) and use them to bulk execute a parametrized SQL query. (:issue:`375`) +- The CLI tool can now also be run using ``python -m sqlite_utils``. (:issue:`368`) +- Using ``--fmt`` now implies ``--table``, so you don't need to pass both options. (:issue:`374`) +- The ``--convert`` function applied to rows can now modify the row in place. (:issue:`371`) +- The :ref:`insert-files command ` supports two new columns: ``stem`` and ``suffix``. (:issue:`372`) +- The ``--nl`` import option now ignores blank lines in the input. (:issue:`376`) +- Fixed bug where streaming input to the ``insert`` command with ``--batch-size 1`` would appear to only commit after several rows had been ingested, due to unnecessary input buffering. (:issue:`364`) + .. _v3_20: 3.20 (2022-01-05) diff --git a/docs/cli.rst b/docs/cli.rst index 36ff029..913a9ed 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -761,6 +761,8 @@ You can delete all the existing rows in the table before inserting the new recor $ sqlite-utils insert dogs.db dogs dogs.json --truncate +You can add the ``--analyze`` option to run ``ANALYZE`` against the table after the rows have been inserted. + .. _cli_inserting_data_binary: Inserting binary data @@ -1076,6 +1078,36 @@ The command will fail if you reference columns that do not exist on the table. T .. note:: ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 `__ for details of this change. + +.. _cli_bulk: + +Executing SQL in bulk +===================== + +If you have a JSON, newline-delimited JSON, CSV or TSV file you can execute a bulk SQL query using each of the records in that file using the ``sqlite-utils bulk`` command. + +The command takes the database file, the SQL to be executed and the file containing records to be used when evaluating the SQL query. + +The SQL query should include ``:named`` parameters that match the keys in the records. + +For example, given a ``chickens.csv`` CSV file containing the following:: + + id,name + 1,Blue + 2,Snowy + 3,Azi + 4,Lila + 5,Suna + 6,Cardi + +You could insert those rows into a pre-created ``chickens`` table like so:: + + $ sqlite-utils bulk chickens.db \ + 'insert into chickens (id, name) values (:id, :name)' \ + chickens.csv --csv + +This command takes the same options as the ``sqlite-utils insert`` command - so it defaults to expecting JSON but can accept other formats using ``--csv`` or ``--tsv`` or ``--nl`` or other options described above. + .. _cli_insert_files: Inserting data from files @@ -1697,6 +1729,8 @@ This will create an index on that table on ``(col1, col2 desc, col3)``. If your column names are already prefixed with a hyphen you'll need to manually execute a ``CREATE INDEX`` SQL statement to add indexes to them rather than using this tool. +Add the ``--analyze`` option to run ``ANALYZE`` against the index after it has been created. + .. _cli_fts: Configuring full-text search @@ -1800,6 +1834,25 @@ If the ``_counts`` table ever becomes out-of-sync with the actual table counts y $ sqlite-utils reset-counts mydb.db +.. _cli_analyze: + +Optimizing index usage with ANALYZE +=================================== + +The `SQLite ANALYZE command `__ builds a table of statistics which the query planner can use to make better decisions about which indexes to use for a given query. + +You should run ``ANALYZE`` if your database is large and you do not think your indexes are being efficiently used. + +To run ``ANALYZE`` against every index in a database, use this:: + + $ sqlite-utils analyze mydb.db + +You can run it against specific tables, or against specific named indexes, by passing them as optional arguments:: + + $ sqlite-utils analyze mydb.db mytable idx_mytable_name + +You can also run ``ANALYZE`` as part of another command using the ``--analyze`` option. This is supported by the ``create-index``, ``insert`` and ``upsert`` commands. + .. _cli_vacuum: Vacuum diff --git a/docs/python-api.rst b/docs/python-api.rst index 5d391ee..6b0d542 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -638,8 +638,9 @@ The function can accept an iterator or generator of rows and will commit them ac You can skip inserting any records that have a primary key that already exists using ``ignore=True``. This works with both ``.insert({...}, ignore=True)`` and ``.insert_all([...], ignore=True)``. -You can delete all the existing rows in the table before inserting the new -records using ``truncate=True``. This is useful if you want to replace the data in the table. +You can delete all the existing rows in the table before inserting the new records using ``truncate=True``. This is useful if you want to replace the data in the table. + +Pass ``analyze=True`` to run ``ANALYZE`` against the table after inserting the new records. .. _python_api_insert_replace: @@ -716,6 +717,8 @@ You can delete all records in a table that match a specific WHERE statement usin Calling ``table.delete_where()`` with no other arguments will delete every row in the table. +Pass ``analyze=True`` to run ``ANALYZE`` against the table after deleting the rows. + .. _python_api_upsert: Upserting data @@ -2204,6 +2207,35 @@ You can create a unique index by passing ``unique=True``: Use ``if_not_exists=True`` to do nothing if an index with that name already exists. +Pass ``analyze=True`` to run ``ANALYZE`` against the new index after creating it. + +.. _python_api_analyze: + +Optimizing index usage with ANALYZE +=================================== + +The `SQLite ANALYZE command `__ builds a table of statistics which the query planner can use to make better decisions about which indexes to use for a given query. + +You should run ``ANALYZE`` if your database is large and you do not think your indexes are being efficiently used. + +To run ``ANALYZE`` against every index in a database, use this: + +.. code-block:: python + + db.analyze() + +To run it just against a specific named index, pass the name of the index to that method: + +.. code-block:: python + + db.analyze("idx_countries_country_name") + +To run against all indexes attached to a specific table, you can either pass the table name to ``db.analyze(...)`` or you can call the method directly on the table, like this: + +.. code-block:: python + + db["dogs"].analyze() + .. _python_api_vacuum: Vacuum diff --git a/setup.py b/setup.py index 901725d..4c514e0 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import io import os -VERSION = "3.20" +VERSION = "3.21" def get_long_description(): diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 05a7bcc..0dd7f71 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -304,6 +304,26 @@ def rebuild_fts(path, tables, load_extension): db[table].rebuild_fts() +@cli.command() +@click.argument( + "path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("names", nargs=-1) +def analyze(path, names): + """Run ANALYZE against the whole database, or against specific named indexes and tables""" + db = sqlite_utils.Database(path) + try: + if names: + for name in names: + db.analyze(name) + else: + db.analyze() + except sqlite3.OperationalError as e: + raise click.ClickException(e) + + @cli.command() @click.argument( "path", @@ -470,8 +490,15 @@ def index_foreign_keys(path, load_extension): default=False, is_flag=True, ) +@click.option( + "--analyze", + help="Run ANALYZE after creating the index", + is_flag=True, +) @load_extension_option -def create_index(path, table, column, name, unique, if_not_exists, load_extension): +def create_index( + path, table, column, name, unique, if_not_exists, analyze, load_extension +): """ Add an index to the specified table covering the specified columns. Use "sqlite-utils create-index mydb -- -column" to specify descending @@ -486,7 +513,11 @@ def create_index(path, table, column, name, unique, if_not_exists, load_extensio col = DescIndex(col[1:]) columns.append(col) db[table].create_index( - columns, index_name=name, unique=unique, if_not_exists=if_not_exists + columns, + index_name=name, + unique=unique, + if_not_exists=if_not_exists, + analyze=analyze, ) @@ -629,6 +660,50 @@ def reset_counts(path, load_extension): db.reset_counts() +_import_options = ( + click.option( + "--flatten", + is_flag=True, + help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}', + ), + click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), + click.option("-c", "--csv", is_flag=True, help="Expect CSV input"), + click.option("--tsv", is_flag=True, help="Expect TSV input"), + click.option( + "--lines", + is_flag=True, + help="Treat each line as a single value called 'line'", + ), + click.option( + "--text", + is_flag=True, + help="Treat input as a single value called 'text'", + ), + click.option("--convert", help="Python code to convert each item"), + click.option( + "--import", + "imports", + type=str, + multiple=True, + help="Python modules to import", + ), + click.option("--delimiter", help="Delimiter to use for CSV files"), + click.option("--quotechar", help="Quote character to use for CSV/TSV"), + click.option("--sniff", is_flag=True, help="Detect delimiter and quote character"), + click.option("--no-headers", is_flag=True, help="CSV file has no header row"), + click.option( + "--encoding", + help="Character encoding for input, defaults to utf-8", + ), +) + + +def import_options(fn): + for decorator in reversed(_import_options): + fn = decorator(fn) + return fn + + def insert_upsert_options(fn): for decorator in reversed( ( @@ -642,40 +717,9 @@ def insert_upsert_options(fn): click.option( "--pk", help="Columns to use as the primary key, e.g. id", multiple=True ), - click.option( - "--flatten", - is_flag=True, - help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}', - ), - click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), - click.option("-c", "--csv", is_flag=True, help="Expect CSV input"), - click.option("--tsv", is_flag=True, help="Expect TSV input"), - click.option( - "--lines", - is_flag=True, - help="Treat each line as a single value called 'line'", - ), - click.option( - "--text", - is_flag=True, - help="Treat input as a single value called 'text'", - ), - click.option("--convert", help="Python code to convert each item"), - click.option( - "--import", - "imports", - type=str, - multiple=True, - help="Python modules to import", - ), - click.option("--delimiter", help="Delimiter to use for CSV files"), - click.option("--quotechar", help="Quote character to use for CSV/TSV"), - click.option( - "--sniff", is_flag=True, help="Detect delimiter and quote character" - ), - click.option( - "--no-headers", is_flag=True, help="CSV file has no header row" - ), + ) + + _import_options + + ( click.option( "--batch-size", type=int, default=100, help="Commit every X records" ), @@ -695,10 +739,6 @@ def insert_upsert_options(fn): type=(str, str), help="Default value that should be set for a column", ), - click.option( - "--encoding", - help="Character encoding for input, defaults to utf-8", - ), click.option( "-d", "--detect-types", @@ -706,6 +746,11 @@ def insert_upsert_options(fn): envvar="SQLITE_UTILS_DETECT_TYPES", help="Detect types for columns in CSV/TSV data", ), + click.option( + "--analyze", + is_flag=True, + help="Run ANALYZE at the end of this operation", + ), load_extension_option, click.option("--silent", is_flag=True, help="Do not show progress bar"), ) @@ -731,6 +776,7 @@ def insert_upsert_implementation( quotechar, sniff, no_headers, + encoding, batch_size, alter, upsert, @@ -739,10 +785,11 @@ def insert_upsert_implementation( truncate=False, not_null=None, default=None, - encoding=None, detect_types=None, + analyze=False, load_extension=None, silent=False, + bulk_sql=None, ): db = sqlite_utils.Database(path) _load_extensions(db, load_extension) @@ -833,7 +880,12 @@ def insert_upsert_implementation( else: docs = (fn(doc) or doc for doc in docs) - extra_kwargs = {"ignore": ignore, "replace": replace, "truncate": truncate} + extra_kwargs = { + "ignore": ignore, + "replace": replace, + "truncate": truncate, + "analyze": analyze, + } if not_null: extra_kwargs["not_null"] = set(not_null) if default: @@ -844,6 +896,12 @@ def insert_upsert_implementation( # Apply {"$base64": true, ...} decoding, if needed docs = (decode_base64_values(doc) for doc in docs) + # For bulk_sql= we use cursor.executemany() instead + if bulk_sql: + with db.conn: + db.conn.cursor().executemany(bulk_sql, docs) + return + try: db[table].insert_all( docs, pk=pk, batch_size=batch_size, alter=alter, **extra_kwargs @@ -926,10 +984,11 @@ def insert( quotechar, sniff, no_headers, + encoding, batch_size, alter, - encoding, detect_types, + analyze, load_extension, silent, ignore, @@ -977,14 +1036,15 @@ def insert( quotechar, sniff, no_headers, + encoding, batch_size, alter=alter, upsert=False, ignore=ignore, replace=replace, truncate=truncate, - encoding=encoding, detect_types=detect_types, + analyze=analyze, load_extension=load_extension, silent=silent, not_null=not_null, @@ -1014,11 +1074,12 @@ def upsert( quotechar, sniff, no_headers, + encoding, alter, not_null, default, - encoding, detect_types, + analyze, load_extension, silent, ): @@ -1045,13 +1106,14 @@ def upsert( quotechar, sniff, no_headers, + encoding, batch_size, alter=alter, upsert=True, not_null=not_null, default=default, - encoding=encoding, detect_types=detect_types, + analyze=analyze, load_extension=load_extension, silent=silent, ) @@ -1059,6 +1121,71 @@ def upsert( raise click.ClickException(UNICODE_ERROR.format(ex)) +@cli.command() +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("sql") +@click.argument("file", type=click.File("rb"), required=True) +@import_options +@load_extension_option +def bulk( + path, + file, + sql, + flatten, + nl, + csv, + tsv, + lines, + text, + convert, + imports, + delimiter, + quotechar, + sniff, + no_headers, + encoding, + load_extension, +): + """ + Execute parameterized SQL against the provided list of documents. + """ + try: + insert_upsert_implementation( + path=path, + table=None, + file=file, + pk=None, + flatten=flatten, + nl=nl, + csv=csv, + tsv=tsv, + lines=lines, + text=text, + convert=convert, + imports=imports, + delimiter=delimiter, + quotechar=quotechar, + sniff=sniff, + no_headers=no_headers, + encoding=encoding, + batch_size=1, + alter=False, + upsert=False, + not_null=set(), + default={}, + detect_types=False, + load_extension=load_extension, + silent=False, + bulk_sql=sql, + ) + except (sqlite3.OperationalError, sqlite3.IntegrityError) as e: + raise click.ClickException(str(e)) + + @cli.command(name="create-database") @click.argument( "path", diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index dfc4723..e40eb12 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -923,6 +923,13 @@ class Database: "Run a SQLite ``VACUUM`` against the database." self.execute("VACUUM;") + def analyze(self, name=None): + "Run ``ANALYZE`` against the entire database or a named table or index." + sql = "ANALYZE" + if name is not None: + sql += " [{}]".format(name) + self.execute(sql) + class Queryable: def exists(self) -> bool: @@ -1547,6 +1554,7 @@ class Table(Queryable): unique: bool = False, if_not_exists: bool = False, find_unique_name: bool = False, + analyze: bool = False, ): """ Create an index on this table. @@ -1558,6 +1566,7 @@ class Table(Queryable): - ``if_not_exists`` - only create the index if one with that name does not already exist. - ``find_unique_name`` - if ``index_name`` is not provided and the automatically derived name already exists, keep incrementing a suffix number to find an available name. + - ``analyze`` - run ``ANALYZE`` against this index after creating it. See :ref:`python_api_create_index`. """ @@ -1574,7 +1583,11 @@ class Table(Queryable): columns_sql.append(fmt.format(column)) suffix = None + created_index_name = None while True: + created_index_name = ( + "{}_{}".format(index_name, suffix) if suffix else index_name + ) sql = ( textwrap.dedent( """ @@ -1584,9 +1597,7 @@ class Table(Queryable): ) .strip() .format( - index_name="{}_{}".format(index_name, suffix) - if suffix - else index_name, + index_name=created_index_name, table_name=self.name, columns=", ".join(columns_sql), unique="UNIQUE " if unique else "", @@ -1611,6 +1622,8 @@ class Table(Queryable): continue else: raise e + if analyze: + self.db.analyze(created_index_name) return self def add_column( @@ -2106,15 +2119,28 @@ class Table(Queryable): return self def delete_where( - self, where: str = None, where_args: Optional[Union[Iterable, dict]] = None + self, + where: str = None, + where_args: Optional[Union[Iterable, dict]] = None, + analyze: bool = False, ) -> "Table": - "Delete rows matching specified where clause, or delete all rows in the table." + """ + Delete rows matching the specified where clause, or delete all rows in the table. + + - ``where`` - a SQL fragment to use as a ``WHERE`` clause, for example ``age > ?`` or ``age > :age``. + - ``where_args`` - a list of arguments (if using ``?``) or a dictionary (if using ``:age``). + - ``analyze`` - set to ``True`` to run ``ANALYZE`` after the rows have been deleted. + + See :ref:`python_api_delete_where`. + """ if not self.exists(): return self sql = "delete from [{}]".format(self.name) if where is not None: sql += " where " + where self.db.execute(sql, where_args or []) + if analyze: + self.analyze() return self def update( @@ -2562,10 +2588,13 @@ class Table(Queryable): conversions=DEFAULT, columns=DEFAULT, upsert=False, + analyze=False, ) -> "Table": """ Like ``.insert()`` but takes a list of records and ensures that the table that it creates (if table does not exist) has columns for ALL of that data. + + Use ``analyze=True`` to run ``ANALYZE`` after the insert has completed. """ pk = self.value_or_default("pk", pk) foreign_keys = self.value_or_default("foreign_keys", foreign_keys) @@ -2658,6 +2687,9 @@ class Table(Queryable): ignore, ) + if analyze: + self.analyze() + return self def upsert( @@ -2708,6 +2740,7 @@ class Table(Queryable): extracts=DEFAULT, conversions=DEFAULT, columns=DEFAULT, + analyze=False, ) -> "Table": """ Like ``.upsert()`` but can be applied to a list of records. @@ -2726,6 +2759,7 @@ class Table(Queryable): conversions=conversions, columns=columns, upsert=True, + analyze=analyze, ) def add_missing_columns(self, records: Iterable[Dict[str, Any]]) -> "Table": @@ -2902,6 +2936,10 @@ class Table(Queryable): ) return self + def analyze(self): + "Run ANALYZE against this table" + self.db.analyze(self.name) + def analyze_column( self, column: str, common_limit: int = 10, value_truncate=None, total_rows=None ) -> "ColumnDetails": diff --git a/tests/test_analyze.py b/tests/test_analyze.py new file mode 100644 index 0000000..a47c8be --- /dev/null +++ b/tests/test_analyze.py @@ -0,0 +1,45 @@ +import pytest + + +@pytest.fixture +def db(fresh_db): + fresh_db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id") + fresh_db["one_index"].create_index(["name"]) + fresh_db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") + fresh_db["two_indexes"].create_index(["name"]) + fresh_db["two_indexes"].create_index(["species"]) + return fresh_db + + +def test_analyze_whole_database(db): + assert set(db.table_names()) == {"one_index", "two_indexes"} + db.analyze() + assert set(db.table_names()) == {"one_index", "two_indexes", "sqlite_stat1"} + assert list(db["sqlite_stat1"].rows) == [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, + {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, + ] + + +@pytest.mark.parametrize("method", ("db_method_with_name", "table_method")) +def test_analyze_one_table(db, method): + assert set(db.table_names()) == {"one_index", "two_indexes"} + if method == "db_method_with_name": + db.analyze("one_index") + elif method == "table_method": + db["one_index"].analyze() + + assert set(db.table_names()) == {"one_index", "two_indexes", "sqlite_stat1"} + assert list(db["sqlite_stat1"].rows) == [ + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"} + ] + + +def test_analyze_index_by_name(db): + assert set(db.table_names()) == {"one_index", "two_indexes"} + db.analyze("idx_two_indexes_species") + assert set(db.table_names()) == {"one_index", "two_indexes", "sqlite_stat1"} + assert list(db["sqlite_stat1"].rows) == [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, + ] diff --git a/tests/test_cli.py b/tests/test_cli.py index ca5c820..5f095a6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -224,6 +224,17 @@ def test_create_index(db_path): ) +def test_create_index_analyze(db_path): + db = Database(db_path) + assert "sqlite_stat1" not in db.table_names() + assert [] == db["Gosh"].indexes + result = CliRunner().invoke( + cli.cli, ["create-index", db_path, "Gosh", "c1", "--analyze"] + ) + assert result.exit_code == 0 + assert "sqlite_stat1" in db.table_names() + + def test_create_index_desc(db_path): db = Database(db_path) assert [] == db["Gosh"].indexes @@ -889,6 +900,20 @@ def test_upsert(db_path, tmpdir): ] +def test_upsert_analyze(db_path, tmpdir): + db = Database(db_path) + db["rows"].insert({"id": 1, "foo": "x", "n": 3}, pk="id") + db["rows"].create_index(["n"]) + assert "sqlite_stat1" not in db.table_names() + result = CliRunner().invoke( + cli.cli, + ["upsert", db_path, "rows", "-", "--nl", "--analyze", "--pk", "id"], + input='{"id": 2, "foo": "bar", "n": 1}', + ) + assert 0 == result.exit_code, result.output + assert "sqlite_stat1" in db.table_names() + + def test_upsert_flatten(tmpdir): db_path = str(tmpdir / "flat.db") db = Database(db_path) @@ -2057,3 +2082,41 @@ def test_create_database(tmpdir, enable_wal): assert db.journal_mode == "wal" else: assert db.journal_mode == "delete" + + +@pytest.mark.parametrize( + "options,expected", + ( + ( + [], + [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, + {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, + ], + ), + ( + ["one_index"], + [ + {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, + ], + ), + ( + ["idx_two_indexes_name"], + [ + {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, + ], + ), + ), +) +def test_analyze(tmpdir, options, expected): + db_path = str(tmpdir / "test.db") + db = Database(db_path) + db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id") + db["one_index"].create_index(["name"]) + db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") + db["two_indexes"].create_index(["name"]) + db["two_indexes"].create_index(["species"]) + result = CliRunner().invoke(cli.cli, ["analyze", db_path] + options) + assert result.exit_code == 0 + assert list(db["sqlite_stat1"].rows) == expected diff --git a/tests/test_cli_bulk.py b/tests/test_cli_bulk.py new file mode 100644 index 0000000..37cf60b --- /dev/null +++ b/tests/test_cli_bulk.py @@ -0,0 +1,57 @@ +from click.testing import CliRunner +from sqlite_utils import cli, Database +import pathlib +import pytest + + +@pytest.fixture +def test_db_and_path(tmpdir): + db_path = str(pathlib.Path(tmpdir) / "data.db") + db = Database(db_path) + db["example"].insert_all( + [ + {"id": 1, "name": "One"}, + {"id": 2, "name": "Two"}, + ], + pk="id", + ) + return db, db_path + + +def test_cli_bulk(test_db_and_path): + db, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "bulk", + db_path, + "insert into example (id, name) values (:id, :name)", + "-", + "--nl", + ], + input='{"id": 3, "name": "Three"}\n{"id": 4, "name": "Four"}\n', + ) + assert result.exit_code == 0, result.output + assert [ + {"id": 1, "name": "One"}, + {"id": 2, "name": "Two"}, + {"id": 3, "name": "Three"}, + {"id": 4, "name": "Four"}, + ] == list(db["example"].rows) + + +def test_cli_bulk_error(test_db_and_path): + _, db_path = test_db_and_path + result = CliRunner().invoke( + cli.cli, + [ + "bulk", + db_path, + "insert into example (id, name) value (:id, :name)", + "-", + "--nl", + ], + input='{"id": 3, "name": "Three"}', + ) + assert result.exit_code == 1 + assert result.output == 'Error: near "value": syntax error\n' diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index efa55bd..0d6b482 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -327,6 +327,20 @@ def test_insert_alter(db_path, tmpdir): ] == list(db.query("select foo, n, baz from from_json_nl")) +def test_insert_analyze(db_path): + db = Database(db_path) + db["rows"].insert({"foo": "x", "n": 3}) + db["rows"].create_index(["n"]) + assert "sqlite_stat1" not in db.table_names() + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "rows", "-", "--nl", "--analyze"], + input='{"foo": "bar", "n": 1}\n{"foo": "baz", "n": 2}', + ) + assert 0 == result.exit_code, result.output + assert "sqlite_stat1" in db.table_names() + + def test_insert_lines(db_path): result = CliRunner().invoke( cli.cli, diff --git a/tests/test_create.py b/tests/test_create.py index 187d20a..82c2c72 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -774,6 +774,17 @@ def test_create_index_find_unique_name(fresh_db): assert index_names == {"idx_t_id", "idx_t_id_2", "idx_t_id_3"} +def test_create_index_analyze(fresh_db): + dogs = fresh_db["dogs"] + assert "sqlite_stat1" not in fresh_db.table_names() + dogs.insert({"name": "Cleo", "twitter": "cleopaws"}) + dogs.create_index(["name"], analyze=True) + assert "sqlite_stat1" in fresh_db.table_names() + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "dogs", "idx": "idx_dogs_name", "stat": "1 1"} + ] + + @pytest.mark.parametrize( "data_structure", ( @@ -1017,6 +1028,23 @@ def test_insert_all_single_column(fresh_db): assert table.pks == ["name"] +@pytest.mark.parametrize("method_name", ("insert_all", "upsert_all")) +def test_insert_all_analyze(fresh_db, method_name): + table = fresh_db["table"] + table.insert_all([{"id": 1, "name": "Cleo"}], pk="id") + assert "sqlite_stat1" not in fresh_db.table_names() + table.create_index(["name"], analyze=True) + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_name", "stat": "1 1"} + ] + method = getattr(table, method_name) + method([{"id": 2, "name": "Suna"}], pk="id", analyze=True) + assert "sqlite_stat1" in fresh_db.table_names() + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_name", "stat": "2 1"} + ] + + def test_create_with_a_null_column(fresh_db): record = {"name": "Name", "description": None} fresh_db["t"].insert(record) diff --git a/tests/test_delete.py b/tests/test_delete.py index 1198d06..f057749 100644 --- a/tests/test_delete.py +++ b/tests/test_delete.py @@ -30,3 +30,17 @@ def test_delete_where_all(fresh_db): assert 10 == table.count table.delete_where() assert 0 == table.count + + +def test_delete_where_analyze(fresh_db): + table = fresh_db["table"] + table.insert_all(({"id": i, "i": i} for i in range(10)), pk="id") + table.create_index(["i"], analyze=True) + assert "sqlite_stat1" in fresh_db.table_names() + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_i", "stat": "10 1"} + ] + table.delete_where("id > ?", [5], analyze=True) + assert list(fresh_db["sqlite_stat1"].rows) == [ + {"tbl": "table", "idx": "idx_table_i", "stat": "6 1"} + ]